Worker: separate containers for e-mail and AI generation
CI / test (push) Successful in 1m9s
CI / build (push) Successful in 4m9s
CI / deploy (push) Skipped

WORKER_TASKS (email, cleanup, analyses; all by default) picks the jobs of a
worker process. In production the worker sends e-mails and cleans up (and
will take the other background jobs), worker-ai writes the analyses and
answers with its quick and slow lanes, sized from the server's .env.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012vLHzh5qUhFtmJhzkcTbXL
This commit is contained in:
feco
2026-11-19 06:51:45 +00:00
co-authored by Claude Opus 5.5
parent bc0227042b
commit da533e8fa9
7 changed files with 81 additions and 2 deletions
+4
View File
@@ -20,3 +20,7 @@ SIGNUP_MODE=invite
# Image version; every deployment rewrites it
TAG=latest
# worker-ai: analyses and answers written at the same time.
# QUICK_CONCURRENCY=2 # summaries and answers
# ANALYSIS_CONCURRENCY=4 # detailed analyses and forecasts
+2 -1
View File
@@ -82,7 +82,8 @@ A Google OAuth kliensnél a `https://sorsfonalak.hu` címet fel kell venni az en
## Üzemeltetés
- Naplók: `docker compose logs -f backend worker`
- Naplók: `docker compose logs -f backend worker worker-ai`
- A worker két konténerben fut, ugyanabból az image-ből: a `worker` küldi az e-maileket és takarít (`WORKER_TASKS=email,cleanup`, ide kerülnek a további háttérfeladatok is), a `worker-ai` írja az elemzéseket és a válaszokat (`WORKER_TASKS=analyses`, gyors és lassú sáv). A sávok mérete a szerver `.env` fájljában állítható (`QUICK_CONCURRENCY`, `ANALYSIS_CONCURRENCY`).
- Visszaállás korábbi verzióra: `.env`-ben `TAG=<régebbi hash>`, utána `docker compose up -d`
- Mentés: naponta egy `pg_dump` a `/opt/sorsfonalak/backups` mappába, 14 napig megmarad. A mentéseket érdemes a szerverről is elvinni (DigitalOcean backup, vagy `rsync` otthonra).
- Visszatöltés: `gunzip -c backups/sorsfonalak-<dátum>.sql.gz | docker compose exec -T postgres psql -U sorsfonalak sorsfonalak`
+22
View File
@@ -99,10 +99,12 @@ services:
logging: *logging
restart: unless-stopped
# E-mail delivery, cleanup and the other background jobs.
worker:
image: ${REGISTRY_PREFIX:-registry.sagifer.com/sagifer/sorsfonalak}-worker:${TAG:-latest}
environment:
DATABASE_URL: postgres://sorsfonalak:${POSTGRES_PASSWORD}@postgres:5432/sorsfonalak
WORKER_TASKS: email,cleanup
# SMTP_URL or the SES credentials come from .env.secrets (docs/levelkuldes.md);
# listing them here with an empty default would override that file.
SES_REGION: ${SES_REGION:-eu-central-1}
@@ -118,6 +120,26 @@ services:
logging: *logging
restart: unless-stopped
# The same image writing the analyses and answers (quick and slow lane);
# OPENROUTER_API_KEY comes from .env.secrets.
worker-ai:
image: ${REGISTRY_PREFIX:-registry.sagifer.com/sagifer/sorsfonalak}-worker:${TAG:-latest}
environment:
DATABASE_URL: postgres://sorsfonalak:${POSTGRES_PASSWORD}@postgres:5432/sorsfonalak
WORKER_TASKS: analyses
QUICK_CONCURRENCY: ${QUICK_CONCURRENCY:-2}
ANALYSIS_CONCURRENCY: ${ANALYSIS_CONCURRENCY:-4}
PUBLIC_BASE_URL: https://${DOMAIN}
env_file:
- path: .env.secrets
required: false
depends_on:
postgres:
condition: service_healthy
read_only: true
logging: *logging
restart: unless-stopped
frontend:
image: ${REGISTRY_PREFIX:-registry.sagifer.com/sagifer/sorsfonalak}-frontend:${TAG:-latest}
environment:
+1
View File
@@ -77,6 +77,7 @@ Ugyanazt a radix-elemzést több modellel is megírja, véletlenszerű betűkkel
| `OPENROUTER_API_KEY` | – | OpenRouter-kulcs (compose-ban a `.env.secrets` fájlból); nélküle az elemzések várnak |
| `OPENROUTER_BASE_URL` | `https://openrouter.ai/api/v1` | |
| `PUBLIC_BASE_URL` | – | `HTTP-Referer` fejléc az OpenRouter felé |
| `WORKER_TASKS` | `email,cleanup,analyses` | a folyamat feladatai; élesben `worker`: `email,cleanup`, `worker-ai`: `analyses` |
| `ANALYSIS_CONCURRENCY` | `4` | lassú sáv: egyszerre írt részletes elemzések és előrejelzések (0: kikapcsolva) |
| `QUICK_CONCURRENCY` | `2` | gyors sáv: egyszerre írt rövid összefoglalók és válaszok (0: kikapcsolva) |
| `MAX_ANALYSIS_ATTEMPTS` | `3` | ennyi próbálkozás után sikertelen az elemzés (az egység visszajár) |
+32
View File
@@ -5,7 +5,38 @@ use std::time::Duration;
use anyhow::Context;
use lettre::message::Mailbox;
/// The jobs a worker process runs (`WORKER_TASKS`, comma-separated).
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Tasks {
/// Sending the e-mails of `email_outbox`.
pub email: bool,
/// The hourly removal of expired data.
pub cleanup: bool,
/// Writing the analyses and answers with the LLM (both lanes).
pub analyses: bool,
}
impl Tasks {
pub const ALL: Tasks = Tasks { email: true, cleanup: true, analyses: true };
pub fn parse(raw: &str) -> anyhow::Result<Self> {
let mut tasks = Tasks { email: false, cleanup: false, analyses: false };
for name in raw.split(',').map(str::trim).filter(|n| !n.is_empty()) {
match name {
"email" => tasks.email = true,
"cleanup" => tasks.cleanup = true,
"analyses" => tasks.analyses = true,
other => anyhow::bail!("unknown task '{other}' in WORKER_TASKS (email, cleanup, analyses)"),
}
}
anyhow::ensure!(tasks != Tasks { email: false, cleanup: false, analyses: false }, "WORKER_TASKS names no task");
Ok(tasks)
}
}
pub struct Config {
/// What this process does; all of it by default.
pub tasks: Tasks,
pub database_url: String,
/// `smtp://host:port` (plain, e.g. Mailpit), `smtp://user:pass@host:587?tls=required`
/// (STARTTLS) or `smtps://...` (TLS). Without it, Amazon SES from the AWS
@@ -57,6 +88,7 @@ impl Config {
},
};
Ok(Self {
tasks: var("WORKER_TASKS").map(|raw| Tasks::parse(&raw)).transpose()?.unwrap_or(Tasks::ALL),
database_url: var("DATABASE_URL").context("DATABASE_URL is required")?,
smtp_url,
smtp_about,
+10 -1
View File
@@ -5,6 +5,9 @@
//! * hourly cleanup of expired sessions, tokens, invites and old e-mails
//! * writing the requested text analyses with an LLM (OpenRouter)
//!
//! `WORKER_TASKS` (e.g. `email,cleanup` or `analyses`) picks the jobs of one
//! process, so they can run in separate containers; all of them by default.
//!
//! Configuration: see [`config::Config`]; plus `BIND_ADDR` (default
//! `0.0.0.0:8082`, only `/healthz`), `LOG_FORMAT` and `RUST_LOG`.
//! `worker healthcheck` performs a local health check (for Docker);
@@ -91,7 +94,10 @@ async fn main() -> anyhow::Result<()> {
let (stop_tx, stop) = watch::channel(false);
let mut jobs = tokio::task::JoinSet::new();
let tasks = config.tasks;
tracing::info!(email = tasks.email, cleanup = tasks.cleanup, analyses = tasks.analyses, "worker tasks");
match &config.smtp_url {
_ if !tasks.email => {}
Some(url) => {
let delivery = email::Delivery::new(pool.clone(), email::Mailer::smtp(url)?, &config);
let via = config.smtp_about.clone().unwrap_or_else(|| "SMTP_URL".into());
@@ -101,6 +107,7 @@ async fn main() -> anyhow::Result<()> {
None => tracing::warn!("neither SMTP_URL nor SES credentials are set: e-mails stay in the outbox"),
}
match &config.openrouter_api_key {
_ if !tasks.analyses => {}
Some(key) => {
let llm =
analysis::openrouter::OpenRouter::new(&config.openrouter_base_url, key, config.public_base_url.clone());
@@ -115,7 +122,9 @@ async fn main() -> anyhow::Result<()> {
}
None => tracing::warn!("OPENROUTER_API_KEY is not set: analyses stay pending"),
}
jobs.spawn(cleanup::run(pool.clone(), stop.clone()));
if tasks.cleanup {
jobs.spawn(cleanup::run(pool.clone(), stop.clone()));
}
let app = Router::new().route("/healthz", get(healthz)).with_state(pool);
let listener = tokio::net::TcpListener::bind(addr).await.with_context(|| format!("binding {addr}"))?;
+10
View File
@@ -223,6 +223,16 @@ async fn cleanup_removes_only_expired_data(db: PgPool) {
// --- analyses ----------------------------------------------------------------------------
#[test]
fn worker_tasks_are_chosen_by_name() {
use crate::config::Tasks;
let mail = Tasks::parse("email, cleanup").unwrap();
assert_eq!(mail, Tasks { email: true, cleanup: true, analyses: false });
assert_eq!(Tasks::parse("analyses").unwrap(), Tasks { email: false, cleanup: false, analyses: true });
assert!(Tasks::parse("emails").is_err());
assert!(Tasks::parse(" , ").is_err());
}
mod analyses {
use std::sync::{Arc, Mutex};