From da533e8fa991c67e9713c695ee99d52d50da4348 Mon Sep 17 00:00:00 2001 From: feco Date: Thu, 19 Nov 2026 06:51:45 +0000 Subject: [PATCH] Worker: separate containers for e-mail and AI generation WORKER_TASKS (email, cleanup, analyses; all by default) picks the jobs of a worker process. In production the worker sends e-mails and cleans up (and will take the other background jobs), worker-ai writes the analyses and answers with its quick and slow lanes, sized from the server's .env. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012vLHzh5qUhFtmJhzkcTbXL --- deploy/.env.example | 4 ++++ deploy/README.md | 3 ++- deploy/docker-compose.yml | 22 ++++++++++++++++++++++ services/worker/README.md | 1 + services/worker/src/config.rs | 32 ++++++++++++++++++++++++++++++++ services/worker/src/main.rs | 11 ++++++++++- services/worker/src/tests.rs | 10 ++++++++++ 7 files changed, 81 insertions(+), 2 deletions(-) diff --git a/deploy/.env.example b/deploy/.env.example index 93915f8..350f4ca 100644 --- a/deploy/.env.example +++ b/deploy/.env.example @@ -20,3 +20,7 @@ SIGNUP_MODE=invite # Image version; every deployment rewrites it TAG=latest + +# worker-ai: analyses and answers written at the same time. +# QUICK_CONCURRENCY=2 # summaries and answers +# ANALYSIS_CONCURRENCY=4 # detailed analyses and forecasts diff --git a/deploy/README.md b/deploy/README.md index 55d210a..00abbda 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -82,7 +82,8 @@ A Google OAuth kliensnél a `https://sorsfonalak.hu` címet fel kell venni az en ## Üzemeltetés -- Naplók: `docker compose logs -f backend worker` +- Naplók: `docker compose logs -f backend worker worker-ai` +- A worker két konténerben fut, ugyanabból az image-ből: a `worker` küldi az e-maileket és takarít (`WORKER_TASKS=email,cleanup`, ide kerülnek a további háttérfeladatok is), a `worker-ai` írja az elemzéseket és a válaszokat (`WORKER_TASKS=analyses`, gyors és lassú sáv). A sávok mérete a szerver `.env` fájljában állítható (`QUICK_CONCURRENCY`, `ANALYSIS_CONCURRENCY`). - Visszaállás korábbi verzióra: `.env`-ben `TAG=`, utána `docker compose up -d` - Mentés: naponta egy `pg_dump` a `/opt/sorsfonalak/backups` mappába, 14 napig megmarad. A mentéseket érdemes a szerverről is elvinni (DigitalOcean backup, vagy `rsync` otthonra). - Visszatöltés: `gunzip -c backups/sorsfonalak-.sql.gz | docker compose exec -T postgres psql -U sorsfonalak sorsfonalak` diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 5312f76..03761a2 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -99,10 +99,12 @@ services: logging: *logging restart: unless-stopped + # E-mail delivery, cleanup and the other background jobs. worker: image: ${REGISTRY_PREFIX:-registry.sagifer.com/sagifer/sorsfonalak}-worker:${TAG:-latest} environment: DATABASE_URL: postgres://sorsfonalak:${POSTGRES_PASSWORD}@postgres:5432/sorsfonalak + WORKER_TASKS: email,cleanup # SMTP_URL or the SES credentials come from .env.secrets (docs/levelkuldes.md); # listing them here with an empty default would override that file. SES_REGION: ${SES_REGION:-eu-central-1} @@ -118,6 +120,26 @@ services: logging: *logging restart: unless-stopped + # The same image writing the analyses and answers (quick and slow lane); + # OPENROUTER_API_KEY comes from .env.secrets. + worker-ai: + image: ${REGISTRY_PREFIX:-registry.sagifer.com/sagifer/sorsfonalak}-worker:${TAG:-latest} + environment: + DATABASE_URL: postgres://sorsfonalak:${POSTGRES_PASSWORD}@postgres:5432/sorsfonalak + WORKER_TASKS: analyses + QUICK_CONCURRENCY: ${QUICK_CONCURRENCY:-2} + ANALYSIS_CONCURRENCY: ${ANALYSIS_CONCURRENCY:-4} + PUBLIC_BASE_URL: https://${DOMAIN} + env_file: + - path: .env.secrets + required: false + depends_on: + postgres: + condition: service_healthy + read_only: true + logging: *logging + restart: unless-stopped + frontend: image: ${REGISTRY_PREFIX:-registry.sagifer.com/sagifer/sorsfonalak}-frontend:${TAG:-latest} environment: diff --git a/services/worker/README.md b/services/worker/README.md index c270144..ee5c528 100644 --- a/services/worker/README.md +++ b/services/worker/README.md @@ -77,6 +77,7 @@ Ugyanazt a radix-elemzést több modellel is megírja, véletlenszerű betűkkel | `OPENROUTER_API_KEY` | – | OpenRouter-kulcs (compose-ban a `.env.secrets` fájlból); nélküle az elemzések várnak | | `OPENROUTER_BASE_URL` | `https://openrouter.ai/api/v1` | | | `PUBLIC_BASE_URL` | – | `HTTP-Referer` fejléc az OpenRouter felé | +| `WORKER_TASKS` | `email,cleanup,analyses` | a folyamat feladatai; élesben `worker`: `email,cleanup`, `worker-ai`: `analyses` | | `ANALYSIS_CONCURRENCY` | `4` | lassú sáv: egyszerre írt részletes elemzések és előrejelzések (0: kikapcsolva) | | `QUICK_CONCURRENCY` | `2` | gyors sáv: egyszerre írt rövid összefoglalók és válaszok (0: kikapcsolva) | | `MAX_ANALYSIS_ATTEMPTS` | `3` | ennyi próbálkozás után sikertelen az elemzés (az egység visszajár) | diff --git a/services/worker/src/config.rs b/services/worker/src/config.rs index 5f98d8a..27030cb 100644 --- a/services/worker/src/config.rs +++ b/services/worker/src/config.rs @@ -5,7 +5,38 @@ use std::time::Duration; use anyhow::Context; use lettre::message::Mailbox; +/// The jobs a worker process runs (`WORKER_TASKS`, comma-separated). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Tasks { + /// Sending the e-mails of `email_outbox`. + pub email: bool, + /// The hourly removal of expired data. + pub cleanup: bool, + /// Writing the analyses and answers with the LLM (both lanes). + pub analyses: bool, +} + +impl Tasks { + pub const ALL: Tasks = Tasks { email: true, cleanup: true, analyses: true }; + + pub fn parse(raw: &str) -> anyhow::Result { + let mut tasks = Tasks { email: false, cleanup: false, analyses: false }; + for name in raw.split(',').map(str::trim).filter(|n| !n.is_empty()) { + match name { + "email" => tasks.email = true, + "cleanup" => tasks.cleanup = true, + "analyses" => tasks.analyses = true, + other => anyhow::bail!("unknown task '{other}' in WORKER_TASKS (email, cleanup, analyses)"), + } + } + anyhow::ensure!(tasks != Tasks { email: false, cleanup: false, analyses: false }, "WORKER_TASKS names no task"); + Ok(tasks) + } +} + pub struct Config { + /// What this process does; all of it by default. + pub tasks: Tasks, pub database_url: String, /// `smtp://host:port` (plain, e.g. Mailpit), `smtp://user:pass@host:587?tls=required` /// (STARTTLS) or `smtps://...` (TLS). Without it, Amazon SES from the AWS @@ -57,6 +88,7 @@ impl Config { }, }; Ok(Self { + tasks: var("WORKER_TASKS").map(|raw| Tasks::parse(&raw)).transpose()?.unwrap_or(Tasks::ALL), database_url: var("DATABASE_URL").context("DATABASE_URL is required")?, smtp_url, smtp_about, diff --git a/services/worker/src/main.rs b/services/worker/src/main.rs index b86f050..9594684 100644 --- a/services/worker/src/main.rs +++ b/services/worker/src/main.rs @@ -5,6 +5,9 @@ //! * hourly cleanup of expired sessions, tokens, invites and old e-mails //! * writing the requested text analyses with an LLM (OpenRouter) //! +//! `WORKER_TASKS` (e.g. `email,cleanup` or `analyses`) picks the jobs of one +//! process, so they can run in separate containers; all of them by default. +//! //! Configuration: see [`config::Config`]; plus `BIND_ADDR` (default //! `0.0.0.0:8082`, only `/healthz`), `LOG_FORMAT` and `RUST_LOG`. //! `worker healthcheck` performs a local health check (for Docker); @@ -91,7 +94,10 @@ async fn main() -> anyhow::Result<()> { let (stop_tx, stop) = watch::channel(false); let mut jobs = tokio::task::JoinSet::new(); + let tasks = config.tasks; + tracing::info!(email = tasks.email, cleanup = tasks.cleanup, analyses = tasks.analyses, "worker tasks"); match &config.smtp_url { + _ if !tasks.email => {} Some(url) => { let delivery = email::Delivery::new(pool.clone(), email::Mailer::smtp(url)?, &config); let via = config.smtp_about.clone().unwrap_or_else(|| "SMTP_URL".into()); @@ -101,6 +107,7 @@ async fn main() -> anyhow::Result<()> { None => tracing::warn!("neither SMTP_URL nor SES credentials are set: e-mails stay in the outbox"), } match &config.openrouter_api_key { + _ if !tasks.analyses => {} Some(key) => { let llm = analysis::openrouter::OpenRouter::new(&config.openrouter_base_url, key, config.public_base_url.clone()); @@ -115,7 +122,9 @@ async fn main() -> anyhow::Result<()> { } None => tracing::warn!("OPENROUTER_API_KEY is not set: analyses stay pending"), } - jobs.spawn(cleanup::run(pool.clone(), stop.clone())); + if tasks.cleanup { + jobs.spawn(cleanup::run(pool.clone(), stop.clone())); + } let app = Router::new().route("/healthz", get(healthz)).with_state(pool); let listener = tokio::net::TcpListener::bind(addr).await.with_context(|| format!("binding {addr}"))?; diff --git a/services/worker/src/tests.rs b/services/worker/src/tests.rs index a53aecc..7581279 100644 --- a/services/worker/src/tests.rs +++ b/services/worker/src/tests.rs @@ -223,6 +223,16 @@ async fn cleanup_removes_only_expired_data(db: PgPool) { // --- analyses ---------------------------------------------------------------------------- +#[test] +fn worker_tasks_are_chosen_by_name() { + use crate::config::Tasks; + let mail = Tasks::parse("email, cleanup").unwrap(); + assert_eq!(mail, Tasks { email: true, cleanup: true, analyses: false }); + assert_eq!(Tasks::parse("analyses").unwrap(), Tasks { email: false, cleanup: false, analyses: true }); + assert!(Tasks::parse("emails").is_err()); + assert!(Tasks::parse(" , ").is_err()); +} + mod analyses { use std::sync::{Arc, Mutex};