Worker: separate containers for e-mail and AI generation
WORKER_TASKS (email, cleanup, analyses; all by default) picks the jobs of a worker process. In production the worker sends e-mails and cleans up (and will take the other background jobs), worker-ai writes the analyses and answers with its quick and slow lanes, sized from the server's .env. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012vLHzh5qUhFtmJhzkcTbXL
This commit is contained in:
@@ -20,3 +20,7 @@ SIGNUP_MODE=invite
|
||||
|
||||
# Image version; every deployment rewrites it
|
||||
TAG=latest
|
||||
|
||||
# worker-ai: analyses and answers written at the same time.
|
||||
# QUICK_CONCURRENCY=2 # summaries and answers
|
||||
# ANALYSIS_CONCURRENCY=4 # detailed analyses and forecasts
|
||||
|
||||
+2
-1
@@ -82,7 +82,8 @@ A Google OAuth kliensnél a `https://sorsfonalak.hu` címet fel kell venni az en
|
||||
|
||||
## Üzemeltetés
|
||||
|
||||
- Naplók: `docker compose logs -f backend worker`
|
||||
- Naplók: `docker compose logs -f backend worker worker-ai`
|
||||
- A worker két konténerben fut, ugyanabból az image-ből: a `worker` küldi az e-maileket és takarít (`WORKER_TASKS=email,cleanup`, ide kerülnek a további háttérfeladatok is), a `worker-ai` írja az elemzéseket és a válaszokat (`WORKER_TASKS=analyses`, gyors és lassú sáv). A sávok mérete a szerver `.env` fájljában állítható (`QUICK_CONCURRENCY`, `ANALYSIS_CONCURRENCY`).
|
||||
- Visszaállás korábbi verzióra: `.env`-ben `TAG=<régebbi hash>`, utána `docker compose up -d`
|
||||
- Mentés: naponta egy `pg_dump` a `/opt/sorsfonalak/backups` mappába, 14 napig megmarad. A mentéseket érdemes a szerverről is elvinni (DigitalOcean backup, vagy `rsync` otthonra).
|
||||
- Visszatöltés: `gunzip -c backups/sorsfonalak-<dátum>.sql.gz | docker compose exec -T postgres psql -U sorsfonalak sorsfonalak`
|
||||
|
||||
@@ -99,10 +99,12 @@ services:
|
||||
logging: *logging
|
||||
restart: unless-stopped
|
||||
|
||||
# E-mail delivery, cleanup and the other background jobs.
|
||||
worker:
|
||||
image: ${REGISTRY_PREFIX:-registry.sagifer.com/sagifer/sorsfonalak}-worker:${TAG:-latest}
|
||||
environment:
|
||||
DATABASE_URL: postgres://sorsfonalak:${POSTGRES_PASSWORD}@postgres:5432/sorsfonalak
|
||||
WORKER_TASKS: email,cleanup
|
||||
# SMTP_URL or the SES credentials come from .env.secrets (docs/levelkuldes.md);
|
||||
# listing them here with an empty default would override that file.
|
||||
SES_REGION: ${SES_REGION:-eu-central-1}
|
||||
@@ -118,6 +120,26 @@ services:
|
||||
logging: *logging
|
||||
restart: unless-stopped
|
||||
|
||||
# The same image writing the analyses and answers (quick and slow lane);
|
||||
# OPENROUTER_API_KEY comes from .env.secrets.
|
||||
worker-ai:
|
||||
image: ${REGISTRY_PREFIX:-registry.sagifer.com/sagifer/sorsfonalak}-worker:${TAG:-latest}
|
||||
environment:
|
||||
DATABASE_URL: postgres://sorsfonalak:${POSTGRES_PASSWORD}@postgres:5432/sorsfonalak
|
||||
WORKER_TASKS: analyses
|
||||
QUICK_CONCURRENCY: ${QUICK_CONCURRENCY:-2}
|
||||
ANALYSIS_CONCURRENCY: ${ANALYSIS_CONCURRENCY:-4}
|
||||
PUBLIC_BASE_URL: https://${DOMAIN}
|
||||
env_file:
|
||||
- path: .env.secrets
|
||||
required: false
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
read_only: true
|
||||
logging: *logging
|
||||
restart: unless-stopped
|
||||
|
||||
frontend:
|
||||
image: ${REGISTRY_PREFIX:-registry.sagifer.com/sagifer/sorsfonalak}-frontend:${TAG:-latest}
|
||||
environment:
|
||||
|
||||
@@ -77,6 +77,7 @@ Ugyanazt a radix-elemzést több modellel is megírja, véletlenszerű betűkkel
|
||||
| `OPENROUTER_API_KEY` | – | OpenRouter-kulcs (compose-ban a `.env.secrets` fájlból); nélküle az elemzések várnak |
|
||||
| `OPENROUTER_BASE_URL` | `https://openrouter.ai/api/v1` | |
|
||||
| `PUBLIC_BASE_URL` | – | `HTTP-Referer` fejléc az OpenRouter felé |
|
||||
| `WORKER_TASKS` | `email,cleanup,analyses` | a folyamat feladatai; élesben `worker`: `email,cleanup`, `worker-ai`: `analyses` |
|
||||
| `ANALYSIS_CONCURRENCY` | `4` | lassú sáv: egyszerre írt részletes elemzések és előrejelzések (0: kikapcsolva) |
|
||||
| `QUICK_CONCURRENCY` | `2` | gyors sáv: egyszerre írt rövid összefoglalók és válaszok (0: kikapcsolva) |
|
||||
| `MAX_ANALYSIS_ATTEMPTS` | `3` | ennyi próbálkozás után sikertelen az elemzés (az egység visszajár) |
|
||||
|
||||
@@ -5,7 +5,38 @@ use std::time::Duration;
|
||||
use anyhow::Context;
|
||||
use lettre::message::Mailbox;
|
||||
|
||||
/// The jobs a worker process runs (`WORKER_TASKS`, comma-separated).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct Tasks {
|
||||
/// Sending the e-mails of `email_outbox`.
|
||||
pub email: bool,
|
||||
/// The hourly removal of expired data.
|
||||
pub cleanup: bool,
|
||||
/// Writing the analyses and answers with the LLM (both lanes).
|
||||
pub analyses: bool,
|
||||
}
|
||||
|
||||
impl Tasks {
|
||||
pub const ALL: Tasks = Tasks { email: true, cleanup: true, analyses: true };
|
||||
|
||||
pub fn parse(raw: &str) -> anyhow::Result<Self> {
|
||||
let mut tasks = Tasks { email: false, cleanup: false, analyses: false };
|
||||
for name in raw.split(',').map(str::trim).filter(|n| !n.is_empty()) {
|
||||
match name {
|
||||
"email" => tasks.email = true,
|
||||
"cleanup" => tasks.cleanup = true,
|
||||
"analyses" => tasks.analyses = true,
|
||||
other => anyhow::bail!("unknown task '{other}' in WORKER_TASKS (email, cleanup, analyses)"),
|
||||
}
|
||||
}
|
||||
anyhow::ensure!(tasks != Tasks { email: false, cleanup: false, analyses: false }, "WORKER_TASKS names no task");
|
||||
Ok(tasks)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct Config {
|
||||
/// What this process does; all of it by default.
|
||||
pub tasks: Tasks,
|
||||
pub database_url: String,
|
||||
/// `smtp://host:port` (plain, e.g. Mailpit), `smtp://user:pass@host:587?tls=required`
|
||||
/// (STARTTLS) or `smtps://...` (TLS). Without it, Amazon SES from the AWS
|
||||
@@ -57,6 +88,7 @@ impl Config {
|
||||
},
|
||||
};
|
||||
Ok(Self {
|
||||
tasks: var("WORKER_TASKS").map(|raw| Tasks::parse(&raw)).transpose()?.unwrap_or(Tasks::ALL),
|
||||
database_url: var("DATABASE_URL").context("DATABASE_URL is required")?,
|
||||
smtp_url,
|
||||
smtp_about,
|
||||
|
||||
@@ -5,6 +5,9 @@
|
||||
//! * hourly cleanup of expired sessions, tokens, invites and old e-mails
|
||||
//! * writing the requested text analyses with an LLM (OpenRouter)
|
||||
//!
|
||||
//! `WORKER_TASKS` (e.g. `email,cleanup` or `analyses`) picks the jobs of one
|
||||
//! process, so they can run in separate containers; all of them by default.
|
||||
//!
|
||||
//! Configuration: see [`config::Config`]; plus `BIND_ADDR` (default
|
||||
//! `0.0.0.0:8082`, only `/healthz`), `LOG_FORMAT` and `RUST_LOG`.
|
||||
//! `worker healthcheck` performs a local health check (for Docker);
|
||||
@@ -91,7 +94,10 @@ async fn main() -> anyhow::Result<()> {
|
||||
|
||||
let (stop_tx, stop) = watch::channel(false);
|
||||
let mut jobs = tokio::task::JoinSet::new();
|
||||
let tasks = config.tasks;
|
||||
tracing::info!(email = tasks.email, cleanup = tasks.cleanup, analyses = tasks.analyses, "worker tasks");
|
||||
match &config.smtp_url {
|
||||
_ if !tasks.email => {}
|
||||
Some(url) => {
|
||||
let delivery = email::Delivery::new(pool.clone(), email::Mailer::smtp(url)?, &config);
|
||||
let via = config.smtp_about.clone().unwrap_or_else(|| "SMTP_URL".into());
|
||||
@@ -101,6 +107,7 @@ async fn main() -> anyhow::Result<()> {
|
||||
None => tracing::warn!("neither SMTP_URL nor SES credentials are set: e-mails stay in the outbox"),
|
||||
}
|
||||
match &config.openrouter_api_key {
|
||||
_ if !tasks.analyses => {}
|
||||
Some(key) => {
|
||||
let llm =
|
||||
analysis::openrouter::OpenRouter::new(&config.openrouter_base_url, key, config.public_base_url.clone());
|
||||
@@ -115,7 +122,9 @@ async fn main() -> anyhow::Result<()> {
|
||||
}
|
||||
None => tracing::warn!("OPENROUTER_API_KEY is not set: analyses stay pending"),
|
||||
}
|
||||
jobs.spawn(cleanup::run(pool.clone(), stop.clone()));
|
||||
if tasks.cleanup {
|
||||
jobs.spawn(cleanup::run(pool.clone(), stop.clone()));
|
||||
}
|
||||
|
||||
let app = Router::new().route("/healthz", get(healthz)).with_state(pool);
|
||||
let listener = tokio::net::TcpListener::bind(addr).await.with_context(|| format!("binding {addr}"))?;
|
||||
|
||||
@@ -223,6 +223,16 @@ async fn cleanup_removes_only_expired_data(db: PgPool) {
|
||||
|
||||
// --- analyses ----------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn worker_tasks_are_chosen_by_name() {
|
||||
use crate::config::Tasks;
|
||||
let mail = Tasks::parse("email, cleanup").unwrap();
|
||||
assert_eq!(mail, Tasks { email: true, cleanup: true, analyses: false });
|
||||
assert_eq!(Tasks::parse("analyses").unwrap(), Tasks { email: false, cleanup: false, analyses: true });
|
||||
assert!(Tasks::parse("emails").is_err());
|
||||
assert!(Tasks::parse(" , ").is_err());
|
||||
}
|
||||
|
||||
mod analyses {
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user