Worker: separate containers for e-mail and AI generation
WORKER_TASKS (email, cleanup, analyses; all by default) picks the jobs of a worker process. In production the worker sends e-mails and cleans up (and will take the other background jobs), worker-ai writes the analyses and answers with its quick and slow lanes, sized from the server's .env. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012vLHzh5qUhFtmJhzkcTbXL
This commit is contained in:
@@ -77,6 +77,7 @@ Ugyanazt a radix-elemzést több modellel is megírja, véletlenszerű betűkkel
|
||||
| `OPENROUTER_API_KEY` | – | OpenRouter-kulcs (compose-ban a `.env.secrets` fájlból); nélküle az elemzések várnak |
|
||||
| `OPENROUTER_BASE_URL` | `https://openrouter.ai/api/v1` | |
|
||||
| `PUBLIC_BASE_URL` | – | `HTTP-Referer` fejléc az OpenRouter felé |
|
||||
| `WORKER_TASKS` | `email,cleanup,analyses` | a folyamat feladatai; élesben `worker`: `email,cleanup`, `worker-ai`: `analyses` |
|
||||
| `ANALYSIS_CONCURRENCY` | `4` | lassú sáv: egyszerre írt részletes elemzések és előrejelzések (0: kikapcsolva) |
|
||||
| `QUICK_CONCURRENCY` | `2` | gyors sáv: egyszerre írt rövid összefoglalók és válaszok (0: kikapcsolva) |
|
||||
| `MAX_ANALYSIS_ATTEMPTS` | `3` | ennyi próbálkozás után sikertelen az elemzés (az egység visszajár) |
|
||||
|
||||
@@ -5,7 +5,38 @@ use std::time::Duration;
|
||||
use anyhow::Context;
|
||||
use lettre::message::Mailbox;
|
||||
|
||||
/// The jobs a worker process runs (`WORKER_TASKS`, comma-separated).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct Tasks {
|
||||
/// Sending the e-mails of `email_outbox`.
|
||||
pub email: bool,
|
||||
/// The hourly removal of expired data.
|
||||
pub cleanup: bool,
|
||||
/// Writing the analyses and answers with the LLM (both lanes).
|
||||
pub analyses: bool,
|
||||
}
|
||||
|
||||
impl Tasks {
|
||||
pub const ALL: Tasks = Tasks { email: true, cleanup: true, analyses: true };
|
||||
|
||||
pub fn parse(raw: &str) -> anyhow::Result<Self> {
|
||||
let mut tasks = Tasks { email: false, cleanup: false, analyses: false };
|
||||
for name in raw.split(',').map(str::trim).filter(|n| !n.is_empty()) {
|
||||
match name {
|
||||
"email" => tasks.email = true,
|
||||
"cleanup" => tasks.cleanup = true,
|
||||
"analyses" => tasks.analyses = true,
|
||||
other => anyhow::bail!("unknown task '{other}' in WORKER_TASKS (email, cleanup, analyses)"),
|
||||
}
|
||||
}
|
||||
anyhow::ensure!(tasks != Tasks { email: false, cleanup: false, analyses: false }, "WORKER_TASKS names no task");
|
||||
Ok(tasks)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct Config {
|
||||
/// What this process does; all of it by default.
|
||||
pub tasks: Tasks,
|
||||
pub database_url: String,
|
||||
/// `smtp://host:port` (plain, e.g. Mailpit), `smtp://user:pass@host:587?tls=required`
|
||||
/// (STARTTLS) or `smtps://...` (TLS). Without it, Amazon SES from the AWS
|
||||
@@ -57,6 +88,7 @@ impl Config {
|
||||
},
|
||||
};
|
||||
Ok(Self {
|
||||
tasks: var("WORKER_TASKS").map(|raw| Tasks::parse(&raw)).transpose()?.unwrap_or(Tasks::ALL),
|
||||
database_url: var("DATABASE_URL").context("DATABASE_URL is required")?,
|
||||
smtp_url,
|
||||
smtp_about,
|
||||
|
||||
@@ -5,6 +5,9 @@
|
||||
//! * hourly cleanup of expired sessions, tokens, invites and old e-mails
|
||||
//! * writing the requested text analyses with an LLM (OpenRouter)
|
||||
//!
|
||||
//! `WORKER_TASKS` (e.g. `email,cleanup` or `analyses`) picks the jobs of one
|
||||
//! process, so they can run in separate containers; all of them by default.
|
||||
//!
|
||||
//! Configuration: see [`config::Config`]; plus `BIND_ADDR` (default
|
||||
//! `0.0.0.0:8082`, only `/healthz`), `LOG_FORMAT` and `RUST_LOG`.
|
||||
//! `worker healthcheck` performs a local health check (for Docker);
|
||||
@@ -91,7 +94,10 @@ async fn main() -> anyhow::Result<()> {
|
||||
|
||||
let (stop_tx, stop) = watch::channel(false);
|
||||
let mut jobs = tokio::task::JoinSet::new();
|
||||
let tasks = config.tasks;
|
||||
tracing::info!(email = tasks.email, cleanup = tasks.cleanup, analyses = tasks.analyses, "worker tasks");
|
||||
match &config.smtp_url {
|
||||
_ if !tasks.email => {}
|
||||
Some(url) => {
|
||||
let delivery = email::Delivery::new(pool.clone(), email::Mailer::smtp(url)?, &config);
|
||||
let via = config.smtp_about.clone().unwrap_or_else(|| "SMTP_URL".into());
|
||||
@@ -101,6 +107,7 @@ async fn main() -> anyhow::Result<()> {
|
||||
None => tracing::warn!("neither SMTP_URL nor SES credentials are set: e-mails stay in the outbox"),
|
||||
}
|
||||
match &config.openrouter_api_key {
|
||||
_ if !tasks.analyses => {}
|
||||
Some(key) => {
|
||||
let llm =
|
||||
analysis::openrouter::OpenRouter::new(&config.openrouter_base_url, key, config.public_base_url.clone());
|
||||
@@ -115,7 +122,9 @@ async fn main() -> anyhow::Result<()> {
|
||||
}
|
||||
None => tracing::warn!("OPENROUTER_API_KEY is not set: analyses stay pending"),
|
||||
}
|
||||
jobs.spawn(cleanup::run(pool.clone(), stop.clone()));
|
||||
if tasks.cleanup {
|
||||
jobs.spawn(cleanup::run(pool.clone(), stop.clone()));
|
||||
}
|
||||
|
||||
let app = Router::new().route("/healthz", get(healthz)).with_state(pool);
|
||||
let listener = tokio::net::TcpListener::bind(addr).await.with_context(|| format!("binding {addr}"))?;
|
||||
|
||||
@@ -223,6 +223,16 @@ async fn cleanup_removes_only_expired_data(db: PgPool) {
|
||||
|
||||
// --- analyses ----------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn worker_tasks_are_chosen_by_name() {
|
||||
use crate::config::Tasks;
|
||||
let mail = Tasks::parse("email, cleanup").unwrap();
|
||||
assert_eq!(mail, Tasks { email: true, cleanup: true, analyses: false });
|
||||
assert_eq!(Tasks::parse("analyses").unwrap(), Tasks { email: false, cleanup: false, analyses: true });
|
||||
assert!(Tasks::parse("emails").is_err());
|
||||
assert!(Tasks::parse(" , ").is_err());
|
||||
}
|
||||
|
||||
mod analyses {
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user