From 16c40864bb082aed7ce07820713c7f666c6375c9 Mon Sep 17 00:00:00 2001 From: feco Date: Fri, 20 Nov 2026 18:57:09 +0000 Subject: [PATCH] Logs: own folders, a daily cap, masked paths; workers finish running work on a deploy Review fixes for the persistent logs: - each service writes its own folder (the old-file cleanup matches by prefix, so 'worker' deleted the 'worker-ai' files); - at most LOG_MAX_DAY_MB (50) a day, and at least 2 days kept; - written by a background thread, not on the async workers; - e-mail addresses and tokens in request paths are masked; - a failing health check is logged again, a passing one is not; - one-off commands (compare-models) log to the console only. On a stop the worker takes nothing new and lets running work finish (SHUTDOWN_DRAIN_SECS, worker 40 s, worker-ai 270 s, with matching stop grace periods), so a deploy no longer cuts an analysis off, wastes its model call and delays it by the 15-minute stuck-job requeue. The CI cleanup caps the build cache at 10 GB: by age it had grown to 33 GB on the runner and filled its disk. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012vLHzh5qUhFtmJhzkcTbXL --- .gitea/workflows/ci.yml | 3 +- Cargo.lock | 2 + crates/service-kit/Cargo.toml | 2 + crates/service-kit/src/lib.rs | 244 ++++++++++++++++++++++++--- deploy/README.md | 3 +- deploy/docker-compose.yml | 7 + services/backend/src/main.rs | 2 +- services/backend/src/routes/mod.rs | 13 +- services/horoscope-api/src/main.rs | 2 +- services/horoscope-api/src/routes.rs | 13 +- services/worker/src/analysis/mod.rs | 10 +- services/worker/src/config.rs | 6 + services/worker/src/main.rs | 21 ++- services/worker/src/tests.rs | 43 +++++ 14 files changed, 315 insertions(+), 56 deletions(-) diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 69bda50..6088acd 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -134,7 +134,8 @@ jobs: docker images "$IMAGE_PREFIX-$svc" --format '{{.Repository}}:{{.Tag}}' | xargs -r docker rmi >/dev/null 2>&1 || true done docker image prune -f >/dev/null - docker builder prune -af --filter until=72h >/dev/null || true + # A size cap, not an age: a busy day builds tens of gigabytes. + docker builder prune -af --max-used-space 10GB >/dev/null || true deploy: needs: build diff --git a/Cargo.lock b/Cargo.lock index aa466d6..e7ef3da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1862,7 +1862,9 @@ name = "service-kit" version = "0.1.0" dependencies = [ "anyhow", + "http", "tokio", + "tower-http", "tracing", "tracing-appender", "tracing-subscriber", diff --git a/crates/service-kit/Cargo.toml b/crates/service-kit/Cargo.toml index 8ff806c..780c41e 100644 --- a/crates/service-kit/Cargo.toml +++ b/crates/service-kit/Cargo.toml @@ -12,3 +12,5 @@ tokio.workspace = true tracing.workspace = true tracing-subscriber.workspace = true tracing-appender.workspace = true +tower-http.workspace = true +http = "1" diff --git a/crates/service-kit/src/lib.rs b/crates/service-kit/src/lib.rs index 94ef200..b065ac1 100644 --- a/crates/service-kit/src/lib.rs +++ b/crates/service-kit/src/lib.rs @@ -3,9 +3,10 @@ //! Environment: //! * `BIND_ADDR` listen address (each service has its own default) //! * `LOG_FORMAT` `text` (default) or `json`; `RUST_LOG` sets the level -//! * `LOG_DIR` also write the log there, as JSON, one file a day -//! (`.YYYY-MM-DD.log`), keeping `LOG_KEEP_DAYS` (default 30) files -//! * `LOG_NAME` the file name prefix (default: the service's name) +//! * `LOG_DIR` also write the log there, as JSON, one file a UTC day +//! (`/.YYYY-MM-DD.log`), keeping `LOG_KEEP_DAYS` +//! (default 30) files of at most `LOG_MAX_DAY_MB` (default 50) each +//! * `LOG_NAME` the folder and file name (default: the service's name) use std::net::SocketAddr; use std::time::Duration; @@ -14,41 +15,201 @@ use anyhow::Context; /// Days of log files kept in `LOG_DIR` by default. const LOG_KEEP_DAYS: usize = 30; +/// At most this many megabytes of log a day per service by default. +const LOG_MAX_DAY_MB: u64 = 50; -/// Initializes `tracing` from `RUST_LOG` and `LOG_FORMAT`, and the log files -/// in `LOG_DIR` (named after `LOG_NAME`, else `name`), so the log outlives the -/// container. -pub fn init_logging(name: &str) { +/// Keeps the background log writer alive: hold it until `main` returns, so +/// the last lines reach the file. +#[must_use = "the log files stop being written when this is dropped"] +pub struct LogGuard(#[allow(dead_code)] Option); + +/// Logging for a one-off command: the console only, never the log files. +pub fn init_console_logging() { use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; - use tracing_subscriber::{EnvFilter, Layer, fmt}; + tracing_subscriber::registry().with(env_filter()).with(console_layer()).init(); +} - let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info,tower_http=info")); - let json = std::env::var("LOG_FORMAT").is_ok_and(|f| f.eq_ignore_ascii_case("json")); - let stdout = if json { fmt::layer().json().boxed() } else { fmt::layer().boxed() }; - let mut problem = None; +fn env_filter() -> tracing_subscriber::EnvFilter { + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info,tower_http=info")) +} + +fn console_layer() -> Box + Send + Sync> +where + S: tracing::Subscriber + for<'a> tracing_subscriber::registry::LookupSpan<'a>, +{ + use tracing_subscriber::{Layer, fmt}; + if std::env::var("LOG_FORMAT").is_ok_and(|f| f.eq_ignore_ascii_case("json")) { + fmt::layer().json().boxed() + } else { + fmt::layer().boxed() + } +} + +/// A whole number from the environment; a value that is not one is reported. +fn env_number(name: &str, default: T, problems: &mut Vec) -> T { + match std::env::var(name) { + Ok(raw) if !raw.trim().is_empty() => raw.trim().parse().unwrap_or_else(|_| { + problems.push(format!("{name}={raw} is not a whole number; using the default")); + default + }), + _ => default, + } +} + +/// Initializes `tracing` from `RUST_LOG` and `LOG_FORMAT`, and the log files +/// in `LOG_DIR`, so the log outlives the container: each service writes its +/// own folder `LOG_DIR//` (`` = `LOG_NAME`, else `name`), one +/// file a UTC day, keeping `LOG_KEEP_DAYS` files (at least 2), at most +/// `LOG_MAX_DAY_MB` megabytes a day. +pub fn init_logging(name: &str) -> LogGuard { + use tracing_subscriber::layer::SubscriberExt; + use tracing_subscriber::util::SubscriberInitExt; + use tracing_subscriber::{Layer, fmt}; + + let mut problems = Vec::new(); + let mut guard = None; let file = std::env::var("LOG_DIR").ok().filter(|d| !d.trim().is_empty()).and_then(|dir| { - let prefix = - std::env::var("LOG_NAME").ok().filter(|n| !n.trim().is_empty()).unwrap_or_else(|| name.to_string()); - let keep = std::env::var("LOG_KEEP_DAYS").ok().and_then(|d| d.parse().ok()).unwrap_or(LOG_KEEP_DAYS); - let appender = tracing_appender::rolling::Builder::new() - .rotation(tracing_appender::rolling::Rotation::DAILY) - .filename_prefix(prefix) - .filename_suffix("log") - .max_log_files(keep.max(1)) - .build(&dir); + let name = std::env::var("LOG_NAME").ok().filter(|n| !n.trim().is_empty()).unwrap_or_else(|| name.to_string()); + let keep: usize = env_number("LOG_KEEP_DAYS", LOG_KEEP_DAYS, &mut problems); + if keep < 2 { + problems.push(format!("LOG_KEEP_DAYS={keep} would delete today's file on a restart; keeping 2 days")); + } + let max_mb: u64 = env_number("LOG_MAX_DAY_MB", LOG_MAX_DAY_MB, &mut problems); + // Its own folder: the old-file cleanup takes every file whose name + // starts with the prefix ("worker" would take "worker-ai" files too). + let folder = std::path::Path::new(&dir).join(&name); + let appender = std::fs::create_dir_all(&folder).map_err(|e| e.to_string()).and_then(|()| { + tracing_appender::rolling::Builder::new() + .rotation(tracing_appender::rolling::Rotation::DAILY) + .filename_prefix(&name) + .filename_suffix("log") + .max_log_files(keep.max(2)) + .build(&folder) + .map_err(|e| e.to_string()) + }); match appender { - Ok(appender) => Some(fmt::layer().json().with_ansi(false).with_writer(appender).boxed()), + Ok(appender) => { + let (writer, g) = tracing_appender::non_blocking(DailyCap::new(appender, max_mb.max(1) * 1024 * 1024)); + guard = Some(g); + Some(fmt::layer().json().with_ansi(false).with_writer(writer).boxed()) + } Err(e) => { - problem = Some(format!("cannot write the log files in {dir}: {e}")); + problems + .push(format!("cannot write the log files in {}: {e}; logging to stdout only", folder.display())); None } } }); - tracing_subscriber::registry().with(filter).with(stdout).with(file).init(); - if let Some(problem) = problem { - tracing::warn!("{problem}; logging to stdout only"); + tracing_subscriber::registry().with(env_filter()).with(console_layer()).with(file).init(); + for problem in problems { + tracing::warn!("{problem}"); } + LogGuard(guard) +} + +/// A daily size limit in front of the log file: past `limit` bytes in a UTC +/// day the rest of that day's lines are dropped (after one note saying so), +/// so a log storm cannot fill the disk. +struct DailyCap { + inner: W, + limit: u64, + day: u64, + written: u64, +} + +impl DailyCap { + fn new(inner: W, limit: u64) -> Self { + Self { inner, limit, day: utc_day(), written: 0 } + } +} + +fn utc_day() -> u64 { + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_secs() / 86_400) +} + +impl std::io::Write for DailyCap { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let today = utc_day(); + if today != self.day { + self.day = today; + self.written = 0; + } + if self.written >= self.limit { + return Ok(buf.len()); + } + self.written += buf.len() as u64; + if self.written >= self.limit { + let note = format!( + "{{\"level\":\"WARN\",\"fields\":{{\"message\":\"daily log limit of {} MB reached; the rest of today is only on stdout\"}}}}\n", + self.limit / 1024 / 1024 + ); + self.inner.write_all(buf)?; + self.inner.write_all(note.as_bytes())?; + return Ok(buf.len()); + } + self.inner.write(buf) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.inner.flush() + } +} + +/// The request log of an HTTP service (`TraceLayer::make_span_with` and +/// `on_response`): the Docker health check stays out of it unless it fails, +/// and e-mail addresses and tokens in the path are masked (the log is kept). +#[derive(Debug, Clone, Copy, Default)] +pub struct RequestLog; + +impl tower_http::trace::MakeSpan for RequestLog { + fn make_span(&mut self, request: &http::Request) -> tracing::Span { + let path = request.uri().path(); + if path == "/healthz" { + return tracing::Span::none(); + } + tracing::info_span!("request", method = %request.method(), path = %masked_path(path)) + } +} + +impl tower_http::trace::OnResponse for RequestLog { + fn on_response(self, response: &http::Response, latency: Duration, span: &tracing::Span) { + // A health check (no span) that went well is not logged. + if span.is_none() && response.status().is_success() { + return; + } + let ms = latency.as_millis() as u64; + let status = response.status().as_u16(); + if span.is_none() { + tracing::warn!(path = "/healthz", status, latency_ms = ms, "health check failed"); + } else { + tracing::info!(status, latency_ms = ms, "finished processing request"); + } + } +} + +/// The path with secrets masked: a segment holding an e-mail address, and a +/// long one that is not a UUID (share slugs, invitation tokens). Queries are +/// never logged. +pub fn masked_path(path: &str) -> String { + let uuid = |s: &str| { + s.len() == 36 + && s.char_indices() + .all(|(i, c)| if [8, 13, 18, 23].contains(&i) { c == '-' } else { c.is_ascii_hexdigit() }) + }; + path.split('/') + .map(|segment| { + if segment.contains('@') || segment.contains("%40") { + ":email" + } else if segment.len() >= 12 && !uuid(segment) { + ":token" + } else { + segment + } + }) + .collect::>() + .join("/") } /// Listen address from `BIND_ADDR`, or `default`. @@ -92,3 +253,34 @@ pub async fn shutdown_signal() { } tracing::info!("shutting down"); } + +#[cfg(test)] +mod tests { + use std::io::Write; + + use super::*; + + #[test] + fn secrets_in_paths_are_masked() { + assert_eq!(masked_path("/v1/admin/waitlist/anna%40example.com"), "/v1/admin/waitlist/:email"); + assert_eq!(masked_path("/v1/public/shares/Ab3dEf6hIj9kLm2n/visit"), "/v1/public/shares/:token/visit"); + assert_eq!(masked_path("/v1/public/invites/k3jh4k5j6h7k8j9h0k1j2h3k"), "/v1/public/invites/:token"); + let id = "01a1f960-96cb-765f-899a-4d76b45b236e"; + assert_eq!(masked_path(&format!("/v1/tarot/readings/{id}")), format!("/v1/tarot/readings/{id}")); + assert_eq!(masked_path("/v1/admin/prompts/forecast/hu"), "/v1/admin/prompts/forecast/hu"); + } + + #[test] + fn the_daily_cap_drops_the_rest_of_the_day() { + let mut cap = DailyCap::new(Vec::new(), 10); + cap.write_all(b"12345\n").unwrap(); + cap.write_all(b"67890\n").unwrap(); + cap.write_all(b"dropped\n").unwrap(); + let text = String::from_utf8(cap.inner.clone()).unwrap(); + assert!(text.starts_with("12345\n67890\n") && text.contains("daily log limit") && !text.contains("dropped")); + // A new day starts again. + cap.day -= 1; + cap.write_all(b"next\n").unwrap(); + assert!(String::from_utf8(cap.inner).unwrap().ends_with("next\n")); + } +} diff --git a/deploy/README.md b/deploy/README.md index 77601a2..7c49ae2 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -83,7 +83,8 @@ A Google OAuth kliensnél a `https://sorsfonalak.hu` címet fel kell venni az en ## Üzemeltetés - Naplók élőben: `docker compose logs -f backend worker worker-ai` (ez a konténer újraindításáig/cseréjéig él). -- Naplók utólag: a backend és a két worker JSON-sorokat ír a `logs` volume-ba is, naponta új fájlba (`backend.2026-11-20.log`, `worker…`, `worker-ai…`), 30 napig megőrizve (`LOG_DIR`, `LOG_KEEP_DAYS`, `LOG_NAME`). A fájlok helye a szerveren: `docker volume inspect -f '{{.Mountpoint}}' sorsfonalak_logs`, például `grep -h '"level":"ERROR"' /var/lib/docker/volumes/sorsfonalak_logs/_data/backend.*.log | tail`. A Docker 30 másodpercenkénti `/healthz` ellenőrzése nem kerül a naplóba. +- Naplók utólag: a backend és a két worker JSON-sorokat ír a `logs` volume-ba is, szolgáltatásonként külön mappába, UTC-nap szerint új fájlba (`backend/backend.2026-11-20.log`, `worker/…`, `worker-ai/…`; a nap magyar idő szerint hajnali 1-2 órakor fordul). 30 napig maradnak meg, naponta legfeljebb 50 MB (`LOG_KEEP_DAYS`, `LOG_MAX_DAY_MB`, `LOG_DIR`, `LOG_NAME`); a kérések útvonalában az e-mail-címek és a tokenek maszkolva vannak. A fájlok helye: `docker volume inspect -f '{{.Mountpoint}}' sorsfonalak_logs`, például `grep -h '"level":"ERROR"' /var/lib/docker/volumes/sorsfonalak_logs/_data/backend/*.log | tail`. A Docker 30 másodpercenkénti `/healthz` ellenőrzése csak akkor kerül a naplóba, ha hibás. +- Deploy és futó feladatok: leállításkor a worker nem vesz fel újat, a futókat befejezi (`SHUTDOWN_DRAIN_SECS`: `worker` 40 s, `worker-ai` 270 s, `stop_grace_period` 1, illetve 5 perc); ami addig nem készül el, visszakerül a sorba, a próbálkozás nem számít. Ha éppen elemzés készül, a deploy ennyivel tovább tarthat. - A worker két konténerben fut, ugyanabból az image-ből: a `worker` küldi az e-maileket és takarít (`WORKER_TASKS=email,cleanup`, ide kerülnek a további háttérfeladatok is), a `worker-ai` írja az elemzéseket és a válaszokat (`WORKER_TASKS=analyses`, gyors és lassú sáv). A sávok mérete a szerver `.env` fájljában állítható (`QUICK_CONCURRENCY`, `ANALYSIS_CONCURRENCY`). - Visszaállás korábbi verzióra: `.env`-ben `TAG=`, utána `docker compose up -d` - Mentés: naponta egy `pg_dump` a `/opt/sorsfonalak/backups` mappába, 14 napig megmarad. A mentéseket érdemes a szerverről is elvinni (DigitalOcean backup, vagy `rsync` otthonra). diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 9a7b0bf..02761c5 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -134,6 +134,8 @@ services: MAIL_FROM: ${MAIL_FROM:-Sorsfonalak } PUBLIC_BASE_URL: https://${DOMAIN} LOG_DIR: /logs + # On a deploy the e-mails and invoices being sent finish first. + SHUTDOWN_DRAIN_SECS: 40 env_file: - path: .env.secrets required: false @@ -143,6 +145,7 @@ services: read_only: true volumes: - logs:/logs + stop_grace_period: 1m logging: *logging restart: unless-stopped @@ -158,6 +161,9 @@ services: PUBLIC_BASE_URL: https://${DOMAIN} LOG_DIR: /logs LOG_NAME: worker-ai + # On a deploy the analyses being written finish first (a detailed one + # takes a few minutes); what is not done by then goes back to the queue. + SHUTDOWN_DRAIN_SECS: 270 env_file: - path: .env.secrets required: false @@ -167,6 +173,7 @@ services: read_only: true volumes: - logs:/logs + stop_grace_period: 5m logging: *logging restart: unless-stopped diff --git a/services/backend/src/main.rs b/services/backend/src/main.rs index 88b6281..c2e83c6 100644 --- a/services/backend/src/main.rs +++ b/services/backend/src/main.rs @@ -66,7 +66,7 @@ async fn main() -> anyhow::Result<()> { if std::env::args().nth(1).as_deref() == Some("healthcheck") { return service_kit::healthcheck(addr); } - service_kit::init_logging("backend"); + let _log = service_kit::init_logging("backend"); let config = Config::from_env()?; let pool = db::connect(&config.database_url).await.context("connecting to the database")?; db::migrate(&pool).await.context("running database migrations")?; diff --git a/services/backend/src/routes/mod.rs b/services/backend/src/routes/mod.rs index 7d90b75..87d3c99 100644 --- a/services/backend/src/routes/mod.rs +++ b/services/backend/src/routes/mod.rs @@ -36,8 +36,7 @@ use axum::{Json, Router}; use axum_extra::extract::WithRejection; use serde_json::{Value, json}; use tower_http::cors::{AllowOrigin, CorsLayer}; -use tower_http::trace::{DefaultMakeSpan, DefaultOnResponse, TraceLayer}; -use tracing::Level; +use tower_http::trace::TraceLayer; use uuid::Uuid; use crate::AppState; @@ -60,6 +59,7 @@ where pub fn router(state: Arc) -> Router { Router::new() + .route("/healthz", get(health)) .route("/v1/config", get(public_config)) .route("/v1/packages", get(payments::catalog)) // login @@ -140,13 +140,8 @@ pub fn router(state: Arc) -> Router { .route("/v1/admin/analyses/{id}/regenerate", post(analyses::regenerate)) .route("/v1/admin/questions/{id}/regenerate", post(questions::regenerate)) .layer(cors(&state.config.cors_allowed_origins)) - .layer( - TraceLayer::new_for_http() - .make_span_with(DefaultMakeSpan::new().level(Level::INFO)) - .on_response(DefaultOnResponse::new().level(Level::INFO)), - ) - // After the layers: the Docker health check (every 30 s) stays out of the log. - .route("/healthz", get(health)) + // The request log: masked paths, health checks only when they fail. + .layer(TraceLayer::new_for_http().make_span_with(service_kit::RequestLog).on_response(service_kit::RequestLog)) .with_state(state) } diff --git a/services/horoscope-api/src/main.rs b/services/horoscope-api/src/main.rs index 740bdf3..ca769aa 100644 --- a/services/horoscope-api/src/main.rs +++ b/services/horoscope-api/src/main.rs @@ -54,7 +54,7 @@ async fn main() -> anyhow::Result<()> { if std::env::args().nth(1).as_deref() == Some("healthcheck") { return service_kit::healthcheck(addr); } - service_kit::init_logging("horoscope-api"); + let _log = service_kit::init_logging("horoscope-api"); let state = Arc::new(tokio::task::spawn_blocking(load_state).await??); let app = routes::router(state); let listener = tokio::net::TcpListener::bind(addr).await.with_context(|| format!("binding {addr}"))?; diff --git a/services/horoscope-api/src/routes.rs b/services/horoscope-api/src/routes.rs index 47ee34b..3411be8 100644 --- a/services/horoscope-api/src/routes.rs +++ b/services/horoscope-api/src/routes.rs @@ -9,8 +9,7 @@ use axum::{Json, Router}; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use tower_http::cors::{AllowOrigin, Any, CorsLayer}; -use tower_http::trace::{DefaultMakeSpan, DefaultOnResponse, TraceLayer}; -use tracing::Level; +use tower_http::trace::TraceLayer; use crate::AppState; use crate::error::ApiError; @@ -22,6 +21,7 @@ type AppResult = Result, ApiError>; pub fn router(state: Arc) -> Router { Router::new() + .route("/healthz", get(health)) .route("/v1/info", get(info)) .route("/v1/countries", get(countries)) .route("/v1/places", get(places)) @@ -30,13 +30,8 @@ pub fn router(state: Arc) -> Router { .route("/v1/synastry", post(synastry_chart)) .route("/v1/forecast", post(forecast_year)) .layer(cors()) - .layer( - TraceLayer::new_for_http() - .make_span_with(DefaultMakeSpan::new().level(Level::INFO)) - .on_response(DefaultOnResponse::new().level(Level::INFO)), - ) - // After the layers: the Docker health check (every 30 s) stays out of the log. - .route("/healthz", get(health)) + // The request log: masked paths, health checks only when they fail. + .layer(TraceLayer::new_for_http().make_span_with(service_kit::RequestLog).on_response(service_kit::RequestLog)) .with_state(state) } diff --git a/services/worker/src/analysis/mod.rs b/services/worker/src/analysis/mod.rs index b053971..add9697 100644 --- a/services/worker/src/analysis/mod.rs +++ b/services/worker/src/analysis/mod.rs @@ -611,7 +611,7 @@ impl Analyzer { } } -pub async fn run(analyzer: Arc, poll: Duration, stop: watch::Receiver) { +pub async fn run(analyzer: Arc, poll: Duration, drain: Duration, stop: watch::Receiver) { let quick = Arc::new(Semaphore::new(analyzer.lanes.quick)); let slow = Arc::new(Semaphore::new(analyzer.lanes.slow)); let mut tasks = tokio::task::JoinSet::new(); @@ -662,8 +662,12 @@ pub async fn run(analyzer: Arc, poll: Duration, stop: watch::Receiver< () = crate::stopped(stop.clone()) => break, } } - // Give running analyses a moment, then hand the rest back to the queue. - let _ = tokio::time::timeout(Duration::from_secs(10), async { while tasks.join_next().await.is_some() {} }).await; + // Let running analyses finish within the drain time, then hand the rest + // back to the queue (without counting the attempt). + if !tasks.is_empty() { + tracing::info!(running = tasks.len(), drain_secs = drain.as_secs(), "waiting for running analyses"); + } + let _ = tokio::time::timeout(drain, async { while tasks.join_next().await.is_some() {} }).await; tasks.abort_all(); analyzer.release_running().await; } diff --git a/services/worker/src/config.rs b/services/worker/src/config.rs index 30caa2a..72fd724 100644 --- a/services/worker/src/config.rs +++ b/services/worker/src/config.rs @@ -52,6 +52,11 @@ pub struct Config { pub mail_from: Mailbox, /// Fallback polling interval; new e-mails are picked up at once via LISTEN/NOTIFY. pub poll_interval: Duration, + /// On a stop (a deploy): how long running work may still finish, taking + /// nothing new (`SHUTDOWN_DRAIN_SECS`, default 20); analyses not done by + /// then go back to the queue. Keep it below the container's stop grace + /// period (`stop_grace_period` in compose, Docker's default is 10 s). + pub shutdown_drain: Duration, /// After this many failed attempts an e-mail is marked as failed. pub max_email_attempts: i32, /// Unset: analyses stay pending. @@ -107,6 +112,7 @@ impl Config { smtp_about, mail_from: mail_from.parse().with_context(|| format!("invalid MAIL_FROM '{mail_from}'"))?, poll_interval: Duration::from_secs(parsed("POLL_INTERVAL_SECS", 30)?), + shutdown_drain: Duration::from_secs(parsed("SHUTDOWN_DRAIN_SECS", 20)?), max_email_attempts: parsed("MAX_EMAIL_ATTEMPTS", 8)?, // The lower-case name is accepted too (as written in some secrets files). openrouter_api_key: var("OPENROUTER_API_KEY").or_else(|| var("openrouter_api_key")), diff --git a/services/worker/src/main.rs b/services/worker/src/main.rs index bddbaeb..54956bc 100644 --- a/services/worker/src/main.rs +++ b/services/worker/src/main.rs @@ -84,12 +84,12 @@ async fn main() -> anyhow::Result<()> { match std::env::args().nth(1).as_deref() { Some("healthcheck") => return service_kit::healthcheck(addr), Some("compare-models") => { - service_kit::init_logging("worker"); + service_kit::init_console_logging(); return compare::main(std::env::args().skip(2).collect()).await; } _ => {} } - service_kit::init_logging("worker"); + let _log = service_kit::init_logging("worker"); let config = Config::from_env()?; let pool = db::connect(&config.database_url).await.context("connecting to the database")?; db::migrate(&pool).await.context("running database migrations")?; @@ -125,7 +125,12 @@ async fn main() -> anyhow::Result<()> { } else { let analyzer = analysis::Analyzer::new(pool.clone(), llm, config.max_analysis_attempts, lanes); tracing::info!(quick = lanes.quick, slow = lanes.slow, "analyses enabled"); - jobs.spawn(analysis::run(Arc::new(analyzer), config.poll_interval, stop.clone())); + jobs.spawn(analysis::run( + Arc::new(analyzer), + config.poll_interval, + config.shutdown_drain, + stop.clone(), + )); } } None => tracing::warn!("OPENROUTER_API_KEY is not set: analyses stay pending"), @@ -158,10 +163,16 @@ async fn main() -> anyhow::Result<()> { () = service_kit::shutdown_signal() => {} } let _ = stop_tx.send(true); - // Let jobs finish the batch they are working on. - let finished = tokio::time::timeout(Duration::from_secs(30), async { while jobs.join_next().await.is_some() {} }); + // Nothing new is taken; running work finishes (analyses hand back what is + // not done within the drain time, a few seconds before this gives up). + tracing::info!(drain_secs = config.shutdown_drain.as_secs(), "stopping: letting running work finish"); + let finished = tokio::time::timeout(config.shutdown_drain + Duration::from_secs(5), async { + while jobs.join_next().await.is_some() {} + }); if finished.await.is_err() { tracing::warn!("jobs did not stop in time"); + } else { + tracing::info!("all work finished"); } Ok(()) } diff --git a/services/worker/src/tests.rs b/services/worker/src/tests.rs index d48f90c..3bdc450 100644 --- a/services/worker/src/tests.rs +++ b/services/worker/src/tests.rs @@ -928,6 +928,49 @@ mod analyses { assert!(!user.contains("Szeged") && !user.contains("Pécs")); } + #[sqlx::test(migrator = "db::MIGRATOR")] + async fn a_stop_lets_running_analyses_finish(db: PgPool) { + let s = setup(&db).await; + let id = request_analysis(&db, &s).await; + // A model that takes a while to answer. + let body = reply(&content(Kind::Natal)).1; + let app = axum::Router::new().route( + "/chat/completions", + axum::routing::post(move || { + let body = body.clone(); + async move { + tokio::time::sleep(std::time::Duration::from_millis(1500)).await; + Json(body) + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let a = std::sync::Arc::new(analyzer(&db, &url)); + let (stop_tx, stop) = tokio::sync::watch::channel(false); + let drain = std::time::Duration::from_secs(10); + let run = tokio::spawn(crate::analysis::run(a, std::time::Duration::from_secs(30), drain, stop)); + let status = || async { + sqlx::query_scalar::<_, String>("SELECT status FROM analyses WHERE id = $1") + .bind(id) + .fetch_one(&db) + .await + .unwrap() + }; + for _ in 0..60 { + if status().await == "running" { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + assert_eq!(status().await, "running"); + // A deploy stops the worker while the model is still writing. + stop_tx.send(true).unwrap(); + run.await.unwrap(); + assert_eq!(status().await, "done"); + } + #[sqlx::test(migrator = "db::MIGRATOR")] async fn tarot_readings_are_written_in_the_quick_lane_and_refunded_on_failure(db: PgPool) { let s = setup(&db).await;