Logs: own folders, a daily cap, masked paths; workers finish running work on a deploy
Review fixes for the persistent logs: - each service writes its own folder (the old-file cleanup matches by prefix, so 'worker' deleted the 'worker-ai' files); - at most LOG_MAX_DAY_MB (50) a day, and at least 2 days kept; - written by a background thread, not on the async workers; - e-mail addresses and tokens in request paths are masked; - a failing health check is logged again, a passing one is not; - one-off commands (compare-models) log to the console only. On a stop the worker takes nothing new and lets running work finish (SHUTDOWN_DRAIN_SECS, worker 40 s, worker-ai 270 s, with matching stop grace periods), so a deploy no longer cuts an analysis off, wastes its model call and delays it by the 15-minute stuck-job requeue. The CI cleanup caps the build cache at 10 GB: by age it had grown to 33 GB on the runner and filled its disk. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012vLHzh5qUhFtmJhzkcTbXL
This commit is contained in:
@@ -66,7 +66,7 @@ async fn main() -> anyhow::Result<()> {
|
||||
if std::env::args().nth(1).as_deref() == Some("healthcheck") {
|
||||
return service_kit::healthcheck(addr);
|
||||
}
|
||||
service_kit::init_logging("backend");
|
||||
let _log = service_kit::init_logging("backend");
|
||||
let config = Config::from_env()?;
|
||||
let pool = db::connect(&config.database_url).await.context("connecting to the database")?;
|
||||
db::migrate(&pool).await.context("running database migrations")?;
|
||||
|
||||
@@ -36,8 +36,7 @@ use axum::{Json, Router};
|
||||
use axum_extra::extract::WithRejection;
|
||||
use serde_json::{Value, json};
|
||||
use tower_http::cors::{AllowOrigin, CorsLayer};
|
||||
use tower_http::trace::{DefaultMakeSpan, DefaultOnResponse, TraceLayer};
|
||||
use tracing::Level;
|
||||
use tower_http::trace::TraceLayer;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::AppState;
|
||||
@@ -60,6 +59,7 @@ where
|
||||
|
||||
pub fn router(state: Arc<AppState>) -> Router {
|
||||
Router::new()
|
||||
.route("/healthz", get(health))
|
||||
.route("/v1/config", get(public_config))
|
||||
.route("/v1/packages", get(payments::catalog))
|
||||
// login
|
||||
@@ -140,13 +140,8 @@ pub fn router(state: Arc<AppState>) -> Router {
|
||||
.route("/v1/admin/analyses/{id}/regenerate", post(analyses::regenerate))
|
||||
.route("/v1/admin/questions/{id}/regenerate", post(questions::regenerate))
|
||||
.layer(cors(&state.config.cors_allowed_origins))
|
||||
.layer(
|
||||
TraceLayer::new_for_http()
|
||||
.make_span_with(DefaultMakeSpan::new().level(Level::INFO))
|
||||
.on_response(DefaultOnResponse::new().level(Level::INFO)),
|
||||
)
|
||||
// After the layers: the Docker health check (every 30 s) stays out of the log.
|
||||
.route("/healthz", get(health))
|
||||
// The request log: masked paths, health checks only when they fail.
|
||||
.layer(TraceLayer::new_for_http().make_span_with(service_kit::RequestLog).on_response(service_kit::RequestLog))
|
||||
.with_state(state)
|
||||
}
|
||||
|
||||
|
||||
@@ -54,7 +54,7 @@ async fn main() -> anyhow::Result<()> {
|
||||
if std::env::args().nth(1).as_deref() == Some("healthcheck") {
|
||||
return service_kit::healthcheck(addr);
|
||||
}
|
||||
service_kit::init_logging("horoscope-api");
|
||||
let _log = service_kit::init_logging("horoscope-api");
|
||||
let state = Arc::new(tokio::task::spawn_blocking(load_state).await??);
|
||||
let app = routes::router(state);
|
||||
let listener = tokio::net::TcpListener::bind(addr).await.with_context(|| format!("binding {addr}"))?;
|
||||
|
||||
@@ -9,8 +9,7 @@ use axum::{Json, Router};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Value, json};
|
||||
use tower_http::cors::{AllowOrigin, Any, CorsLayer};
|
||||
use tower_http::trace::{DefaultMakeSpan, DefaultOnResponse, TraceLayer};
|
||||
use tracing::Level;
|
||||
use tower_http::trace::TraceLayer;
|
||||
|
||||
use crate::AppState;
|
||||
use crate::error::ApiError;
|
||||
@@ -22,6 +21,7 @@ type AppResult<T> = Result<Json<T>, ApiError>;
|
||||
|
||||
pub fn router(state: Arc<AppState>) -> Router {
|
||||
Router::new()
|
||||
.route("/healthz", get(health))
|
||||
.route("/v1/info", get(info))
|
||||
.route("/v1/countries", get(countries))
|
||||
.route("/v1/places", get(places))
|
||||
@@ -30,13 +30,8 @@ pub fn router(state: Arc<AppState>) -> Router {
|
||||
.route("/v1/synastry", post(synastry_chart))
|
||||
.route("/v1/forecast", post(forecast_year))
|
||||
.layer(cors())
|
||||
.layer(
|
||||
TraceLayer::new_for_http()
|
||||
.make_span_with(DefaultMakeSpan::new().level(Level::INFO))
|
||||
.on_response(DefaultOnResponse::new().level(Level::INFO)),
|
||||
)
|
||||
// After the layers: the Docker health check (every 30 s) stays out of the log.
|
||||
.route("/healthz", get(health))
|
||||
// The request log: masked paths, health checks only when they fail.
|
||||
.layer(TraceLayer::new_for_http().make_span_with(service_kit::RequestLog).on_response(service_kit::RequestLog))
|
||||
.with_state(state)
|
||||
}
|
||||
|
||||
|
||||
@@ -611,7 +611,7 @@ impl Analyzer {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn run(analyzer: Arc<Analyzer>, poll: Duration, stop: watch::Receiver<bool>) {
|
||||
pub async fn run(analyzer: Arc<Analyzer>, poll: Duration, drain: Duration, stop: watch::Receiver<bool>) {
|
||||
let quick = Arc::new(Semaphore::new(analyzer.lanes.quick));
|
||||
let slow = Arc::new(Semaphore::new(analyzer.lanes.slow));
|
||||
let mut tasks = tokio::task::JoinSet::new();
|
||||
@@ -662,8 +662,12 @@ pub async fn run(analyzer: Arc<Analyzer>, poll: Duration, stop: watch::Receiver<
|
||||
() = crate::stopped(stop.clone()) => break,
|
||||
}
|
||||
}
|
||||
// Give running analyses a moment, then hand the rest back to the queue.
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), async { while tasks.join_next().await.is_some() {} }).await;
|
||||
// Let running analyses finish within the drain time, then hand the rest
|
||||
// back to the queue (without counting the attempt).
|
||||
if !tasks.is_empty() {
|
||||
tracing::info!(running = tasks.len(), drain_secs = drain.as_secs(), "waiting for running analyses");
|
||||
}
|
||||
let _ = tokio::time::timeout(drain, async { while tasks.join_next().await.is_some() {} }).await;
|
||||
tasks.abort_all();
|
||||
analyzer.release_running().await;
|
||||
}
|
||||
|
||||
@@ -52,6 +52,11 @@ pub struct Config {
|
||||
pub mail_from: Mailbox,
|
||||
/// Fallback polling interval; new e-mails are picked up at once via LISTEN/NOTIFY.
|
||||
pub poll_interval: Duration,
|
||||
/// On a stop (a deploy): how long running work may still finish, taking
|
||||
/// nothing new (`SHUTDOWN_DRAIN_SECS`, default 20); analyses not done by
|
||||
/// then go back to the queue. Keep it below the container's stop grace
|
||||
/// period (`stop_grace_period` in compose, Docker's default is 10 s).
|
||||
pub shutdown_drain: Duration,
|
||||
/// After this many failed attempts an e-mail is marked as failed.
|
||||
pub max_email_attempts: i32,
|
||||
/// Unset: analyses stay pending.
|
||||
@@ -107,6 +112,7 @@ impl Config {
|
||||
smtp_about,
|
||||
mail_from: mail_from.parse().with_context(|| format!("invalid MAIL_FROM '{mail_from}'"))?,
|
||||
poll_interval: Duration::from_secs(parsed("POLL_INTERVAL_SECS", 30)?),
|
||||
shutdown_drain: Duration::from_secs(parsed("SHUTDOWN_DRAIN_SECS", 20)?),
|
||||
max_email_attempts: parsed("MAX_EMAIL_ATTEMPTS", 8)?,
|
||||
// The lower-case name is accepted too (as written in some secrets files).
|
||||
openrouter_api_key: var("OPENROUTER_API_KEY").or_else(|| var("openrouter_api_key")),
|
||||
|
||||
@@ -84,12 +84,12 @@ async fn main() -> anyhow::Result<()> {
|
||||
match std::env::args().nth(1).as_deref() {
|
||||
Some("healthcheck") => return service_kit::healthcheck(addr),
|
||||
Some("compare-models") => {
|
||||
service_kit::init_logging("worker");
|
||||
service_kit::init_console_logging();
|
||||
return compare::main(std::env::args().skip(2).collect()).await;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
service_kit::init_logging("worker");
|
||||
let _log = service_kit::init_logging("worker");
|
||||
let config = Config::from_env()?;
|
||||
let pool = db::connect(&config.database_url).await.context("connecting to the database")?;
|
||||
db::migrate(&pool).await.context("running database migrations")?;
|
||||
@@ -125,7 +125,12 @@ async fn main() -> anyhow::Result<()> {
|
||||
} else {
|
||||
let analyzer = analysis::Analyzer::new(pool.clone(), llm, config.max_analysis_attempts, lanes);
|
||||
tracing::info!(quick = lanes.quick, slow = lanes.slow, "analyses enabled");
|
||||
jobs.spawn(analysis::run(Arc::new(analyzer), config.poll_interval, stop.clone()));
|
||||
jobs.spawn(analysis::run(
|
||||
Arc::new(analyzer),
|
||||
config.poll_interval,
|
||||
config.shutdown_drain,
|
||||
stop.clone(),
|
||||
));
|
||||
}
|
||||
}
|
||||
None => tracing::warn!("OPENROUTER_API_KEY is not set: analyses stay pending"),
|
||||
@@ -158,10 +163,16 @@ async fn main() -> anyhow::Result<()> {
|
||||
() = service_kit::shutdown_signal() => {}
|
||||
}
|
||||
let _ = stop_tx.send(true);
|
||||
// Let jobs finish the batch they are working on.
|
||||
let finished = tokio::time::timeout(Duration::from_secs(30), async { while jobs.join_next().await.is_some() {} });
|
||||
// Nothing new is taken; running work finishes (analyses hand back what is
|
||||
// not done within the drain time, a few seconds before this gives up).
|
||||
tracing::info!(drain_secs = config.shutdown_drain.as_secs(), "stopping: letting running work finish");
|
||||
let finished = tokio::time::timeout(config.shutdown_drain + Duration::from_secs(5), async {
|
||||
while jobs.join_next().await.is_some() {}
|
||||
});
|
||||
if finished.await.is_err() {
|
||||
tracing::warn!("jobs did not stop in time");
|
||||
} else {
|
||||
tracing::info!("all work finished");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -928,6 +928,49 @@ mod analyses {
|
||||
assert!(!user.contains("Szeged") && !user.contains("Pécs"));
|
||||
}
|
||||
|
||||
#[sqlx::test(migrator = "db::MIGRATOR")]
|
||||
async fn a_stop_lets_running_analyses_finish(db: PgPool) {
|
||||
let s = setup(&db).await;
|
||||
let id = request_analysis(&db, &s).await;
|
||||
// A model that takes a while to answer.
|
||||
let body = reply(&content(Kind::Natal)).1;
|
||||
let app = axum::Router::new().route(
|
||||
"/chat/completions",
|
||||
axum::routing::post(move || {
|
||||
let body = body.clone();
|
||||
async move {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(1500)).await;
|
||||
Json(body)
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let url = format!("http://{}", listener.local_addr().unwrap());
|
||||
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
|
||||
let a = std::sync::Arc::new(analyzer(&db, &url));
|
||||
let (stop_tx, stop) = tokio::sync::watch::channel(false);
|
||||
let drain = std::time::Duration::from_secs(10);
|
||||
let run = tokio::spawn(crate::analysis::run(a, std::time::Duration::from_secs(30), drain, stop));
|
||||
let status = || async {
|
||||
sqlx::query_scalar::<_, String>("SELECT status FROM analyses WHERE id = $1")
|
||||
.bind(id)
|
||||
.fetch_one(&db)
|
||||
.await
|
||||
.unwrap()
|
||||
};
|
||||
for _ in 0..60 {
|
||||
if status().await == "running" {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
}
|
||||
assert_eq!(status().await, "running");
|
||||
// A deploy stops the worker while the model is still writing.
|
||||
stop_tx.send(true).unwrap();
|
||||
run.await.unwrap();
|
||||
assert_eq!(status().await, "done");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrator = "db::MIGRATOR")]
|
||||
async fn tarot_readings_are_written_in_the_quick_lane_and_refunded_on_failure(db: PgPool) {
|
||||
let s = setup(&db).await;
|
||||
|
||||
Reference in New Issue
Block a user