Logs: own folders, a daily cap, masked paths; workers finish running work on a deploy
Review fixes for the persistent logs: - each service writes its own folder (the old-file cleanup matches by prefix, so 'worker' deleted the 'worker-ai' files); - at most LOG_MAX_DAY_MB (50) a day, and at least 2 days kept; - written by a background thread, not on the async workers; - e-mail addresses and tokens in request paths are masked; - a failing health check is logged again, a passing one is not; - one-off commands (compare-models) log to the console only. On a stop the worker takes nothing new and lets running work finish (SHUTDOWN_DRAIN_SECS, worker 40 s, worker-ai 270 s, with matching stop grace periods), so a deploy no longer cuts an analysis off, wastes its model call and delays it by the 15-minute stuck-job requeue. The CI cleanup caps the build cache at 10 GB: by age it had grown to 33 GB on the runner and filled its disk. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012vLHzh5qUhFtmJhzkcTbXL
This commit is contained in:
@@ -928,6 +928,49 @@ mod analyses {
|
||||
assert!(!user.contains("Szeged") && !user.contains("Pécs"));
|
||||
}
|
||||
|
||||
#[sqlx::test(migrator = "db::MIGRATOR")]
|
||||
async fn a_stop_lets_running_analyses_finish(db: PgPool) {
|
||||
let s = setup(&db).await;
|
||||
let id = request_analysis(&db, &s).await;
|
||||
// A model that takes a while to answer.
|
||||
let body = reply(&content(Kind::Natal)).1;
|
||||
let app = axum::Router::new().route(
|
||||
"/chat/completions",
|
||||
axum::routing::post(move || {
|
||||
let body = body.clone();
|
||||
async move {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(1500)).await;
|
||||
Json(body)
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let url = format!("http://{}", listener.local_addr().unwrap());
|
||||
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
|
||||
let a = std::sync::Arc::new(analyzer(&db, &url));
|
||||
let (stop_tx, stop) = tokio::sync::watch::channel(false);
|
||||
let drain = std::time::Duration::from_secs(10);
|
||||
let run = tokio::spawn(crate::analysis::run(a, std::time::Duration::from_secs(30), drain, stop));
|
||||
let status = || async {
|
||||
sqlx::query_scalar::<_, String>("SELECT status FROM analyses WHERE id = $1")
|
||||
.bind(id)
|
||||
.fetch_one(&db)
|
||||
.await
|
||||
.unwrap()
|
||||
};
|
||||
for _ in 0..60 {
|
||||
if status().await == "running" {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
}
|
||||
assert_eq!(status().await, "running");
|
||||
// A deploy stops the worker while the model is still writing.
|
||||
stop_tx.send(true).unwrap();
|
||||
run.await.unwrap();
|
||||
assert_eq!(status().await, "done");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrator = "db::MIGRATOR")]
|
||||
async fn tarot_readings_are_written_in_the_quick_lane_and_refunded_on_failure(db: PgPool) {
|
||||
let s = setup(&db).await;
|
||||
|
||||
Reference in New Issue
Block a user