Logs: own folders, a daily cap, masked paths; workers finish running work on a deploy
Review fixes for the persistent logs: - each service writes its own folder (the old-file cleanup matches by prefix, so 'worker' deleted the 'worker-ai' files); - at most LOG_MAX_DAY_MB (50) a day, and at least 2 days kept; - written by a background thread, not on the async workers; - e-mail addresses and tokens in request paths are masked; - a failing health check is logged again, a passing one is not; - one-off commands (compare-models) log to the console only. On a stop the worker takes nothing new and lets running work finish (SHUTDOWN_DRAIN_SECS, worker 40 s, worker-ai 270 s, with matching stop grace periods), so a deploy no longer cuts an analysis off, wastes its model call and delays it by the 15-minute stuck-job requeue. The CI cleanup caps the build cache at 10 GB: by age it had grown to 33 GB on the runner and filled its disk. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012vLHzh5qUhFtmJhzkcTbXL
This commit is contained in:
@@ -84,12 +84,12 @@ async fn main() -> anyhow::Result<()> {
|
||||
match std::env::args().nth(1).as_deref() {
|
||||
Some("healthcheck") => return service_kit::healthcheck(addr),
|
||||
Some("compare-models") => {
|
||||
service_kit::init_logging("worker");
|
||||
service_kit::init_console_logging();
|
||||
return compare::main(std::env::args().skip(2).collect()).await;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
service_kit::init_logging("worker");
|
||||
let _log = service_kit::init_logging("worker");
|
||||
let config = Config::from_env()?;
|
||||
let pool = db::connect(&config.database_url).await.context("connecting to the database")?;
|
||||
db::migrate(&pool).await.context("running database migrations")?;
|
||||
@@ -125,7 +125,12 @@ async fn main() -> anyhow::Result<()> {
|
||||
} else {
|
||||
let analyzer = analysis::Analyzer::new(pool.clone(), llm, config.max_analysis_attempts, lanes);
|
||||
tracing::info!(quick = lanes.quick, slow = lanes.slow, "analyses enabled");
|
||||
jobs.spawn(analysis::run(Arc::new(analyzer), config.poll_interval, stop.clone()));
|
||||
jobs.spawn(analysis::run(
|
||||
Arc::new(analyzer),
|
||||
config.poll_interval,
|
||||
config.shutdown_drain,
|
||||
stop.clone(),
|
||||
));
|
||||
}
|
||||
}
|
||||
None => tracing::warn!("OPENROUTER_API_KEY is not set: analyses stay pending"),
|
||||
@@ -158,10 +163,16 @@ async fn main() -> anyhow::Result<()> {
|
||||
() = service_kit::shutdown_signal() => {}
|
||||
}
|
||||
let _ = stop_tx.send(true);
|
||||
// Let jobs finish the batch they are working on.
|
||||
let finished = tokio::time::timeout(Duration::from_secs(30), async { while jobs.join_next().await.is_some() {} });
|
||||
// Nothing new is taken; running work finishes (analyses hand back what is
|
||||
// not done within the drain time, a few seconds before this gives up).
|
||||
tracing::info!(drain_secs = config.shutdown_drain.as_secs(), "stopping: letting running work finish");
|
||||
let finished = tokio::time::timeout(config.shutdown_drain + Duration::from_secs(5), async {
|
||||
while jobs.join_next().await.is_some() {}
|
||||
});
|
||||
if finished.await.is_err() {
|
||||
tracing::warn!("jobs did not stop in time");
|
||||
} else {
|
||||
tracing::info!("all work finished");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user