Admin: list and restart running generations with the models set now
CI / test (push) Successful in 1m17s
CI / build (push) Successful in 5m43s
CI / deploy (push) Skipped

- Modellek page: analyses and answers being written or waiting, with their model
  and the one a restart would use; restart one or all
- Each attempt's started_at is its token: a restarted attempt still running in the
  worker cannot write its result and is dropped within 10 seconds

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012vLHzh5qUhFtmJhzkcTbXL
This commit is contained in:
feco
2026-11-18 21:31:09 +00:00
co-authored by Claude Opus 5.5
parent 769b4791ae
commit b5743ec7d1
13 changed files with 492 additions and 51 deletions
+45 -13
View File
@@ -47,6 +47,9 @@ pub struct Job {
/// The default model, when the chosen one is another (see the module docs).
pub fallback_model: Option<String>,
pub on_fallback: bool,
/// When this attempt started; also its token: an admin restart clears it,
/// so a superseded attempt neither writes its result nor keeps running.
pub started_at: time::OffsetDateTime,
}
impl Job {
@@ -125,7 +128,7 @@ impl Analyzer {
WHERE id = (SELECT id FROM analyses WHERE status = 'pending' AND next_attempt_at <= now() \
ORDER BY next_attempt_at LIMIT 1 FOR UPDATE SKIP LOCKED) \
RETURNING id, kind, chart_id, synastry_id, forecast_id, language, model, prompt_version, attempts, \
fallback_model, on_fallback",
fallback_model, on_fallback, started_at",
)
.fetch_optional(&self.db)
.await?;
@@ -273,12 +276,21 @@ impl Analyzer {
/// Writes one claimed analysis and records the outcome.
pub async fn handle(&self, job: Job) {
let started = Instant::now();
let result = match self.prompt_for(&job).await {
Ok(prompt) => self.llm.complete(job.active_model(), &prompt).await.map(|mut c| {
prompt::polish(&job.language, &mut c.content);
c
}),
Err(e) => Err(e),
let work = async {
match self.prompt_for(&job).await {
Ok(prompt) => self.llm.complete(job.active_model(), &prompt).await.map(|mut c| {
prompt::polish(&job.language, &mut c.content);
c
}),
Err(e) => Err(e),
}
};
let result = tokio::select! {
result = work => result,
() = self.superseded("analyses", job.id, job.started_at) => {
tracing::info!(id = %job.id, "analysis restarted by an admin; this attempt is dropped");
return;
}
};
if let Err(e) = self.finish(&job, result, started.elapsed()).await {
tracing::error!(id = %job.id, error = %e, "saving the analysis result failed");
@@ -292,7 +304,7 @@ impl Analyzer {
sqlx::query(
"UPDATE analyses SET status = 'done', content = $2, model_used = $3, input_tokens = $4, \
output_tokens = $5, cost_usd = $6, error = NULL, completed_at = now() \
WHERE id = $1 AND status = 'running'",
WHERE id = $1 AND status = 'running' AND started_at = $7",
)
.bind(job.id)
.bind(&c.content)
@@ -300,6 +312,7 @@ impl Analyzer {
.bind(c.input_tokens)
.bind(c.output_tokens)
.bind(c.cost_usd)
.bind(job.started_at)
.execute(&self.db)
.await?;
tracing::info!(
@@ -313,11 +326,13 @@ impl Analyzer {
Next::Retry(wait) => {
sqlx::query(
"UPDATE analyses SET status = 'pending', error = $2, \
next_attempt_at = now() + make_interval(secs => $3) WHERE id = $1 AND status = 'running'",
next_attempt_at = now() + make_interval(secs => $3) \
WHERE id = $1 AND status = 'running' AND started_at = $4",
)
.bind(job.id)
.bind(e.message())
.bind(wait.as_secs_f64())
.bind(job.started_at)
.execute(&self.db)
.await?;
tracing::warn!(id = %job.id, attempts = job.attempts, retry_in_secs = wait.as_secs(), error = %e.message(), "analysis will be retried");
@@ -325,10 +340,11 @@ impl Analyzer {
Next::Fallback => {
sqlx::query(
"UPDATE analyses SET status = 'pending', error = $2, on_fallback = true, attempts = 0, \
next_attempt_at = now() WHERE id = $1 AND status = 'running'",
next_attempt_at = now() WHERE id = $1 AND status = 'running' AND started_at = $3",
)
.bind(job.id)
.bind(e.message())
.bind(job.started_at)
.execute(&self.db)
.await?;
tracing::warn!(
@@ -337,7 +353,7 @@ impl Analyzer {
);
}
Next::Fail => {
self.fail(job.id, e.message()).await?;
self.fail(job.id, job.started_at, e.message()).await?;
tracing::error!(id = %job.id, attempts = job.attempts, error = %e.message(), "analysis failed");
}
}
@@ -347,14 +363,15 @@ impl Analyzer {
}
/// Marks the analysis failed and gives its unit back (once).
async fn fail(&self, id: Uuid, message: &str) -> Result<(), sqlx::Error> {
async fn fail(&self, id: Uuid, started_at: time::OffsetDateTime, message: &str) -> Result<(), sqlx::Error> {
let mut tx = self.db.begin().await?;
let row: Option<(Option<Uuid>, i32, String)> = sqlx::query_as(
"UPDATE analyses SET status = 'failed', error = $2, completed_at = now() \
WHERE id = $1 AND status = 'running' RETURNING requested_by, credits_charged, kind",
WHERE id = $1 AND status = 'running' AND started_at = $3 RETURNING requested_by, credits_charged, kind",
)
.bind(id)
.bind(message)
.bind(started_at)
.fetch_optional(&mut *tx)
.await?;
if let Some((Some(user), units, kind)) = row
@@ -407,6 +424,21 @@ impl Analyzer {
Ok(result.rows_affected())
}
/// Resolves when this attempt is no longer the current one (an admin
/// restarted it); checked every 10 seconds, never on a database error.
async fn superseded(&self, table: &str, id: Uuid, started_at: time::OffsetDateTime) {
let query = format!("SELECT status = 'running' AND started_at = $2 FROM {table} WHERE id = $1");
loop {
tokio::time::sleep(Duration::from_secs(10)).await;
match sqlx::query_scalar::<_, Option<bool>>(&query).bind(id).bind(started_at).fetch_optional(&self.db).await
{
Ok(Some(Some(true))) => {}
Ok(_) => return,
Err(_) => {}
}
}
}
/// On shutdown: analyses still being written go back to `pending` at once.
async fn release_running(&self) {
let ids: Vec<Uuid> = self.running.lock().unwrap().drain().collect();
+36 -15
View File
@@ -27,6 +27,8 @@ pub struct QuestionJob {
pub created_at: time::OffsetDateTime,
pub fallback_model: Option<String>,
pub on_fallback: bool,
/// The token of this attempt (see `Job::started_at`).
pub started_at: time::OffsetDateTime,
}
impl Analyzer {
@@ -37,7 +39,7 @@ impl Analyzer {
WHERE id = (SELECT id FROM questions WHERE status = 'pending' AND next_attempt_at <= now() \
ORDER BY next_attempt_at LIMIT 1 FOR UPDATE SKIP LOCKED) \
RETURNING id, analysis_id, question, language, model, prompt_version, attempts, created_at, \
fallback_model, on_fallback",
fallback_model, on_fallback, started_at",
)
.fetch_optional(&self.db)
.await?;
@@ -91,15 +93,24 @@ impl Analyzer {
/// Answers one claimed question and records the outcome.
pub async fn handle_question(&self, job: QuestionJob) {
let started = Instant::now();
let result = match self.question_prompt(&job).await {
Ok(p) => {
let model = active_model(&job.model, job.fallback_model.as_deref(), job.on_fallback);
self.llm.complete(model, &p).await.map(|mut c| {
prompt::polish(&job.language, &mut c.content);
c
})
let work = async {
match self.question_prompt(&job).await {
Ok(p) => {
let model = active_model(&job.model, job.fallback_model.as_deref(), job.on_fallback);
self.llm.complete(model, &p).await.map(|mut c| {
prompt::polish(&job.language, &mut c.content);
c
})
}
Err(e) => Err(e),
}
};
let result = tokio::select! {
result = work => result,
() = self.superseded("questions", job.id, job.started_at) => {
tracing::info!(id = %job.id, "answer restarted by an admin; this attempt is dropped");
return;
}
Err(e) => Err(e),
};
if let Err(e) = self.finish_question(&job, result, started.elapsed()).await {
tracing::error!(id = %job.id, error = %e, "saving the answer failed");
@@ -119,7 +130,7 @@ impl Analyzer {
sqlx::query(
"UPDATE questions SET status = 'done', answer = $2, model_used = $3, input_tokens = $4, \
output_tokens = $5, cost_usd = $6, error = NULL, completed_at = now() \
WHERE id = $1 AND status = 'running'",
WHERE id = $1 AND status = 'running' AND started_at = $7",
)
.bind(job.id)
.bind(&answer)
@@ -127,6 +138,7 @@ impl Analyzer {
.bind(c.input_tokens)
.bind(c.output_tokens)
.bind(c.cost_usd)
.bind(job.started_at)
.execute(&self.db)
.await?;
tracing::info!(
@@ -140,11 +152,13 @@ impl Analyzer {
Next::Retry(wait) => {
sqlx::query(
"UPDATE questions SET status = 'pending', error = $2, \
next_attempt_at = now() + make_interval(secs => $3) WHERE id = $1 AND status = 'running'",
next_attempt_at = now() + make_interval(secs => $3) \
WHERE id = $1 AND status = 'running' AND started_at = $4",
)
.bind(job.id)
.bind(e.message())
.bind(wait.as_secs_f64())
.bind(job.started_at)
.execute(&self.db)
.await?;
tracing::warn!(id = %job.id, attempts = job.attempts, error = %e.message(), "answer will be retried");
@@ -152,10 +166,11 @@ impl Analyzer {
Next::Fallback => {
sqlx::query(
"UPDATE questions SET status = 'pending', error = $2, on_fallback = true, attempts = 0, \
next_attempt_at = now() WHERE id = $1 AND status = 'running'",
next_attempt_at = now() WHERE id = $1 AND status = 'running' AND started_at = $3",
)
.bind(job.id)
.bind(e.message())
.bind(job.started_at)
.execute(&self.db)
.await?;
tracing::warn!(
@@ -164,7 +179,7 @@ impl Analyzer {
);
}
Next::Fail => {
self.fail_question(job.id, e.message()).await?;
self.fail_question(job.id, job.started_at, e.message()).await?;
tracing::error!(id = %job.id, attempts = job.attempts, error = %e.message(), "answer failed");
}
}
@@ -174,14 +189,20 @@ impl Analyzer {
}
/// Marks the question failed and gives its unit back (once).
async fn fail_question(&self, id: Uuid, message: &str) -> Result<(), sqlx::Error> {
async fn fail_question(
&self,
id: Uuid,
started_at: time::OffsetDateTime,
message: &str,
) -> Result<(), sqlx::Error> {
let mut tx = self.db.begin().await?;
let row: Option<(Option<Uuid>, i32)> = sqlx::query_as(
"UPDATE questions SET status = 'failed', error = $2, completed_at = now() \
WHERE id = $1 AND status = 'running' RETURNING asked_by, credits_charged",
WHERE id = $1 AND status = 'running' AND started_at = $3 RETURNING asked_by, credits_charged",
)
.bind(id)
.bind(message)
.bind(started_at)
.fetch_optional(&mut *tx)
.await?;
if let Some((Some(user), units)) = row
+24
View File
@@ -781,4 +781,28 @@ mod analyses {
fake.requests.lock().unwrap().iter().map(|b| b["model"].as_str().unwrap().to_string()).collect();
assert_eq!(models, ["cheap/model", "cheap/model", "test/model"]);
}
#[sqlx::test(migrator = "db::MIGRATOR")]
async fn a_restarted_attempt_does_not_write_its_result(db: PgPool) {
let s = setup(&db).await;
let id = request_analysis(&db, &s).await;
let (_, url) = fake_openrouter(vec![reply(&content(Kind::Natal))]).await;
let a = analyzer(&db, &url);
let job = a.claim().await.unwrap().expect("a job");
// an admin restarts it while the worker is still writing it
sqlx::query("UPDATE analyses SET status = 'pending', started_at = NULL, attempts = 0 WHERE id = $1")
.bind(id)
.execute(&db)
.await
.unwrap();
a.handle(job).await;
let (status, content): (String, Option<Value>) =
sqlx::query_as("SELECT status, content FROM analyses WHERE id = $1").bind(id).fetch_one(&db).await.unwrap();
assert_eq!((status.as_str(), content), ("pending", None));
// the new attempt writes it
assert!(run_once(&a).await);
let status: String =
sqlx::query_scalar("SELECT status FROM analyses WHERE id = $1").bind(id).fetch_one(&db).await.unwrap();
assert_eq!(status, "done");
}
}