Admin: list and restart running generations with the models set now
CI / test (push) Successful in 1m17s
CI / build (push) Successful in 5m43s
CI / deploy (push) Skipped

- Modellek page: analyses and answers being written or waiting, with their model
  and the one a restart would use; restart one or all
- Each attempt's started_at is its token: a restarted attempt still running in the
  worker cannot write its result and is dropped within 10 seconds

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012vLHzh5qUhFtmJhzkcTbXL
This commit is contained in:
feco
2026-11-18 21:31:09 +00:00
co-authored by Claude Opus 5.5
parent 769b4791ae
commit b5743ec7d1
13 changed files with 492 additions and 51 deletions
+36 -15
View File
@@ -27,6 +27,8 @@ pub struct QuestionJob {
pub created_at: time::OffsetDateTime,
pub fallback_model: Option<String>,
pub on_fallback: bool,
/// The token of this attempt (see `Job::started_at`).
pub started_at: time::OffsetDateTime,
}
impl Analyzer {
@@ -37,7 +39,7 @@ impl Analyzer {
WHERE id = (SELECT id FROM questions WHERE status = 'pending' AND next_attempt_at <= now() \
ORDER BY next_attempt_at LIMIT 1 FOR UPDATE SKIP LOCKED) \
RETURNING id, analysis_id, question, language, model, prompt_version, attempts, created_at, \
fallback_model, on_fallback",
fallback_model, on_fallback, started_at",
)
.fetch_optional(&self.db)
.await?;
@@ -91,15 +93,24 @@ impl Analyzer {
/// Answers one claimed question and records the outcome.
pub async fn handle_question(&self, job: QuestionJob) {
let started = Instant::now();
let result = match self.question_prompt(&job).await {
Ok(p) => {
let model = active_model(&job.model, job.fallback_model.as_deref(), job.on_fallback);
self.llm.complete(model, &p).await.map(|mut c| {
prompt::polish(&job.language, &mut c.content);
c
})
let work = async {
match self.question_prompt(&job).await {
Ok(p) => {
let model = active_model(&job.model, job.fallback_model.as_deref(), job.on_fallback);
self.llm.complete(model, &p).await.map(|mut c| {
prompt::polish(&job.language, &mut c.content);
c
})
}
Err(e) => Err(e),
}
};
let result = tokio::select! {
result = work => result,
() = self.superseded("questions", job.id, job.started_at) => {
tracing::info!(id = %job.id, "answer restarted by an admin; this attempt is dropped");
return;
}
Err(e) => Err(e),
};
if let Err(e) = self.finish_question(&job, result, started.elapsed()).await {
tracing::error!(id = %job.id, error = %e, "saving the answer failed");
@@ -119,7 +130,7 @@ impl Analyzer {
sqlx::query(
"UPDATE questions SET status = 'done', answer = $2, model_used = $3, input_tokens = $4, \
output_tokens = $5, cost_usd = $6, error = NULL, completed_at = now() \
WHERE id = $1 AND status = 'running'",
WHERE id = $1 AND status = 'running' AND started_at = $7",
)
.bind(job.id)
.bind(&answer)
@@ -127,6 +138,7 @@ impl Analyzer {
.bind(c.input_tokens)
.bind(c.output_tokens)
.bind(c.cost_usd)
.bind(job.started_at)
.execute(&self.db)
.await?;
tracing::info!(
@@ -140,11 +152,13 @@ impl Analyzer {
Next::Retry(wait) => {
sqlx::query(
"UPDATE questions SET status = 'pending', error = $2, \
next_attempt_at = now() + make_interval(secs => $3) WHERE id = $1 AND status = 'running'",
next_attempt_at = now() + make_interval(secs => $3) \
WHERE id = $1 AND status = 'running' AND started_at = $4",
)
.bind(job.id)
.bind(e.message())
.bind(wait.as_secs_f64())
.bind(job.started_at)
.execute(&self.db)
.await?;
tracing::warn!(id = %job.id, attempts = job.attempts, error = %e.message(), "answer will be retried");
@@ -152,10 +166,11 @@ impl Analyzer {
Next::Fallback => {
sqlx::query(
"UPDATE questions SET status = 'pending', error = $2, on_fallback = true, attempts = 0, \
next_attempt_at = now() WHERE id = $1 AND status = 'running'",
next_attempt_at = now() WHERE id = $1 AND status = 'running' AND started_at = $3",
)
.bind(job.id)
.bind(e.message())
.bind(job.started_at)
.execute(&self.db)
.await?;
tracing::warn!(
@@ -164,7 +179,7 @@ impl Analyzer {
);
}
Next::Fail => {
self.fail_question(job.id, e.message()).await?;
self.fail_question(job.id, job.started_at, e.message()).await?;
tracing::error!(id = %job.id, attempts = job.attempts, error = %e.message(), "answer failed");
}
}
@@ -174,14 +189,20 @@ impl Analyzer {
}
/// Marks the question failed and gives its unit back (once).
async fn fail_question(&self, id: Uuid, message: &str) -> Result<(), sqlx::Error> {
async fn fail_question(
&self,
id: Uuid,
started_at: time::OffsetDateTime,
message: &str,
) -> Result<(), sqlx::Error> {
let mut tx = self.db.begin().await?;
let row: Option<(Option<Uuid>, i32)> = sqlx::query_as(
"UPDATE questions SET status = 'failed', error = $2, completed_at = now() \
WHERE id = $1 AND status = 'running' RETURNING asked_by, credits_charged",
WHERE id = $1 AND status = 'running' AND started_at = $3 RETURNING asked_by, credits_charged",
)
.bind(id)
.bind(message)
.bind(started_at)
.fetch_optional(&mut *tx)
.await?;
if let Some((Some(user), units)) = row