Admin: list and restart running generations with the models set now
- Modellek page: analyses and answers being written or waiting, with their model and the one a restart would use; restart one or all - Each attempt's started_at is its token: a restarted attempt still running in the worker cannot write its result and is dropped within 10 seconds Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012vLHzh5qUhFtmJhzkcTbXL
This commit is contained in:
@@ -47,6 +47,9 @@ pub struct Job {
|
||||
/// The default model, when the chosen one is another (see the module docs).
|
||||
pub fallback_model: Option<String>,
|
||||
pub on_fallback: bool,
|
||||
/// When this attempt started; also its token: an admin restart clears it,
|
||||
/// so a superseded attempt neither writes its result nor keeps running.
|
||||
pub started_at: time::OffsetDateTime,
|
||||
}
|
||||
|
||||
impl Job {
|
||||
@@ -125,7 +128,7 @@ impl Analyzer {
|
||||
WHERE id = (SELECT id FROM analyses WHERE status = 'pending' AND next_attempt_at <= now() \
|
||||
ORDER BY next_attempt_at LIMIT 1 FOR UPDATE SKIP LOCKED) \
|
||||
RETURNING id, kind, chart_id, synastry_id, forecast_id, language, model, prompt_version, attempts, \
|
||||
fallback_model, on_fallback",
|
||||
fallback_model, on_fallback, started_at",
|
||||
)
|
||||
.fetch_optional(&self.db)
|
||||
.await?;
|
||||
@@ -273,12 +276,21 @@ impl Analyzer {
|
||||
/// Writes one claimed analysis and records the outcome.
|
||||
pub async fn handle(&self, job: Job) {
|
||||
let started = Instant::now();
|
||||
let result = match self.prompt_for(&job).await {
|
||||
Ok(prompt) => self.llm.complete(job.active_model(), &prompt).await.map(|mut c| {
|
||||
prompt::polish(&job.language, &mut c.content);
|
||||
c
|
||||
}),
|
||||
Err(e) => Err(e),
|
||||
let work = async {
|
||||
match self.prompt_for(&job).await {
|
||||
Ok(prompt) => self.llm.complete(job.active_model(), &prompt).await.map(|mut c| {
|
||||
prompt::polish(&job.language, &mut c.content);
|
||||
c
|
||||
}),
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
};
|
||||
let result = tokio::select! {
|
||||
result = work => result,
|
||||
() = self.superseded("analyses", job.id, job.started_at) => {
|
||||
tracing::info!(id = %job.id, "analysis restarted by an admin; this attempt is dropped");
|
||||
return;
|
||||
}
|
||||
};
|
||||
if let Err(e) = self.finish(&job, result, started.elapsed()).await {
|
||||
tracing::error!(id = %job.id, error = %e, "saving the analysis result failed");
|
||||
@@ -292,7 +304,7 @@ impl Analyzer {
|
||||
sqlx::query(
|
||||
"UPDATE analyses SET status = 'done', content = $2, model_used = $3, input_tokens = $4, \
|
||||
output_tokens = $5, cost_usd = $6, error = NULL, completed_at = now() \
|
||||
WHERE id = $1 AND status = 'running'",
|
||||
WHERE id = $1 AND status = 'running' AND started_at = $7",
|
||||
)
|
||||
.bind(job.id)
|
||||
.bind(&c.content)
|
||||
@@ -300,6 +312,7 @@ impl Analyzer {
|
||||
.bind(c.input_tokens)
|
||||
.bind(c.output_tokens)
|
||||
.bind(c.cost_usd)
|
||||
.bind(job.started_at)
|
||||
.execute(&self.db)
|
||||
.await?;
|
||||
tracing::info!(
|
||||
@@ -313,11 +326,13 @@ impl Analyzer {
|
||||
Next::Retry(wait) => {
|
||||
sqlx::query(
|
||||
"UPDATE analyses SET status = 'pending', error = $2, \
|
||||
next_attempt_at = now() + make_interval(secs => $3) WHERE id = $1 AND status = 'running'",
|
||||
next_attempt_at = now() + make_interval(secs => $3) \
|
||||
WHERE id = $1 AND status = 'running' AND started_at = $4",
|
||||
)
|
||||
.bind(job.id)
|
||||
.bind(e.message())
|
||||
.bind(wait.as_secs_f64())
|
||||
.bind(job.started_at)
|
||||
.execute(&self.db)
|
||||
.await?;
|
||||
tracing::warn!(id = %job.id, attempts = job.attempts, retry_in_secs = wait.as_secs(), error = %e.message(), "analysis will be retried");
|
||||
@@ -325,10 +340,11 @@ impl Analyzer {
|
||||
Next::Fallback => {
|
||||
sqlx::query(
|
||||
"UPDATE analyses SET status = 'pending', error = $2, on_fallback = true, attempts = 0, \
|
||||
next_attempt_at = now() WHERE id = $1 AND status = 'running'",
|
||||
next_attempt_at = now() WHERE id = $1 AND status = 'running' AND started_at = $3",
|
||||
)
|
||||
.bind(job.id)
|
||||
.bind(e.message())
|
||||
.bind(job.started_at)
|
||||
.execute(&self.db)
|
||||
.await?;
|
||||
tracing::warn!(
|
||||
@@ -337,7 +353,7 @@ impl Analyzer {
|
||||
);
|
||||
}
|
||||
Next::Fail => {
|
||||
self.fail(job.id, e.message()).await?;
|
||||
self.fail(job.id, job.started_at, e.message()).await?;
|
||||
tracing::error!(id = %job.id, attempts = job.attempts, error = %e.message(), "analysis failed");
|
||||
}
|
||||
}
|
||||
@@ -347,14 +363,15 @@ impl Analyzer {
|
||||
}
|
||||
|
||||
/// Marks the analysis failed and gives its unit back (once).
|
||||
async fn fail(&self, id: Uuid, message: &str) -> Result<(), sqlx::Error> {
|
||||
async fn fail(&self, id: Uuid, started_at: time::OffsetDateTime, message: &str) -> Result<(), sqlx::Error> {
|
||||
let mut tx = self.db.begin().await?;
|
||||
let row: Option<(Option<Uuid>, i32, String)> = sqlx::query_as(
|
||||
"UPDATE analyses SET status = 'failed', error = $2, completed_at = now() \
|
||||
WHERE id = $1 AND status = 'running' RETURNING requested_by, credits_charged, kind",
|
||||
WHERE id = $1 AND status = 'running' AND started_at = $3 RETURNING requested_by, credits_charged, kind",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(message)
|
||||
.bind(started_at)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await?;
|
||||
if let Some((Some(user), units, kind)) = row
|
||||
@@ -407,6 +424,21 @@ impl Analyzer {
|
||||
Ok(result.rows_affected())
|
||||
}
|
||||
|
||||
/// Resolves when this attempt is no longer the current one (an admin
|
||||
/// restarted it); checked every 10 seconds, never on a database error.
|
||||
async fn superseded(&self, table: &str, id: Uuid, started_at: time::OffsetDateTime) {
|
||||
let query = format!("SELECT status = 'running' AND started_at = $2 FROM {table} WHERE id = $1");
|
||||
loop {
|
||||
tokio::time::sleep(Duration::from_secs(10)).await;
|
||||
match sqlx::query_scalar::<_, Option<bool>>(&query).bind(id).bind(started_at).fetch_optional(&self.db).await
|
||||
{
|
||||
Ok(Some(Some(true))) => {}
|
||||
Ok(_) => return,
|
||||
Err(_) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// On shutdown: analyses still being written go back to `pending` at once.
|
||||
async fn release_running(&self) {
|
||||
let ids: Vec<Uuid> = self.running.lock().unwrap().drain().collect();
|
||||
|
||||
@@ -27,6 +27,8 @@ pub struct QuestionJob {
|
||||
pub created_at: time::OffsetDateTime,
|
||||
pub fallback_model: Option<String>,
|
||||
pub on_fallback: bool,
|
||||
/// The token of this attempt (see `Job::started_at`).
|
||||
pub started_at: time::OffsetDateTime,
|
||||
}
|
||||
|
||||
impl Analyzer {
|
||||
@@ -37,7 +39,7 @@ impl Analyzer {
|
||||
WHERE id = (SELECT id FROM questions WHERE status = 'pending' AND next_attempt_at <= now() \
|
||||
ORDER BY next_attempt_at LIMIT 1 FOR UPDATE SKIP LOCKED) \
|
||||
RETURNING id, analysis_id, question, language, model, prompt_version, attempts, created_at, \
|
||||
fallback_model, on_fallback",
|
||||
fallback_model, on_fallback, started_at",
|
||||
)
|
||||
.fetch_optional(&self.db)
|
||||
.await?;
|
||||
@@ -91,15 +93,24 @@ impl Analyzer {
|
||||
/// Answers one claimed question and records the outcome.
|
||||
pub async fn handle_question(&self, job: QuestionJob) {
|
||||
let started = Instant::now();
|
||||
let result = match self.question_prompt(&job).await {
|
||||
Ok(p) => {
|
||||
let model = active_model(&job.model, job.fallback_model.as_deref(), job.on_fallback);
|
||||
self.llm.complete(model, &p).await.map(|mut c| {
|
||||
prompt::polish(&job.language, &mut c.content);
|
||||
c
|
||||
})
|
||||
let work = async {
|
||||
match self.question_prompt(&job).await {
|
||||
Ok(p) => {
|
||||
let model = active_model(&job.model, job.fallback_model.as_deref(), job.on_fallback);
|
||||
self.llm.complete(model, &p).await.map(|mut c| {
|
||||
prompt::polish(&job.language, &mut c.content);
|
||||
c
|
||||
})
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
};
|
||||
let result = tokio::select! {
|
||||
result = work => result,
|
||||
() = self.superseded("questions", job.id, job.started_at) => {
|
||||
tracing::info!(id = %job.id, "answer restarted by an admin; this attempt is dropped");
|
||||
return;
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
};
|
||||
if let Err(e) = self.finish_question(&job, result, started.elapsed()).await {
|
||||
tracing::error!(id = %job.id, error = %e, "saving the answer failed");
|
||||
@@ -119,7 +130,7 @@ impl Analyzer {
|
||||
sqlx::query(
|
||||
"UPDATE questions SET status = 'done', answer = $2, model_used = $3, input_tokens = $4, \
|
||||
output_tokens = $5, cost_usd = $6, error = NULL, completed_at = now() \
|
||||
WHERE id = $1 AND status = 'running'",
|
||||
WHERE id = $1 AND status = 'running' AND started_at = $7",
|
||||
)
|
||||
.bind(job.id)
|
||||
.bind(&answer)
|
||||
@@ -127,6 +138,7 @@ impl Analyzer {
|
||||
.bind(c.input_tokens)
|
||||
.bind(c.output_tokens)
|
||||
.bind(c.cost_usd)
|
||||
.bind(job.started_at)
|
||||
.execute(&self.db)
|
||||
.await?;
|
||||
tracing::info!(
|
||||
@@ -140,11 +152,13 @@ impl Analyzer {
|
||||
Next::Retry(wait) => {
|
||||
sqlx::query(
|
||||
"UPDATE questions SET status = 'pending', error = $2, \
|
||||
next_attempt_at = now() + make_interval(secs => $3) WHERE id = $1 AND status = 'running'",
|
||||
next_attempt_at = now() + make_interval(secs => $3) \
|
||||
WHERE id = $1 AND status = 'running' AND started_at = $4",
|
||||
)
|
||||
.bind(job.id)
|
||||
.bind(e.message())
|
||||
.bind(wait.as_secs_f64())
|
||||
.bind(job.started_at)
|
||||
.execute(&self.db)
|
||||
.await?;
|
||||
tracing::warn!(id = %job.id, attempts = job.attempts, error = %e.message(), "answer will be retried");
|
||||
@@ -152,10 +166,11 @@ impl Analyzer {
|
||||
Next::Fallback => {
|
||||
sqlx::query(
|
||||
"UPDATE questions SET status = 'pending', error = $2, on_fallback = true, attempts = 0, \
|
||||
next_attempt_at = now() WHERE id = $1 AND status = 'running'",
|
||||
next_attempt_at = now() WHERE id = $1 AND status = 'running' AND started_at = $3",
|
||||
)
|
||||
.bind(job.id)
|
||||
.bind(e.message())
|
||||
.bind(job.started_at)
|
||||
.execute(&self.db)
|
||||
.await?;
|
||||
tracing::warn!(
|
||||
@@ -164,7 +179,7 @@ impl Analyzer {
|
||||
);
|
||||
}
|
||||
Next::Fail => {
|
||||
self.fail_question(job.id, e.message()).await?;
|
||||
self.fail_question(job.id, job.started_at, e.message()).await?;
|
||||
tracing::error!(id = %job.id, attempts = job.attempts, error = %e.message(), "answer failed");
|
||||
}
|
||||
}
|
||||
@@ -174,14 +189,20 @@ impl Analyzer {
|
||||
}
|
||||
|
||||
/// Marks the question failed and gives its unit back (once).
|
||||
async fn fail_question(&self, id: Uuid, message: &str) -> Result<(), sqlx::Error> {
|
||||
async fn fail_question(
|
||||
&self,
|
||||
id: Uuid,
|
||||
started_at: time::OffsetDateTime,
|
||||
message: &str,
|
||||
) -> Result<(), sqlx::Error> {
|
||||
let mut tx = self.db.begin().await?;
|
||||
let row: Option<(Option<Uuid>, i32)> = sqlx::query_as(
|
||||
"UPDATE questions SET status = 'failed', error = $2, completed_at = now() \
|
||||
WHERE id = $1 AND status = 'running' RETURNING asked_by, credits_charged",
|
||||
WHERE id = $1 AND status = 'running' AND started_at = $3 RETURNING asked_by, credits_charged",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(message)
|
||||
.bind(started_at)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await?;
|
||||
if let Some((Some(user), units)) = row
|
||||
|
||||
@@ -781,4 +781,28 @@ mod analyses {
|
||||
fake.requests.lock().unwrap().iter().map(|b| b["model"].as_str().unwrap().to_string()).collect();
|
||||
assert_eq!(models, ["cheap/model", "cheap/model", "test/model"]);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrator = "db::MIGRATOR")]
|
||||
async fn a_restarted_attempt_does_not_write_its_result(db: PgPool) {
|
||||
let s = setup(&db).await;
|
||||
let id = request_analysis(&db, &s).await;
|
||||
let (_, url) = fake_openrouter(vec![reply(&content(Kind::Natal))]).await;
|
||||
let a = analyzer(&db, &url);
|
||||
let job = a.claim().await.unwrap().expect("a job");
|
||||
// an admin restarts it while the worker is still writing it
|
||||
sqlx::query("UPDATE analyses SET status = 'pending', started_at = NULL, attempts = 0 WHERE id = $1")
|
||||
.bind(id)
|
||||
.execute(&db)
|
||||
.await
|
||||
.unwrap();
|
||||
a.handle(job).await;
|
||||
let (status, content): (String, Option<Value>) =
|
||||
sqlx::query_as("SELECT status, content FROM analyses WHERE id = $1").bind(id).fetch_one(&db).await.unwrap();
|
||||
assert_eq!((status.as_str(), content), ("pending", None));
|
||||
// the new attempt writes it
|
||||
assert!(run_once(&a).await);
|
||||
let status: String =
|
||||
sqlx::query_scalar("SELECT status FROM analyses WHERE id = $1").bind(id).fetch_one(&db).await.unwrap();
|
||||
assert_eq!(status, "done");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user