Admin: list and restart running generations with the models set now
- Modellek page: analyses and answers being written or waiting, with their model and the one a restart would use; restart one or all - Each attempt's started_at is its token: a restarted attempt still running in the worker cannot write its result and is dropped within 10 seconds Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012vLHzh5qUhFtmJhzkcTbXL
This commit is contained in:
@@ -47,6 +47,9 @@ pub struct Job {
|
||||
/// The default model, when the chosen one is another (see the module docs).
|
||||
pub fallback_model: Option<String>,
|
||||
pub on_fallback: bool,
|
||||
/// When this attempt started; also its token: an admin restart clears it,
|
||||
/// so a superseded attempt neither writes its result nor keeps running.
|
||||
pub started_at: time::OffsetDateTime,
|
||||
}
|
||||
|
||||
impl Job {
|
||||
@@ -125,7 +128,7 @@ impl Analyzer {
|
||||
WHERE id = (SELECT id FROM analyses WHERE status = 'pending' AND next_attempt_at <= now() \
|
||||
ORDER BY next_attempt_at LIMIT 1 FOR UPDATE SKIP LOCKED) \
|
||||
RETURNING id, kind, chart_id, synastry_id, forecast_id, language, model, prompt_version, attempts, \
|
||||
fallback_model, on_fallback",
|
||||
fallback_model, on_fallback, started_at",
|
||||
)
|
||||
.fetch_optional(&self.db)
|
||||
.await?;
|
||||
@@ -273,12 +276,21 @@ impl Analyzer {
|
||||
/// Writes one claimed analysis and records the outcome.
|
||||
pub async fn handle(&self, job: Job) {
|
||||
let started = Instant::now();
|
||||
let result = match self.prompt_for(&job).await {
|
||||
Ok(prompt) => self.llm.complete(job.active_model(), &prompt).await.map(|mut c| {
|
||||
prompt::polish(&job.language, &mut c.content);
|
||||
c
|
||||
}),
|
||||
Err(e) => Err(e),
|
||||
let work = async {
|
||||
match self.prompt_for(&job).await {
|
||||
Ok(prompt) => self.llm.complete(job.active_model(), &prompt).await.map(|mut c| {
|
||||
prompt::polish(&job.language, &mut c.content);
|
||||
c
|
||||
}),
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
};
|
||||
let result = tokio::select! {
|
||||
result = work => result,
|
||||
() = self.superseded("analyses", job.id, job.started_at) => {
|
||||
tracing::info!(id = %job.id, "analysis restarted by an admin; this attempt is dropped");
|
||||
return;
|
||||
}
|
||||
};
|
||||
if let Err(e) = self.finish(&job, result, started.elapsed()).await {
|
||||
tracing::error!(id = %job.id, error = %e, "saving the analysis result failed");
|
||||
@@ -292,7 +304,7 @@ impl Analyzer {
|
||||
sqlx::query(
|
||||
"UPDATE analyses SET status = 'done', content = $2, model_used = $3, input_tokens = $4, \
|
||||
output_tokens = $5, cost_usd = $6, error = NULL, completed_at = now() \
|
||||
WHERE id = $1 AND status = 'running'",
|
||||
WHERE id = $1 AND status = 'running' AND started_at = $7",
|
||||
)
|
||||
.bind(job.id)
|
||||
.bind(&c.content)
|
||||
@@ -300,6 +312,7 @@ impl Analyzer {
|
||||
.bind(c.input_tokens)
|
||||
.bind(c.output_tokens)
|
||||
.bind(c.cost_usd)
|
||||
.bind(job.started_at)
|
||||
.execute(&self.db)
|
||||
.await?;
|
||||
tracing::info!(
|
||||
@@ -313,11 +326,13 @@ impl Analyzer {
|
||||
Next::Retry(wait) => {
|
||||
sqlx::query(
|
||||
"UPDATE analyses SET status = 'pending', error = $2, \
|
||||
next_attempt_at = now() + make_interval(secs => $3) WHERE id = $1 AND status = 'running'",
|
||||
next_attempt_at = now() + make_interval(secs => $3) \
|
||||
WHERE id = $1 AND status = 'running' AND started_at = $4",
|
||||
)
|
||||
.bind(job.id)
|
||||
.bind(e.message())
|
||||
.bind(wait.as_secs_f64())
|
||||
.bind(job.started_at)
|
||||
.execute(&self.db)
|
||||
.await?;
|
||||
tracing::warn!(id = %job.id, attempts = job.attempts, retry_in_secs = wait.as_secs(), error = %e.message(), "analysis will be retried");
|
||||
@@ -325,10 +340,11 @@ impl Analyzer {
|
||||
Next::Fallback => {
|
||||
sqlx::query(
|
||||
"UPDATE analyses SET status = 'pending', error = $2, on_fallback = true, attempts = 0, \
|
||||
next_attempt_at = now() WHERE id = $1 AND status = 'running'",
|
||||
next_attempt_at = now() WHERE id = $1 AND status = 'running' AND started_at = $3",
|
||||
)
|
||||
.bind(job.id)
|
||||
.bind(e.message())
|
||||
.bind(job.started_at)
|
||||
.execute(&self.db)
|
||||
.await?;
|
||||
tracing::warn!(
|
||||
@@ -337,7 +353,7 @@ impl Analyzer {
|
||||
);
|
||||
}
|
||||
Next::Fail => {
|
||||
self.fail(job.id, e.message()).await?;
|
||||
self.fail(job.id, job.started_at, e.message()).await?;
|
||||
tracing::error!(id = %job.id, attempts = job.attempts, error = %e.message(), "analysis failed");
|
||||
}
|
||||
}
|
||||
@@ -347,14 +363,15 @@ impl Analyzer {
|
||||
}
|
||||
|
||||
/// Marks the analysis failed and gives its unit back (once).
|
||||
async fn fail(&self, id: Uuid, message: &str) -> Result<(), sqlx::Error> {
|
||||
async fn fail(&self, id: Uuid, started_at: time::OffsetDateTime, message: &str) -> Result<(), sqlx::Error> {
|
||||
let mut tx = self.db.begin().await?;
|
||||
let row: Option<(Option<Uuid>, i32, String)> = sqlx::query_as(
|
||||
"UPDATE analyses SET status = 'failed', error = $2, completed_at = now() \
|
||||
WHERE id = $1 AND status = 'running' RETURNING requested_by, credits_charged, kind",
|
||||
WHERE id = $1 AND status = 'running' AND started_at = $3 RETURNING requested_by, credits_charged, kind",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(message)
|
||||
.bind(started_at)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await?;
|
||||
if let Some((Some(user), units, kind)) = row
|
||||
@@ -407,6 +424,21 @@ impl Analyzer {
|
||||
Ok(result.rows_affected())
|
||||
}
|
||||
|
||||
/// Resolves when this attempt is no longer the current one (an admin
|
||||
/// restarted it); checked every 10 seconds, never on a database error.
|
||||
async fn superseded(&self, table: &str, id: Uuid, started_at: time::OffsetDateTime) {
|
||||
let query = format!("SELECT status = 'running' AND started_at = $2 FROM {table} WHERE id = $1");
|
||||
loop {
|
||||
tokio::time::sleep(Duration::from_secs(10)).await;
|
||||
match sqlx::query_scalar::<_, Option<bool>>(&query).bind(id).bind(started_at).fetch_optional(&self.db).await
|
||||
{
|
||||
Ok(Some(Some(true))) => {}
|
||||
Ok(_) => return,
|
||||
Err(_) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// On shutdown: analyses still being written go back to `pending` at once.
|
||||
async fn release_running(&self) {
|
||||
let ids: Vec<Uuid> = self.running.lock().unwrap().drain().collect();
|
||||
|
||||
Reference in New Issue
Block a user