Admin: list and restart running generations with the models set now
- Modellek page: analyses and answers being written or waiting, with their model and the one a restart would use; restart one or all - Each attempt's started_at is its token: a restarted attempt still running in the worker cannot write its result and is dropped within 10 seconds Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012vLHzh5qUhFtmJhzkcTbXL
This commit is contained in:
@@ -0,0 +1,150 @@
|
||||
//! Admin: the analyses and answers being written or waiting in the queue,
|
||||
//! and restarting them with the models set now (e.g. after switching away
|
||||
//! from a slow model). A restarted attempt that is still running in the
|
||||
//! worker is dropped there and cannot overwrite the new one.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use analysis_prompt::Kind;
|
||||
use axum::Json;
|
||||
use axum::extract::State;
|
||||
use axum_extra::extract::WithRejection;
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Value, json};
|
||||
use time::OffsetDateTime;
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::analyses::{find_analysis, target_hash, versions};
|
||||
use super::{Body, St};
|
||||
use crate::AppState;
|
||||
use crate::auth::AdminUser;
|
||||
use crate::error::ApiResult;
|
||||
use crate::models::{Function, models_for};
|
||||
use crate::views::{rfc3339, rfc3339_opt};
|
||||
|
||||
type JobRow = (
|
||||
Uuid,
|
||||
String,
|
||||
String,
|
||||
String,
|
||||
Option<String>,
|
||||
bool,
|
||||
i32,
|
||||
Option<OffsetDateTime>,
|
||||
OffsetDateTime,
|
||||
OffsetDateTime,
|
||||
Option<String>,
|
||||
);
|
||||
|
||||
const COLUMNS: &str =
|
||||
"id, kind, status, model, fallback_model, on_fallback, attempts, started_at, created_at, next_attempt_at, error";
|
||||
|
||||
async fn open_jobs(state: &AppState) -> Result<Vec<(&'static str, JobRow)>, sqlx::Error> {
|
||||
let analyses: Vec<JobRow> = sqlx::query_as(&format!(
|
||||
"SELECT {COLUMNS} FROM analyses WHERE status IN ('pending', 'running') ORDER BY created_at"
|
||||
))
|
||||
.fetch_all(&state.db)
|
||||
.await?;
|
||||
let questions: Vec<JobRow> = sqlx::query_as(
|
||||
&"SELECT id, 'question', status, model, fallback_model, on_fallback, attempts, started_at, created_at, \
|
||||
next_attempt_at, error \
|
||||
FROM questions WHERE status IN ('pending', 'running') ORDER BY created_at"
|
||||
.to_string(),
|
||||
)
|
||||
.fetch_all(&state.db)
|
||||
.await?;
|
||||
Ok(analyses.into_iter().map(|r| ("analysis", r)).chain(questions.into_iter().map(|r| ("question", r))).collect())
|
||||
}
|
||||
|
||||
pub async fn list(State(state): St, _admin: AdminUser) -> ApiResult<Json<Value>> {
|
||||
Ok(Json(json!({ "jobs": views(&state).await? })))
|
||||
}
|
||||
|
||||
async fn views(state: &AppState) -> Result<Vec<Value>, sqlx::Error> {
|
||||
// The models a restart would use, per function.
|
||||
let mut next: HashMap<&'static str, (String, Option<String>)> = HashMap::new();
|
||||
for (function, kind) in
|
||||
[(Function::Summary, Kind::Summary), (Function::Analysis, Kind::Natal), (Function::Question, Kind::Question)]
|
||||
{
|
||||
next.insert(function.key(), models_for(state, kind).await?);
|
||||
}
|
||||
Ok(open_jobs(state)
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|(table, (id, kind, status, model, fallback, on_fallback, attempts, started, created, due, error))| {
|
||||
let function = Function::of(Kind::parse(&kind).unwrap_or(Kind::Natal)).key();
|
||||
let (next_model, next_fallback) = &next[function];
|
||||
let active = if on_fallback { fallback.clone().unwrap_or_else(|| model.clone()) } else { model.clone() };
|
||||
json!({
|
||||
"id": id, "type": table, "kind": kind, "status": status, "model": active, "on_fallback": on_fallback,
|
||||
"next_model": next_model, "next_fallback": next_fallback, "attempts": attempts,
|
||||
"started_at": rfc3339_opt(started), "created_at": rfc3339(created), "next_attempt_at": rfc3339(due),
|
||||
"error": error,
|
||||
})
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Restart {
|
||||
/// The jobs to restart; all open ones when missing.
|
||||
ids: Option<Vec<Uuid>>,
|
||||
}
|
||||
|
||||
pub async fn restart(
|
||||
State(state): St,
|
||||
AdminUser(admin): AdminUser,
|
||||
WithRejection(Json(req), _): Body<Restart>,
|
||||
) -> ApiResult<Json<Value>> {
|
||||
let mut restarted = 0;
|
||||
for (table, (id, kind, ..)) in open_jobs(&state).await? {
|
||||
if req.ids.as_ref().is_some_and(|ids| !ids.contains(&id)) {
|
||||
continue;
|
||||
}
|
||||
let done = if table == "question" {
|
||||
let language: String = sqlx::query_scalar("SELECT language FROM questions WHERE id = $1")
|
||||
.bind(id)
|
||||
.fetch_one(&state.db)
|
||||
.await?;
|
||||
let (model, fallback, prompt_version) = super::questions::model_and_version(&state, &language).await?;
|
||||
sqlx::query(
|
||||
"UPDATE questions SET status = 'pending', model = $2, fallback_model = $3, on_fallback = false, \
|
||||
prompt_version = $4, attempts = 0, next_attempt_at = now(), started_at = NULL, error = NULL \
|
||||
WHERE id = $1 AND status IN ('pending', 'running')",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(&model)
|
||||
.bind(&fallback)
|
||||
.bind(&prompt_version)
|
||||
.execute(&state.db)
|
||||
.await?
|
||||
.rows_affected()
|
||||
} else {
|
||||
let Some(row) = find_analysis(&state.db, id).await? else { continue };
|
||||
let Some(hash) = target_hash(&state.db, &row).await? else { continue };
|
||||
let (model, fallback) = models_for(&state, Kind::parse(&kind).unwrap_or(Kind::Natal)).await?;
|
||||
let (cache_key, prompt_version) = versions(&state, &kind, &hash, &model, &row.language).await?;
|
||||
sqlx::query(
|
||||
"UPDATE analyses SET status = 'pending', model = $2, fallback_model = $3, on_fallback = false, \
|
||||
cache_key = $4, prompt_version = $5, attempts = 0, next_attempt_at = now(), started_at = NULL, \
|
||||
error = NULL \
|
||||
WHERE id = $1 AND status IN ('pending', 'running')",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(&model)
|
||||
.bind(&fallback)
|
||||
.bind(&cache_key)
|
||||
.bind(&prompt_version)
|
||||
.execute(&state.db)
|
||||
.await?
|
||||
.rows_affected()
|
||||
};
|
||||
restarted += done;
|
||||
}
|
||||
if restarted > 0 {
|
||||
sqlx::query("SELECT pg_notify('analyses', '')").execute(&state.db).await?;
|
||||
}
|
||||
tracing::info!(admin = %admin.id(), restarted, "generations restarted");
|
||||
Ok(Json(json!({ "restarted": restarted, "jobs": views(&state).await? })))
|
||||
}
|
||||
Reference in New Issue
Block a user