//! Admin: the analyses and answers being written or waiting in the queue, //! and restarting them with the models set now (e.g. after switching away //! from a slow model). A restarted attempt that is still running in the //! worker is dropped there and cannot overwrite the new one. use std::collections::HashMap; use analysis_prompt::Kind; use axum::Json; use axum::extract::State; use axum_extra::extract::WithRejection; use serde::Deserialize; use serde_json::{Value, json}; use time::OffsetDateTime; use uuid::Uuid; use super::analyses::{find_analysis, target_hash, versions}; use super::{Body, St}; use crate::AppState; use crate::auth::AdminUser; use crate::error::ApiResult; use crate::models::{Function, models_for}; use crate::views::{rfc3339, rfc3339_opt}; type JobRow = ( Uuid, String, String, String, Option, bool, i32, Option, OffsetDateTime, OffsetDateTime, Option, ); const COLUMNS: &str = "id, kind, status, model, fallback_model, on_fallback, attempts, started_at, created_at, next_attempt_at, error"; async fn open_jobs(state: &AppState) -> Result, sqlx::Error> { let analyses: Vec = sqlx::query_as(&format!( "SELECT {COLUMNS} FROM analyses WHERE status IN ('pending', 'running') ORDER BY created_at" )) .fetch_all(&state.db) .await?; let questions: Vec = sqlx::query_as( &"SELECT id, 'question', status, model, fallback_model, on_fallback, attempts, started_at, created_at, \ next_attempt_at, error \ FROM questions WHERE status IN ('pending', 'running') ORDER BY created_at" .to_string(), ) .fetch_all(&state.db) .await?; Ok(analyses.into_iter().map(|r| ("analysis", r)).chain(questions.into_iter().map(|r| ("question", r))).collect()) } pub async fn list(State(state): St, _admin: AdminUser) -> ApiResult> { Ok(Json(json!({ "jobs": views(&state).await? }))) } async fn views(state: &AppState) -> Result, sqlx::Error> { // The models a restart would use, per function. let mut next: HashMap<&'static str, (String, Option)> = HashMap::new(); for (function, kind) in [(Function::Summary, Kind::Summary), (Function::Analysis, Kind::Natal), (Function::Question, Kind::Question)] { next.insert(function.key(), models_for(state, kind).await?); } Ok(open_jobs(state) .await? .into_iter() .map(|(table, (id, kind, status, model, fallback, on_fallback, attempts, started, created, due, error))| { let function = Function::of(Kind::parse(&kind).unwrap_or(Kind::Natal)).key(); let (next_model, next_fallback) = &next[function]; let active = if on_fallback { fallback.clone().unwrap_or_else(|| model.clone()) } else { model.clone() }; json!({ "id": id, "type": table, "kind": kind, "status": status, "model": active, "on_fallback": on_fallback, "next_model": next_model, "next_fallback": next_fallback, "attempts": attempts, "started_at": rfc3339_opt(started), "created_at": rfc3339(created), "next_attempt_at": rfc3339(due), "error": error, }) }) .collect()) } #[derive(Deserialize)] #[serde(deny_unknown_fields)] pub struct Restart { /// The jobs to restart; all open ones when missing. ids: Option>, } pub async fn restart( State(state): St, AdminUser(admin): AdminUser, WithRejection(Json(req), _): Body, ) -> ApiResult> { let mut restarted = 0; for (table, (id, kind, ..)) in open_jobs(&state).await? { if req.ids.as_ref().is_some_and(|ids| !ids.contains(&id)) { continue; } let done = if table == "question" { let language: String = sqlx::query_scalar("SELECT language FROM questions WHERE id = $1") .bind(id) .fetch_one(&state.db) .await?; let (model, fallback, prompt_version) = super::questions::model_and_version(&state, &language).await?; sqlx::query( "UPDATE questions SET status = 'pending', model = $2, fallback_model = $3, on_fallback = false, \ prompt_version = $4, attempts = 0, next_attempt_at = now(), started_at = NULL, error = NULL \ WHERE id = $1 AND status IN ('pending', 'running')", ) .bind(id) .bind(&model) .bind(&fallback) .bind(&prompt_version) .execute(&state.db) .await? .rows_affected() } else { let Some(row) = find_analysis(&state.db, id).await? else { continue }; let Some(hash) = target_hash(&state.db, &row).await? else { continue }; let (model, fallback) = models_for(&state, Kind::parse(&kind).unwrap_or(Kind::Natal)).await?; let (cache_key, prompt_version) = versions(&state, &kind, &hash, &model, &row.language).await?; sqlx::query( "UPDATE analyses SET status = 'pending', model = $2, fallback_model = $3, on_fallback = false, \ cache_key = $4, prompt_version = $5, attempts = 0, next_attempt_at = now(), started_at = NULL, \ error = NULL \ WHERE id = $1 AND status IN ('pending', 'running')", ) .bind(id) .bind(&model) .bind(&fallback) .bind(&cache_key) .bind(&prompt_version) .execute(&state.db) .await? .rows_affected() }; restarted += done; } if restarted > 0 { sqlx::query("SELECT pg_notify('analyses', '')").execute(&state.db).await?; } tracing::info!(admin = %admin.id(), restarted, "generations restarted"); Ok(Json(json!({ "restarted": restarted, "jobs": views(&state).await? }))) }