From b5743ec7d1348eea5df9de3999e765baac4f4f51 Mon Sep 17 00:00:00 2001 From: feco Date: Wed, 18 Nov 2026 21:31:09 +0000 Subject: [PATCH] Admin: list and restart running generations with the models set now - Modellek page: analyses and answers being written or waiting, with their model and the one a restart would use; restart one or all - Each attempt's started_at is its token: a restarted attempt still running in the worker cannot write its result and is dropped within 10 seconds Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012vLHzh5qUhFtmJhzkcTbXL --- services/backend/README.md | 1 + services/backend/src/routes/analyses.rs | 35 ++-- services/backend/src/routes/jobs.rs | 150 ++++++++++++++++++ services/backend/src/routes/mod.rs | 3 + services/backend/src/routes/questions.rs | 2 +- services/backend/src/tests.rs | 56 +++++++ .../src/app/admin/modellek/AdminModels.tsx | 3 + .../src/app/admin/modellek/RunningJobs.tsx | 138 ++++++++++++++++ services/frontend/src/app/globals.css | 3 + services/frontend/src/lib/types.ts | 19 +++ services/worker/src/analysis/mod.rs | 58 +++++-- services/worker/src/analysis/questions.rs | 51 ++++-- services/worker/src/tests.rs | 24 +++ 13 files changed, 492 insertions(+), 51 deletions(-) create mode 100644 services/backend/src/routes/jobs.rs create mode 100644 services/frontend/src/app/admin/modellek/RunningJobs.tsx diff --git a/services/backend/README.md b/services/backend/README.md index 0205dd6..3954738 100644 --- a/services/backend/README.md +++ b/services/backend/README.md @@ -127,6 +127,7 @@ Hibák formátuma: `{"error": {"code", "message"}}`. A születési adatok hibái - **Fizetős rész:** a részletes elemzés darabra megy. A képletelemzés egy `natal`, a szinasztria-elemzés egy `synastry` egységet von le (`credit_ledger.unit`). Az egységek csomagvásárlásból (Stripe, később) vagy admin jóváírásból jönnek. Havi ingyenes keret nincs. - **Modell:** három beállítás az admin felületen, az OpenRouter listájából (`model_settings` tábla): az elemzések és az éves előrejelzés (képlet-, kapcsolati elemzés, előrejelzés) közös modellje, a kérdés-válasz modellje és a rövid összefoglalóé. Ha nincs beállítva, az `ANALYSIS_MODEL` / `SUMMARY_MODEL` (alapból mindkettő Opus 5.5, a 2026-11-es összevetés legjobbja, lásd `services/worker/README.md`). Csak kötött JSON-választ adó modell választható. A kliens nem választ modellt, és a modell neve felhasználónak sehol nem jelenik meg. - **Tartalék:** ha a választott modell kétszer hibára fut (végleges hibánál egyszer), a worker az alapmodellel próbálja tovább (`fallback_model`, `on_fallback`); csak ha az is elbukik, jár vissza az egység. A Modellek oldal mutatja, hány szövegnél kellett az elmúlt 30 napban. +- **Futó generálások:** a Modellek oldalon látszik minden készülő és sorban álló elemzés és válasz, a modelljével és azzal, amivel most indulna; újraindításkor a jelenlegi beállítással indul újra (`GET /v1/admin/jobs`, `POST /v1/admin/jobs/restart`). Minden kísérlet `started_at` értéke a „jegye”: az újraindítás törli, így a workerben még futó régi kísérlet nem írhatja felül az újat, és a worker 10 másodpercen belül eldobja. - **Újragenerálás:** admin minden elemzést egyenként újraírathat (`POST /v1/admin/analyses/{id}/regenerate`): ugyanaz a sor (az azonosító és a megosztások maradnak), a jelenlegi prompttal és modellel, egységlevonás nélkül. - **Újraszámolás:** számítási változás után a `backend recalculate` parancs minden mentett képletet, összevetést és előrejelzést újraszámol, majd minden elemzést újra sorba állít (egységlevonás nélkül). - **Kérés:** a `POST .../analyses` (`{language?}`) létrehoz egy `pending` elemzést, és levon egy egységet; a szöveget a worker írja meg, általában 1–2 perc alatt. A kliens a `GET /v1/analyses/{id}` végponton kérdezi le az állapotát. Ha nincs egység, 402 `insufficient_credits` a válasz (`unit`, `balance` részletekkel). Hiba esetén a worker visszaadja az egységet. diff --git a/services/backend/src/routes/analyses.rs b/services/backend/src/routes/analyses.rs index 7eb205f..70df250 100644 --- a/services/backend/src/routes/analyses.rs +++ b/services/backend/src/routes/analyses.rs @@ -146,7 +146,7 @@ async fn request( /// The cache key and the prompt version of a new analysis. The cache key leaves /// out the admin-edited prompt revision, so a chart keeps its analysis when the /// prompt changes; copying from another chart needs the same revision. -async fn versions( +pub(super) async fn versions( state: &AppState, kind: &str, hash: &str, @@ -359,6 +359,17 @@ pub async fn request_for_forecast( request(&state, &auth, Target::Forecast { forecast_id: forecast.id, hash: forecast.forecast_hash }, req).await } +/// The hash of what the analysis is about (chart, comparison or forecast), +/// part of its cache key; `None` when that was deleted. +pub(super) async fn target_hash(db: &PgPool, row: &AnalysisRow) -> Result, sqlx::Error> { + let (query, id) = match row.kind.as_str() { + "synastry" => ("SELECT synastry_hash FROM synastries WHERE id = $1", row.synastry_id), + "forecast" => ("SELECT forecast_hash FROM forecasts WHERE id = $1", row.forecast_id), + _ => ("SELECT chart_hash FROM charts WHERE id = $1", row.chart_id), + }; + sqlx::query_scalar(query).bind(id).fetch_optional(db).await +} + /// Admin: writes an analysis again in place (same id, so links stay), with the /// current prompt and model. Nobody is charged. pub async fn regenerate( @@ -371,27 +382,7 @@ pub async fn regenerate( return Err(ApiError::conflict("analysis_in_progress", "Ez az elemzés éppen készül.")); } let kind = Kind::parse(&row.kind).ok_or_else(|| ApiError::Internal(format!("unknown kind {}", row.kind)))?; - let hash: Option = match kind { - Kind::Question => None, - Kind::Natal | Kind::Summary => { - sqlx::query_scalar("SELECT chart_hash FROM charts WHERE id = $1") - .bind(row.chart_id) - .fetch_optional(&state.db) - .await? - } - Kind::Synastry => { - sqlx::query_scalar("SELECT synastry_hash FROM synastries WHERE id = $1") - .bind(row.synastry_id) - .fetch_optional(&state.db) - .await? - } - Kind::Forecast => { - sqlx::query_scalar("SELECT forecast_hash FROM forecasts WHERE id = $1") - .bind(row.forecast_id) - .fetch_optional(&state.db) - .await? - } - }; + let hash = target_hash(&state.db, &row).await?; let hash = hash.ok_or(ApiError::NotFound("analysis"))?; let (model, fallback) = crate::models::models_for(&state, kind).await?; let (cache_key, prompt_version) = versions(&state, kind.as_str(), &hash, &model, &row.language).await?; diff --git a/services/backend/src/routes/jobs.rs b/services/backend/src/routes/jobs.rs new file mode 100644 index 0000000..9735a0b --- /dev/null +++ b/services/backend/src/routes/jobs.rs @@ -0,0 +1,150 @@ +//! Admin: the analyses and answers being written or waiting in the queue, +//! and restarting them with the models set now (e.g. after switching away +//! from a slow model). A restarted attempt that is still running in the +//! worker is dropped there and cannot overwrite the new one. + +use std::collections::HashMap; + +use analysis_prompt::Kind; +use axum::Json; +use axum::extract::State; +use axum_extra::extract::WithRejection; +use serde::Deserialize; +use serde_json::{Value, json}; +use time::OffsetDateTime; +use uuid::Uuid; + +use super::analyses::{find_analysis, target_hash, versions}; +use super::{Body, St}; +use crate::AppState; +use crate::auth::AdminUser; +use crate::error::ApiResult; +use crate::models::{Function, models_for}; +use crate::views::{rfc3339, rfc3339_opt}; + +type JobRow = ( + Uuid, + String, + String, + String, + Option, + bool, + i32, + Option, + OffsetDateTime, + OffsetDateTime, + Option, +); + +const COLUMNS: &str = + "id, kind, status, model, fallback_model, on_fallback, attempts, started_at, created_at, next_attempt_at, error"; + +async fn open_jobs(state: &AppState) -> Result, sqlx::Error> { + let analyses: Vec = sqlx::query_as(&format!( + "SELECT {COLUMNS} FROM analyses WHERE status IN ('pending', 'running') ORDER BY created_at" + )) + .fetch_all(&state.db) + .await?; + let questions: Vec = sqlx::query_as( + &"SELECT id, 'question', status, model, fallback_model, on_fallback, attempts, started_at, created_at, \ + next_attempt_at, error \ + FROM questions WHERE status IN ('pending', 'running') ORDER BY created_at" + .to_string(), + ) + .fetch_all(&state.db) + .await?; + Ok(analyses.into_iter().map(|r| ("analysis", r)).chain(questions.into_iter().map(|r| ("question", r))).collect()) +} + +pub async fn list(State(state): St, _admin: AdminUser) -> ApiResult> { + Ok(Json(json!({ "jobs": views(&state).await? }))) +} + +async fn views(state: &AppState) -> Result, sqlx::Error> { + // The models a restart would use, per function. + let mut next: HashMap<&'static str, (String, Option)> = HashMap::new(); + for (function, kind) in + [(Function::Summary, Kind::Summary), (Function::Analysis, Kind::Natal), (Function::Question, Kind::Question)] + { + next.insert(function.key(), models_for(state, kind).await?); + } + Ok(open_jobs(state) + .await? + .into_iter() + .map(|(table, (id, kind, status, model, fallback, on_fallback, attempts, started, created, due, error))| { + let function = Function::of(Kind::parse(&kind).unwrap_or(Kind::Natal)).key(); + let (next_model, next_fallback) = &next[function]; + let active = if on_fallback { fallback.clone().unwrap_or_else(|| model.clone()) } else { model.clone() }; + json!({ + "id": id, "type": table, "kind": kind, "status": status, "model": active, "on_fallback": on_fallback, + "next_model": next_model, "next_fallback": next_fallback, "attempts": attempts, + "started_at": rfc3339_opt(started), "created_at": rfc3339(created), "next_attempt_at": rfc3339(due), + "error": error, + }) + }) + .collect()) +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Restart { + /// The jobs to restart; all open ones when missing. + ids: Option>, +} + +pub async fn restart( + State(state): St, + AdminUser(admin): AdminUser, + WithRejection(Json(req), _): Body, +) -> ApiResult> { + let mut restarted = 0; + for (table, (id, kind, ..)) in open_jobs(&state).await? { + if req.ids.as_ref().is_some_and(|ids| !ids.contains(&id)) { + continue; + } + let done = if table == "question" { + let language: String = sqlx::query_scalar("SELECT language FROM questions WHERE id = $1") + .bind(id) + .fetch_one(&state.db) + .await?; + let (model, fallback, prompt_version) = super::questions::model_and_version(&state, &language).await?; + sqlx::query( + "UPDATE questions SET status = 'pending', model = $2, fallback_model = $3, on_fallback = false, \ + prompt_version = $4, attempts = 0, next_attempt_at = now(), started_at = NULL, error = NULL \ + WHERE id = $1 AND status IN ('pending', 'running')", + ) + .bind(id) + .bind(&model) + .bind(&fallback) + .bind(&prompt_version) + .execute(&state.db) + .await? + .rows_affected() + } else { + let Some(row) = find_analysis(&state.db, id).await? else { continue }; + let Some(hash) = target_hash(&state.db, &row).await? else { continue }; + let (model, fallback) = models_for(&state, Kind::parse(&kind).unwrap_or(Kind::Natal)).await?; + let (cache_key, prompt_version) = versions(&state, &kind, &hash, &model, &row.language).await?; + sqlx::query( + "UPDATE analyses SET status = 'pending', model = $2, fallback_model = $3, on_fallback = false, \ + cache_key = $4, prompt_version = $5, attempts = 0, next_attempt_at = now(), started_at = NULL, \ + error = NULL \ + WHERE id = $1 AND status IN ('pending', 'running')", + ) + .bind(id) + .bind(&model) + .bind(&fallback) + .bind(&cache_key) + .bind(&prompt_version) + .execute(&state.db) + .await? + .rows_affected() + }; + restarted += done; + } + if restarted > 0 { + sqlx::query("SELECT pg_notify('analyses', '')").execute(&state.db).await?; + } + tracing::info!(admin = %admin.id(), restarted, "generations restarted"); + Ok(Json(json!({ "restarted": restarted, "jobs": views(&state).await? }))) +} diff --git a/services/backend/src/routes/mod.rs b/services/backend/src/routes/mod.rs index 039e412..96bcfb9 100644 --- a/services/backend/src/routes/mod.rs +++ b/services/backend/src/routes/mod.rs @@ -12,6 +12,7 @@ mod auth; mod charts; mod forecasts; mod invitations; +mod jobs; mod models; mod payments; mod prompts; @@ -116,6 +117,8 @@ pub fn router(state: Arc) -> Router { .route("/v1/admin/prompts/{kind}/{language}", axum::routing::put(prompts::save)) .route("/v1/admin/prompts/{kind}/{language}/preview", post(prompts::start_preview)) .route("/v1/admin/previews/{id}", get(prompts::get_preview)) + .route("/v1/admin/jobs", get(jobs::list)) + .route("/v1/admin/jobs/restart", post(jobs::restart)) .route("/v1/admin/models", get(models::list)) .route("/v1/admin/models/catalog", get(models::catalog)) .route("/v1/admin/models/{kind}", axum::routing::put(models::save)) diff --git a/services/backend/src/routes/questions.rs b/services/backend/src/routes/questions.rs index afa4147..b7b150d 100644 --- a/services/backend/src/routes/questions.rs +++ b/services/backend/src/routes/questions.rs @@ -66,7 +66,7 @@ async fn find_question(db: &PgPool, id: Uuid) -> Result, sql /// The question model (and its fallback) and the version of the question /// prompt in force. -async fn model_and_version( +pub(super) async fn model_and_version( state: &crate::AppState, language: &str, ) -> Result<(String, Option, String), sqlx::Error> { diff --git a/services/backend/src/tests.rs b/services/backend/src/tests.rs index 20df951..0bef6e9 100644 --- a/services/backend/src/tests.rs +++ b/services/backend/src/tests.rs @@ -1639,3 +1639,59 @@ async fn questions_about_an_analysis_use_question_units(pool: PgPool) { ); assert_eq!(left().await, json!(5), "regenerating charges nobody"); } + +#[sqlx::test(migrator = "db::MIGRATOR")] +async fn admins_restart_open_generations_with_the_current_models(pool: PgPool) { + let app = TestApp::new(pool.clone()).await; + let admin = app.register("admin@example.com", "Admin").await; + let anna = app.register("anna@example.com", "Anna").await; + assert_eq!(app.get("/v1/admin/jobs", Some(&anna)).await.status, StatusCode::FORBIDDEN); + let put = |model: &str| { + let (app, admin, body) = (&app, &admin, json!({ "model": model })); + async move { app.call("PUT", "/v1/admin/models/analysis", Some(admin), Some(body)).await } + }; + assert_eq!(put("openai/gpt-x").await.status, StatusCode::OK); + let chart = app.chart(&admin, "Saját", true, "1990-03-15").await; + let chart_id = chart["id"].as_str().unwrap(); + let r = app.post(&format!("/v1/charts/{chart_id}/analyses"), Some(&admin), json!({})).await; + let id = r.body["id"].as_str().unwrap().to_string(); + // the worker is writing it with the chosen model + sqlx::query("UPDATE analyses SET status = 'running', started_at = now(), attempts = 1 WHERE id = $1::uuid") + .bind(&id) + .execute(&pool) + .await + .unwrap(); + + // after switching back to the default, the list shows what a restart would use + assert_eq!( + app.call("PUT", "/v1/admin/models/analysis", Some(&admin), Some(json!({"model": null}))).await.status, + StatusCode::OK + ); + let jobs = app.get("/v1/admin/jobs", Some(&admin)).await.body["jobs"].clone(); + let job = jobs.as_array().unwrap().iter().find(|j| j["id"] == json!(id)).expect("the analysis is listed"); + assert_eq!( + (&job["status"], &job["model"], &job["next_model"]), + (&json!("running"), &json!("openai/gpt-x"), &json!("anthropic/claude-test")) + ); + + let r = app.post("/v1/admin/jobs/restart", Some(&admin), json!({ "ids": [id] })).await; + assert_eq!((r.status, &r.body["restarted"]), (StatusCode::OK, &json!(1)), "{}", r.body); + let (status, model, fallback, started, attempts): ( + String, + String, + Option, + Option, + i32, + ) = sqlx::query_as("SELECT status, model, fallback_model, started_at, attempts FROM analyses WHERE id = $1::uuid") + .bind(&id) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!( + (status.as_str(), model.as_str(), fallback, started, attempts), + ("pending", "anthropic/claude-test", None, None, 0) + ); + // finished ones are left alone + sqlx::query("UPDATE analyses SET status = 'done', content = '{}'").execute(&pool).await.unwrap(); + assert_eq!(app.post("/v1/admin/jobs/restart", Some(&admin), json!({})).await.body["restarted"], 0); +} diff --git a/services/frontend/src/app/admin/modellek/AdminModels.tsx b/services/frontend/src/app/admin/modellek/AdminModels.tsx index 84c5748..f3e8973 100644 --- a/services/frontend/src/app/admin/modellek/AdminModels.tsx +++ b/services/frontend/src/app/admin/modellek/AdminModels.tsx @@ -6,6 +6,8 @@ import { api, errorText } from "@/lib/api"; import { formatDate } from "@/lib/astro"; import type { CatalogModel, ModelFunction } from "@/lib/types"; +import { RunningJobs } from "./RunningJobs"; + const LABELS: Record = { summary: { title: "Tömör összefoglaló", about: "Ingyenes, minden új képlethez magától elkészül. Olcsó, gyors modell elég." }, analysis: { @@ -66,6 +68,7 @@ export function AdminModels() { {functions.map((f) => ( ))} + ); } diff --git a/services/frontend/src/app/admin/modellek/RunningJobs.tsx b/services/frontend/src/app/admin/modellek/RunningJobs.tsx new file mode 100644 index 0000000..11c4a35 --- /dev/null +++ b/services/frontend/src/app/admin/modellek/RunningJobs.tsx @@ -0,0 +1,138 @@ +"use client"; + +import { useCallback, useEffect, useState } from "react"; + +import { api, errorText } from "@/lib/api"; +import type { AdminJob } from "@/lib/types"; + +const KINDS: Record = { + summary: "Összefoglaló", + natal: "Képletelemzés", + synastry: "Kapcsolati elemzés", + forecast: "Éves előrejelzés", + question: "Kérdés-válasz", +}; + +const REFRESH_MS = 10_000; + +function since(iso: string): string { + const minutes = Math.max(0, Math.round((Date.now() - new Date(iso).getTime()) / 60_000)); + return minutes < 1 ? "most" : `${minutes} perce`; +} + +/** + * Analyses and answers being written or waiting. A restart starts them again + * with the models set now; an attempt still running is dropped by the worker. + */ +export function RunningJobs() { + const [jobs, setJobs] = useState(null); + const [busy, setBusy] = useState(false); + const [asking, setAsking] = useState(false); + const [message, setMessage] = useState(null); + const [error, setError] = useState(null); + + const load = useCallback(() => { + api<{ jobs: AdminJob[] }>("/admin/jobs") + .then((d) => setJobs(d.jobs)) + .catch((e) => setError(errorText(e))); + }, []); + + useEffect(() => { + load(); + const timer = setInterval(load, REFRESH_MS); + return () => clearInterval(timer); + }, [load]); + + async function restart(ids?: string[]) { + setBusy(true); + setError(null); + setMessage(null); + try { + const r = await api<{ restarted: number; jobs: AdminJob[] }>("/admin/jobs/restart", { body: ids ? { ids } : {} }); + setJobs(r.jobs); + setMessage(r.restarted > 0 ? `${r.restarted} generálás újraindítva.` : "Nem volt mit újraindítani."); + setAsking(false); + } catch (e) { + setError(errorText(e)); + } finally { + setBusy(false); + } + } + + const changed = (jobs ?? []).filter((j) => j.model !== j.next_model).length; + return ( +
+
+
+

Futó generálások

+

+ Az éppen készülő és a sorban álló elemzések és válaszok. Újraindításkor mindegyik azzal a modellel indul újra, + ami most be van állítva; a még futó régi kérést a worker eldobja. Senkitől nem von le újabb egységet. +

+
+ {jobs && jobs.length > 0 && ( +
+ {asking ? ( + <> + + {jobs.length} generálás indul újra{changed > 0 ? `, ebből ${changed} másik modellel` : ""}. + + + + + ) : ( + + )} +
+ )} +
+ {message && ( +

+ {message} +

+ )} + {error && ( +

+ {error} +

+ )} + {jobs === null ? ( +

Betöltés…

+ ) : jobs.length === 0 ? ( +

Most semmi nem készül és semmi nem vár a sorban.

+ ) : ( +
    + {jobs.map((j) => ( +
  • + + {KINDS[j.kind]} + + {j.status === "running" && j.started_at + ? `fut, ${since(j.started_at)} indult` + : `vár (kérve ${since(j.created_at)})`} + {j.attempts > 0 ? ` · ${j.attempts}. kísérlet` : ""} + {j.on_fallback ? " · tartalék modellel" : ""} + + {j.error && utolsó hiba: {j.error}} + + + {j.model} + {j.model !== j.next_model ? → {j.next_model} : újraindítva is ez} + + +
  • + ))} +
+ )} +
+ ); +} diff --git a/services/frontend/src/app/globals.css b/services/frontend/src/app/globals.css index 4d94d75..138d7f3 100644 --- a/services/frontend/src/app/globals.css +++ b/services/frontend/src/app/globals.css @@ -644,3 +644,6 @@ a.pair-row:hover strong { color: var(--gold); } .qa-form { display: flex; flex-direction: column; gap: 8px; } .qa-form textarea { resize: vertical; min-height: 84px; } .hint.warn { color: var(--gold); } +.job-row { display: grid; grid-template-columns: minmax(0, 1fr) auto auto; gap: 12px; align-items: center; padding: 8px 12px; border: 1px solid var(--line); border-radius: 8px; } +.job-row > span:first-child { display: flex; flex-direction: column; min-width: 0; } +@media (max-width: 640px) { .job-row { grid-template-columns: minmax(0, 1fr) auto; } .job-row .model-price { grid-column: 1 / -1; align-items: flex-start; } } diff --git a/services/frontend/src/lib/types.ts b/services/frontend/src/lib/types.ts index 3fddfa3..abb550c 100644 --- a/services/frontend/src/lib/types.ts +++ b/services/frontend/src/lib/types.ts @@ -266,6 +266,25 @@ export interface ModelFunction { finished_30d: number; } +/** Admin: an analysis or answer being written or waiting in the queue. */ +export interface AdminJob { + id: string; + type: "analysis" | "question"; + kind: "summary" | "natal" | "synastry" | "forecast" | "question"; + status: "pending" | "running"; + /** The model of the current attempt. */ + model: string; + on_fallback: boolean; + /** What a restart would use now. */ + next_model: string; + next_fallback: string | null; + attempts: number; + started_at: string | null; + created_at: string; + next_attempt_at: string; + error: string | null; +} + /** A model OpenRouter offers; prices in USD per million tokens. */ export interface CatalogModel { id: string; diff --git a/services/worker/src/analysis/mod.rs b/services/worker/src/analysis/mod.rs index a22bded..e366c05 100644 --- a/services/worker/src/analysis/mod.rs +++ b/services/worker/src/analysis/mod.rs @@ -47,6 +47,9 @@ pub struct Job { /// The default model, when the chosen one is another (see the module docs). pub fallback_model: Option, pub on_fallback: bool, + /// When this attempt started; also its token: an admin restart clears it, + /// so a superseded attempt neither writes its result nor keeps running. + pub started_at: time::OffsetDateTime, } impl Job { @@ -125,7 +128,7 @@ impl Analyzer { WHERE id = (SELECT id FROM analyses WHERE status = 'pending' AND next_attempt_at <= now() \ ORDER BY next_attempt_at LIMIT 1 FOR UPDATE SKIP LOCKED) \ RETURNING id, kind, chart_id, synastry_id, forecast_id, language, model, prompt_version, attempts, \ - fallback_model, on_fallback", + fallback_model, on_fallback, started_at", ) .fetch_optional(&self.db) .await?; @@ -273,12 +276,21 @@ impl Analyzer { /// Writes one claimed analysis and records the outcome. pub async fn handle(&self, job: Job) { let started = Instant::now(); - let result = match self.prompt_for(&job).await { - Ok(prompt) => self.llm.complete(job.active_model(), &prompt).await.map(|mut c| { - prompt::polish(&job.language, &mut c.content); - c - }), - Err(e) => Err(e), + let work = async { + match self.prompt_for(&job).await { + Ok(prompt) => self.llm.complete(job.active_model(), &prompt).await.map(|mut c| { + prompt::polish(&job.language, &mut c.content); + c + }), + Err(e) => Err(e), + } + }; + let result = tokio::select! { + result = work => result, + () = self.superseded("analyses", job.id, job.started_at) => { + tracing::info!(id = %job.id, "analysis restarted by an admin; this attempt is dropped"); + return; + } }; if let Err(e) = self.finish(&job, result, started.elapsed()).await { tracing::error!(id = %job.id, error = %e, "saving the analysis result failed"); @@ -292,7 +304,7 @@ impl Analyzer { sqlx::query( "UPDATE analyses SET status = 'done', content = $2, model_used = $3, input_tokens = $4, \ output_tokens = $5, cost_usd = $6, error = NULL, completed_at = now() \ - WHERE id = $1 AND status = 'running'", + WHERE id = $1 AND status = 'running' AND started_at = $7", ) .bind(job.id) .bind(&c.content) @@ -300,6 +312,7 @@ impl Analyzer { .bind(c.input_tokens) .bind(c.output_tokens) .bind(c.cost_usd) + .bind(job.started_at) .execute(&self.db) .await?; tracing::info!( @@ -313,11 +326,13 @@ impl Analyzer { Next::Retry(wait) => { sqlx::query( "UPDATE analyses SET status = 'pending', error = $2, \ - next_attempt_at = now() + make_interval(secs => $3) WHERE id = $1 AND status = 'running'", + next_attempt_at = now() + make_interval(secs => $3) \ + WHERE id = $1 AND status = 'running' AND started_at = $4", ) .bind(job.id) .bind(e.message()) .bind(wait.as_secs_f64()) + .bind(job.started_at) .execute(&self.db) .await?; tracing::warn!(id = %job.id, attempts = job.attempts, retry_in_secs = wait.as_secs(), error = %e.message(), "analysis will be retried"); @@ -325,10 +340,11 @@ impl Analyzer { Next::Fallback => { sqlx::query( "UPDATE analyses SET status = 'pending', error = $2, on_fallback = true, attempts = 0, \ - next_attempt_at = now() WHERE id = $1 AND status = 'running'", + next_attempt_at = now() WHERE id = $1 AND status = 'running' AND started_at = $3", ) .bind(job.id) .bind(e.message()) + .bind(job.started_at) .execute(&self.db) .await?; tracing::warn!( @@ -337,7 +353,7 @@ impl Analyzer { ); } Next::Fail => { - self.fail(job.id, e.message()).await?; + self.fail(job.id, job.started_at, e.message()).await?; tracing::error!(id = %job.id, attempts = job.attempts, error = %e.message(), "analysis failed"); } } @@ -347,14 +363,15 @@ impl Analyzer { } /// Marks the analysis failed and gives its unit back (once). - async fn fail(&self, id: Uuid, message: &str) -> Result<(), sqlx::Error> { + async fn fail(&self, id: Uuid, started_at: time::OffsetDateTime, message: &str) -> Result<(), sqlx::Error> { let mut tx = self.db.begin().await?; let row: Option<(Option, i32, String)> = sqlx::query_as( "UPDATE analyses SET status = 'failed', error = $2, completed_at = now() \ - WHERE id = $1 AND status = 'running' RETURNING requested_by, credits_charged, kind", + WHERE id = $1 AND status = 'running' AND started_at = $3 RETURNING requested_by, credits_charged, kind", ) .bind(id) .bind(message) + .bind(started_at) .fetch_optional(&mut *tx) .await?; if let Some((Some(user), units, kind)) = row @@ -407,6 +424,21 @@ impl Analyzer { Ok(result.rows_affected()) } + /// Resolves when this attempt is no longer the current one (an admin + /// restarted it); checked every 10 seconds, never on a database error. + async fn superseded(&self, table: &str, id: Uuid, started_at: time::OffsetDateTime) { + let query = format!("SELECT status = 'running' AND started_at = $2 FROM {table} WHERE id = $1"); + loop { + tokio::time::sleep(Duration::from_secs(10)).await; + match sqlx::query_scalar::<_, Option>(&query).bind(id).bind(started_at).fetch_optional(&self.db).await + { + Ok(Some(Some(true))) => {} + Ok(_) => return, + Err(_) => {} + } + } + } + /// On shutdown: analyses still being written go back to `pending` at once. async fn release_running(&self) { let ids: Vec = self.running.lock().unwrap().drain().collect(); diff --git a/services/worker/src/analysis/questions.rs b/services/worker/src/analysis/questions.rs index ea6623c..d5a7541 100644 --- a/services/worker/src/analysis/questions.rs +++ b/services/worker/src/analysis/questions.rs @@ -27,6 +27,8 @@ pub struct QuestionJob { pub created_at: time::OffsetDateTime, pub fallback_model: Option, pub on_fallback: bool, + /// The token of this attempt (see `Job::started_at`). + pub started_at: time::OffsetDateTime, } impl Analyzer { @@ -37,7 +39,7 @@ impl Analyzer { WHERE id = (SELECT id FROM questions WHERE status = 'pending' AND next_attempt_at <= now() \ ORDER BY next_attempt_at LIMIT 1 FOR UPDATE SKIP LOCKED) \ RETURNING id, analysis_id, question, language, model, prompt_version, attempts, created_at, \ - fallback_model, on_fallback", + fallback_model, on_fallback, started_at", ) .fetch_optional(&self.db) .await?; @@ -91,15 +93,24 @@ impl Analyzer { /// Answers one claimed question and records the outcome. pub async fn handle_question(&self, job: QuestionJob) { let started = Instant::now(); - let result = match self.question_prompt(&job).await { - Ok(p) => { - let model = active_model(&job.model, job.fallback_model.as_deref(), job.on_fallback); - self.llm.complete(model, &p).await.map(|mut c| { - prompt::polish(&job.language, &mut c.content); - c - }) + let work = async { + match self.question_prompt(&job).await { + Ok(p) => { + let model = active_model(&job.model, job.fallback_model.as_deref(), job.on_fallback); + self.llm.complete(model, &p).await.map(|mut c| { + prompt::polish(&job.language, &mut c.content); + c + }) + } + Err(e) => Err(e), + } + }; + let result = tokio::select! { + result = work => result, + () = self.superseded("questions", job.id, job.started_at) => { + tracing::info!(id = %job.id, "answer restarted by an admin; this attempt is dropped"); + return; } - Err(e) => Err(e), }; if let Err(e) = self.finish_question(&job, result, started.elapsed()).await { tracing::error!(id = %job.id, error = %e, "saving the answer failed"); @@ -119,7 +130,7 @@ impl Analyzer { sqlx::query( "UPDATE questions SET status = 'done', answer = $2, model_used = $3, input_tokens = $4, \ output_tokens = $5, cost_usd = $6, error = NULL, completed_at = now() \ - WHERE id = $1 AND status = 'running'", + WHERE id = $1 AND status = 'running' AND started_at = $7", ) .bind(job.id) .bind(&answer) @@ -127,6 +138,7 @@ impl Analyzer { .bind(c.input_tokens) .bind(c.output_tokens) .bind(c.cost_usd) + .bind(job.started_at) .execute(&self.db) .await?; tracing::info!( @@ -140,11 +152,13 @@ impl Analyzer { Next::Retry(wait) => { sqlx::query( "UPDATE questions SET status = 'pending', error = $2, \ - next_attempt_at = now() + make_interval(secs => $3) WHERE id = $1 AND status = 'running'", + next_attempt_at = now() + make_interval(secs => $3) \ + WHERE id = $1 AND status = 'running' AND started_at = $4", ) .bind(job.id) .bind(e.message()) .bind(wait.as_secs_f64()) + .bind(job.started_at) .execute(&self.db) .await?; tracing::warn!(id = %job.id, attempts = job.attempts, error = %e.message(), "answer will be retried"); @@ -152,10 +166,11 @@ impl Analyzer { Next::Fallback => { sqlx::query( "UPDATE questions SET status = 'pending', error = $2, on_fallback = true, attempts = 0, \ - next_attempt_at = now() WHERE id = $1 AND status = 'running'", + next_attempt_at = now() WHERE id = $1 AND status = 'running' AND started_at = $3", ) .bind(job.id) .bind(e.message()) + .bind(job.started_at) .execute(&self.db) .await?; tracing::warn!( @@ -164,7 +179,7 @@ impl Analyzer { ); } Next::Fail => { - self.fail_question(job.id, e.message()).await?; + self.fail_question(job.id, job.started_at, e.message()).await?; tracing::error!(id = %job.id, attempts = job.attempts, error = %e.message(), "answer failed"); } } @@ -174,14 +189,20 @@ impl Analyzer { } /// Marks the question failed and gives its unit back (once). - async fn fail_question(&self, id: Uuid, message: &str) -> Result<(), sqlx::Error> { + async fn fail_question( + &self, + id: Uuid, + started_at: time::OffsetDateTime, + message: &str, + ) -> Result<(), sqlx::Error> { let mut tx = self.db.begin().await?; let row: Option<(Option, i32)> = sqlx::query_as( "UPDATE questions SET status = 'failed', error = $2, completed_at = now() \ - WHERE id = $1 AND status = 'running' RETURNING asked_by, credits_charged", + WHERE id = $1 AND status = 'running' AND started_at = $3 RETURNING asked_by, credits_charged", ) .bind(id) .bind(message) + .bind(started_at) .fetch_optional(&mut *tx) .await?; if let Some((Some(user), units)) = row diff --git a/services/worker/src/tests.rs b/services/worker/src/tests.rs index bf4c9c4..391c85d 100644 --- a/services/worker/src/tests.rs +++ b/services/worker/src/tests.rs @@ -781,4 +781,28 @@ mod analyses { fake.requests.lock().unwrap().iter().map(|b| b["model"].as_str().unwrap().to_string()).collect(); assert_eq!(models, ["cheap/model", "cheap/model", "test/model"]); } + + #[sqlx::test(migrator = "db::MIGRATOR")] + async fn a_restarted_attempt_does_not_write_its_result(db: PgPool) { + let s = setup(&db).await; + let id = request_analysis(&db, &s).await; + let (_, url) = fake_openrouter(vec![reply(&content(Kind::Natal))]).await; + let a = analyzer(&db, &url); + let job = a.claim().await.unwrap().expect("a job"); + // an admin restarts it while the worker is still writing it + sqlx::query("UPDATE analyses SET status = 'pending', started_at = NULL, attempts = 0 WHERE id = $1") + .bind(id) + .execute(&db) + .await + .unwrap(); + a.handle(job).await; + let (status, content): (String, Option) = + sqlx::query_as("SELECT status, content FROM analyses WHERE id = $1").bind(id).fetch_one(&db).await.unwrap(); + assert_eq!((status.as_str(), content), ("pending", None)); + // the new attempt writes it + assert!(run_once(&a).await); + let status: String = + sqlx::query_scalar("SELECT status FROM analyses WHERE id = $1").bind(id).fetch_one(&db).await.unwrap(); + assert_eq!(status, "done"); + } }