Admin: list and restart running generations with the models set now
- Modellek page: analyses and answers being written or waiting, with their model and the one a restart would use; restart one or all - Each attempt's started_at is its token: a restarted attempt still running in the worker cannot write its result and is dropped within 10 seconds Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012vLHzh5qUhFtmJhzkcTbXL
This commit is contained in:
@@ -127,6 +127,7 @@ Hibák formátuma: `{"error": {"code", "message"}}`. A születési adatok hibái
|
||||
- **Fizetős rész:** a részletes elemzés darabra megy. A képletelemzés egy `natal`, a szinasztria-elemzés egy `synastry` egységet von le (`credit_ledger.unit`). Az egységek csomagvásárlásból (Stripe, később) vagy admin jóváírásból jönnek. Havi ingyenes keret nincs.
|
||||
- **Modell:** három beállítás az admin felületen, az OpenRouter listájából (`model_settings` tábla): az elemzések és az éves előrejelzés (képlet-, kapcsolati elemzés, előrejelzés) közös modellje, a kérdés-válasz modellje és a rövid összefoglalóé. Ha nincs beállítva, az `ANALYSIS_MODEL` / `SUMMARY_MODEL` (alapból mindkettő Opus 5.5, a 2026-11-es összevetés legjobbja, lásd `services/worker/README.md`). Csak kötött JSON-választ adó modell választható. A kliens nem választ modellt, és a modell neve felhasználónak sehol nem jelenik meg.
|
||||
- **Tartalék:** ha a választott modell kétszer hibára fut (végleges hibánál egyszer), a worker az alapmodellel próbálja tovább (`fallback_model`, `on_fallback`); csak ha az is elbukik, jár vissza az egység. A Modellek oldal mutatja, hány szövegnél kellett az elmúlt 30 napban.
|
||||
- **Futó generálások:** a Modellek oldalon látszik minden készülő és sorban álló elemzés és válasz, a modelljével és azzal, amivel most indulna; újraindításkor a jelenlegi beállítással indul újra (`GET /v1/admin/jobs`, `POST /v1/admin/jobs/restart`). Minden kísérlet `started_at` értéke a „jegye”: az újraindítás törli, így a workerben még futó régi kísérlet nem írhatja felül az újat, és a worker 10 másodpercen belül eldobja.
|
||||
- **Újragenerálás:** admin minden elemzést egyenként újraírathat (`POST /v1/admin/analyses/{id}/regenerate`): ugyanaz a sor (az azonosító és a megosztások maradnak), a jelenlegi prompttal és modellel, egységlevonás nélkül.
|
||||
- **Újraszámolás:** számítási változás után a `backend recalculate` parancs minden mentett képletet, összevetést és előrejelzést újraszámol, majd minden elemzést újra sorba állít (egységlevonás nélkül).
|
||||
- **Kérés:** a `POST .../analyses` (`{language?}`) létrehoz egy `pending` elemzést, és levon egy egységet; a szöveget a worker írja meg, általában 1–2 perc alatt. A kliens a `GET /v1/analyses/{id}` végponton kérdezi le az állapotát. Ha nincs egység, 402 `insufficient_credits` a válasz (`unit`, `balance` részletekkel). Hiba esetén a worker visszaadja az egységet.
|
||||
|
||||
@@ -146,7 +146,7 @@ async fn request(
|
||||
/// The cache key and the prompt version of a new analysis. The cache key leaves
|
||||
/// out the admin-edited prompt revision, so a chart keeps its analysis when the
|
||||
/// prompt changes; copying from another chart needs the same revision.
|
||||
async fn versions(
|
||||
pub(super) async fn versions(
|
||||
state: &AppState,
|
||||
kind: &str,
|
||||
hash: &str,
|
||||
@@ -359,6 +359,17 @@ pub async fn request_for_forecast(
|
||||
request(&state, &auth, Target::Forecast { forecast_id: forecast.id, hash: forecast.forecast_hash }, req).await
|
||||
}
|
||||
|
||||
/// The hash of what the analysis is about (chart, comparison or forecast),
|
||||
/// part of its cache key; `None` when that was deleted.
|
||||
pub(super) async fn target_hash(db: &PgPool, row: &AnalysisRow) -> Result<Option<String>, sqlx::Error> {
|
||||
let (query, id) = match row.kind.as_str() {
|
||||
"synastry" => ("SELECT synastry_hash FROM synastries WHERE id = $1", row.synastry_id),
|
||||
"forecast" => ("SELECT forecast_hash FROM forecasts WHERE id = $1", row.forecast_id),
|
||||
_ => ("SELECT chart_hash FROM charts WHERE id = $1", row.chart_id),
|
||||
};
|
||||
sqlx::query_scalar(query).bind(id).fetch_optional(db).await
|
||||
}
|
||||
|
||||
/// Admin: writes an analysis again in place (same id, so links stay), with the
|
||||
/// current prompt and model. Nobody is charged.
|
||||
pub async fn regenerate(
|
||||
@@ -371,27 +382,7 @@ pub async fn regenerate(
|
||||
return Err(ApiError::conflict("analysis_in_progress", "Ez az elemzés éppen készül."));
|
||||
}
|
||||
let kind = Kind::parse(&row.kind).ok_or_else(|| ApiError::Internal(format!("unknown kind {}", row.kind)))?;
|
||||
let hash: Option<String> = match kind {
|
||||
Kind::Question => None,
|
||||
Kind::Natal | Kind::Summary => {
|
||||
sqlx::query_scalar("SELECT chart_hash FROM charts WHERE id = $1")
|
||||
.bind(row.chart_id)
|
||||
.fetch_optional(&state.db)
|
||||
.await?
|
||||
}
|
||||
Kind::Synastry => {
|
||||
sqlx::query_scalar("SELECT synastry_hash FROM synastries WHERE id = $1")
|
||||
.bind(row.synastry_id)
|
||||
.fetch_optional(&state.db)
|
||||
.await?
|
||||
}
|
||||
Kind::Forecast => {
|
||||
sqlx::query_scalar("SELECT forecast_hash FROM forecasts WHERE id = $1")
|
||||
.bind(row.forecast_id)
|
||||
.fetch_optional(&state.db)
|
||||
.await?
|
||||
}
|
||||
};
|
||||
let hash = target_hash(&state.db, &row).await?;
|
||||
let hash = hash.ok_or(ApiError::NotFound("analysis"))?;
|
||||
let (model, fallback) = crate::models::models_for(&state, kind).await?;
|
||||
let (cache_key, prompt_version) = versions(&state, kind.as_str(), &hash, &model, &row.language).await?;
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
//! Admin: the analyses and answers being written or waiting in the queue,
|
||||
//! and restarting them with the models set now (e.g. after switching away
|
||||
//! from a slow model). A restarted attempt that is still running in the
|
||||
//! worker is dropped there and cannot overwrite the new one.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use analysis_prompt::Kind;
|
||||
use axum::Json;
|
||||
use axum::extract::State;
|
||||
use axum_extra::extract::WithRejection;
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Value, json};
|
||||
use time::OffsetDateTime;
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::analyses::{find_analysis, target_hash, versions};
|
||||
use super::{Body, St};
|
||||
use crate::AppState;
|
||||
use crate::auth::AdminUser;
|
||||
use crate::error::ApiResult;
|
||||
use crate::models::{Function, models_for};
|
||||
use crate::views::{rfc3339, rfc3339_opt};
|
||||
|
||||
type JobRow = (
|
||||
Uuid,
|
||||
String,
|
||||
String,
|
||||
String,
|
||||
Option<String>,
|
||||
bool,
|
||||
i32,
|
||||
Option<OffsetDateTime>,
|
||||
OffsetDateTime,
|
||||
OffsetDateTime,
|
||||
Option<String>,
|
||||
);
|
||||
|
||||
const COLUMNS: &str =
|
||||
"id, kind, status, model, fallback_model, on_fallback, attempts, started_at, created_at, next_attempt_at, error";
|
||||
|
||||
async fn open_jobs(state: &AppState) -> Result<Vec<(&'static str, JobRow)>, sqlx::Error> {
|
||||
let analyses: Vec<JobRow> = sqlx::query_as(&format!(
|
||||
"SELECT {COLUMNS} FROM analyses WHERE status IN ('pending', 'running') ORDER BY created_at"
|
||||
))
|
||||
.fetch_all(&state.db)
|
||||
.await?;
|
||||
let questions: Vec<JobRow> = sqlx::query_as(
|
||||
&"SELECT id, 'question', status, model, fallback_model, on_fallback, attempts, started_at, created_at, \
|
||||
next_attempt_at, error \
|
||||
FROM questions WHERE status IN ('pending', 'running') ORDER BY created_at"
|
||||
.to_string(),
|
||||
)
|
||||
.fetch_all(&state.db)
|
||||
.await?;
|
||||
Ok(analyses.into_iter().map(|r| ("analysis", r)).chain(questions.into_iter().map(|r| ("question", r))).collect())
|
||||
}
|
||||
|
||||
pub async fn list(State(state): St, _admin: AdminUser) -> ApiResult<Json<Value>> {
|
||||
Ok(Json(json!({ "jobs": views(&state).await? })))
|
||||
}
|
||||
|
||||
async fn views(state: &AppState) -> Result<Vec<Value>, sqlx::Error> {
|
||||
// The models a restart would use, per function.
|
||||
let mut next: HashMap<&'static str, (String, Option<String>)> = HashMap::new();
|
||||
for (function, kind) in
|
||||
[(Function::Summary, Kind::Summary), (Function::Analysis, Kind::Natal), (Function::Question, Kind::Question)]
|
||||
{
|
||||
next.insert(function.key(), models_for(state, kind).await?);
|
||||
}
|
||||
Ok(open_jobs(state)
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|(table, (id, kind, status, model, fallback, on_fallback, attempts, started, created, due, error))| {
|
||||
let function = Function::of(Kind::parse(&kind).unwrap_or(Kind::Natal)).key();
|
||||
let (next_model, next_fallback) = &next[function];
|
||||
let active = if on_fallback { fallback.clone().unwrap_or_else(|| model.clone()) } else { model.clone() };
|
||||
json!({
|
||||
"id": id, "type": table, "kind": kind, "status": status, "model": active, "on_fallback": on_fallback,
|
||||
"next_model": next_model, "next_fallback": next_fallback, "attempts": attempts,
|
||||
"started_at": rfc3339_opt(started), "created_at": rfc3339(created), "next_attempt_at": rfc3339(due),
|
||||
"error": error,
|
||||
})
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Restart {
|
||||
/// The jobs to restart; all open ones when missing.
|
||||
ids: Option<Vec<Uuid>>,
|
||||
}
|
||||
|
||||
pub async fn restart(
|
||||
State(state): St,
|
||||
AdminUser(admin): AdminUser,
|
||||
WithRejection(Json(req), _): Body<Restart>,
|
||||
) -> ApiResult<Json<Value>> {
|
||||
let mut restarted = 0;
|
||||
for (table, (id, kind, ..)) in open_jobs(&state).await? {
|
||||
if req.ids.as_ref().is_some_and(|ids| !ids.contains(&id)) {
|
||||
continue;
|
||||
}
|
||||
let done = if table == "question" {
|
||||
let language: String = sqlx::query_scalar("SELECT language FROM questions WHERE id = $1")
|
||||
.bind(id)
|
||||
.fetch_one(&state.db)
|
||||
.await?;
|
||||
let (model, fallback, prompt_version) = super::questions::model_and_version(&state, &language).await?;
|
||||
sqlx::query(
|
||||
"UPDATE questions SET status = 'pending', model = $2, fallback_model = $3, on_fallback = false, \
|
||||
prompt_version = $4, attempts = 0, next_attempt_at = now(), started_at = NULL, error = NULL \
|
||||
WHERE id = $1 AND status IN ('pending', 'running')",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(&model)
|
||||
.bind(&fallback)
|
||||
.bind(&prompt_version)
|
||||
.execute(&state.db)
|
||||
.await?
|
||||
.rows_affected()
|
||||
} else {
|
||||
let Some(row) = find_analysis(&state.db, id).await? else { continue };
|
||||
let Some(hash) = target_hash(&state.db, &row).await? else { continue };
|
||||
let (model, fallback) = models_for(&state, Kind::parse(&kind).unwrap_or(Kind::Natal)).await?;
|
||||
let (cache_key, prompt_version) = versions(&state, &kind, &hash, &model, &row.language).await?;
|
||||
sqlx::query(
|
||||
"UPDATE analyses SET status = 'pending', model = $2, fallback_model = $3, on_fallback = false, \
|
||||
cache_key = $4, prompt_version = $5, attempts = 0, next_attempt_at = now(), started_at = NULL, \
|
||||
error = NULL \
|
||||
WHERE id = $1 AND status IN ('pending', 'running')",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(&model)
|
||||
.bind(&fallback)
|
||||
.bind(&cache_key)
|
||||
.bind(&prompt_version)
|
||||
.execute(&state.db)
|
||||
.await?
|
||||
.rows_affected()
|
||||
};
|
||||
restarted += done;
|
||||
}
|
||||
if restarted > 0 {
|
||||
sqlx::query("SELECT pg_notify('analyses', '')").execute(&state.db).await?;
|
||||
}
|
||||
tracing::info!(admin = %admin.id(), restarted, "generations restarted");
|
||||
Ok(Json(json!({ "restarted": restarted, "jobs": views(&state).await? })))
|
||||
}
|
||||
@@ -12,6 +12,7 @@ mod auth;
|
||||
mod charts;
|
||||
mod forecasts;
|
||||
mod invitations;
|
||||
mod jobs;
|
||||
mod models;
|
||||
mod payments;
|
||||
mod prompts;
|
||||
@@ -116,6 +117,8 @@ pub fn router(state: Arc<AppState>) -> Router {
|
||||
.route("/v1/admin/prompts/{kind}/{language}", axum::routing::put(prompts::save))
|
||||
.route("/v1/admin/prompts/{kind}/{language}/preview", post(prompts::start_preview))
|
||||
.route("/v1/admin/previews/{id}", get(prompts::get_preview))
|
||||
.route("/v1/admin/jobs", get(jobs::list))
|
||||
.route("/v1/admin/jobs/restart", post(jobs::restart))
|
||||
.route("/v1/admin/models", get(models::list))
|
||||
.route("/v1/admin/models/catalog", get(models::catalog))
|
||||
.route("/v1/admin/models/{kind}", axum::routing::put(models::save))
|
||||
|
||||
@@ -66,7 +66,7 @@ async fn find_question(db: &PgPool, id: Uuid) -> Result<Option<QuestionRow>, sql
|
||||
|
||||
/// The question model (and its fallback) and the version of the question
|
||||
/// prompt in force.
|
||||
async fn model_and_version(
|
||||
pub(super) async fn model_and_version(
|
||||
state: &crate::AppState,
|
||||
language: &str,
|
||||
) -> Result<(String, Option<String>, String), sqlx::Error> {
|
||||
|
||||
@@ -1639,3 +1639,59 @@ async fn questions_about_an_analysis_use_question_units(pool: PgPool) {
|
||||
);
|
||||
assert_eq!(left().await, json!(5), "regenerating charges nobody");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrator = "db::MIGRATOR")]
|
||||
async fn admins_restart_open_generations_with_the_current_models(pool: PgPool) {
|
||||
let app = TestApp::new(pool.clone()).await;
|
||||
let admin = app.register("admin@example.com", "Admin").await;
|
||||
let anna = app.register("anna@example.com", "Anna").await;
|
||||
assert_eq!(app.get("/v1/admin/jobs", Some(&anna)).await.status, StatusCode::FORBIDDEN);
|
||||
let put = |model: &str| {
|
||||
let (app, admin, body) = (&app, &admin, json!({ "model": model }));
|
||||
async move { app.call("PUT", "/v1/admin/models/analysis", Some(admin), Some(body)).await }
|
||||
};
|
||||
assert_eq!(put("openai/gpt-x").await.status, StatusCode::OK);
|
||||
let chart = app.chart(&admin, "Saját", true, "1990-03-15").await;
|
||||
let chart_id = chart["id"].as_str().unwrap();
|
||||
let r = app.post(&format!("/v1/charts/{chart_id}/analyses"), Some(&admin), json!({})).await;
|
||||
let id = r.body["id"].as_str().unwrap().to_string();
|
||||
// the worker is writing it with the chosen model
|
||||
sqlx::query("UPDATE analyses SET status = 'running', started_at = now(), attempts = 1 WHERE id = $1::uuid")
|
||||
.bind(&id)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// after switching back to the default, the list shows what a restart would use
|
||||
assert_eq!(
|
||||
app.call("PUT", "/v1/admin/models/analysis", Some(&admin), Some(json!({"model": null}))).await.status,
|
||||
StatusCode::OK
|
||||
);
|
||||
let jobs = app.get("/v1/admin/jobs", Some(&admin)).await.body["jobs"].clone();
|
||||
let job = jobs.as_array().unwrap().iter().find(|j| j["id"] == json!(id)).expect("the analysis is listed");
|
||||
assert_eq!(
|
||||
(&job["status"], &job["model"], &job["next_model"]),
|
||||
(&json!("running"), &json!("openai/gpt-x"), &json!("anthropic/claude-test"))
|
||||
);
|
||||
|
||||
let r = app.post("/v1/admin/jobs/restart", Some(&admin), json!({ "ids": [id] })).await;
|
||||
assert_eq!((r.status, &r.body["restarted"]), (StatusCode::OK, &json!(1)), "{}", r.body);
|
||||
let (status, model, fallback, started, attempts): (
|
||||
String,
|
||||
String,
|
||||
Option<String>,
|
||||
Option<time::OffsetDateTime>,
|
||||
i32,
|
||||
) = sqlx::query_as("SELECT status, model, fallback_model, started_at, attempts FROM analyses WHERE id = $1::uuid")
|
||||
.bind(&id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
(status.as_str(), model.as_str(), fallback, started, attempts),
|
||||
("pending", "anthropic/claude-test", None, None, 0)
|
||||
);
|
||||
// finished ones are left alone
|
||||
sqlx::query("UPDATE analyses SET status = 'done', content = '{}'").execute(&pool).await.unwrap();
|
||||
assert_eq!(app.post("/v1/admin/jobs/restart", Some(&admin), json!({})).await.body["restarted"], 0);
|
||||
}
|
||||
|
||||
@@ -6,6 +6,8 @@ import { api, errorText } from "@/lib/api";
|
||||
import { formatDate } from "@/lib/astro";
|
||||
import type { CatalogModel, ModelFunction } from "@/lib/types";
|
||||
|
||||
import { RunningJobs } from "./RunningJobs";
|
||||
|
||||
const LABELS: Record<ModelFunction["kind"], { title: string; about: string }> = {
|
||||
summary: { title: "Tömör összefoglaló", about: "Ingyenes, minden új képlethez magától elkészül. Olcsó, gyors modell elég." },
|
||||
analysis: {
|
||||
@@ -66,6 +68,7 @@ export function AdminModels() {
|
||||
{functions.map((f) => (
|
||||
<FunctionModel key={f.kind} fn={f} catalog={catalog} onSaved={replace} />
|
||||
))}
|
||||
<RunningJobs />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
"use client";
|
||||
|
||||
import { useCallback, useEffect, useState } from "react";
|
||||
|
||||
import { api, errorText } from "@/lib/api";
|
||||
import type { AdminJob } from "@/lib/types";
|
||||
|
||||
const KINDS: Record<AdminJob["kind"], string> = {
|
||||
summary: "Összefoglaló",
|
||||
natal: "Képletelemzés",
|
||||
synastry: "Kapcsolati elemzés",
|
||||
forecast: "Éves előrejelzés",
|
||||
question: "Kérdés-válasz",
|
||||
};
|
||||
|
||||
const REFRESH_MS = 10_000;
|
||||
|
||||
function since(iso: string): string {
|
||||
const minutes = Math.max(0, Math.round((Date.now() - new Date(iso).getTime()) / 60_000));
|
||||
return minutes < 1 ? "most" : `${minutes} perce`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyses and answers being written or waiting. A restart starts them again
|
||||
* with the models set now; an attempt still running is dropped by the worker.
|
||||
*/
|
||||
export function RunningJobs() {
|
||||
const [jobs, setJobs] = useState<AdminJob[] | null>(null);
|
||||
const [busy, setBusy] = useState(false);
|
||||
const [asking, setAsking] = useState(false);
|
||||
const [message, setMessage] = useState<string | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
const load = useCallback(() => {
|
||||
api<{ jobs: AdminJob[] }>("/admin/jobs")
|
||||
.then((d) => setJobs(d.jobs))
|
||||
.catch((e) => setError(errorText(e)));
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
load();
|
||||
const timer = setInterval(load, REFRESH_MS);
|
||||
return () => clearInterval(timer);
|
||||
}, [load]);
|
||||
|
||||
async function restart(ids?: string[]) {
|
||||
setBusy(true);
|
||||
setError(null);
|
||||
setMessage(null);
|
||||
try {
|
||||
const r = await api<{ restarted: number; jobs: AdminJob[] }>("/admin/jobs/restart", { body: ids ? { ids } : {} });
|
||||
setJobs(r.jobs);
|
||||
setMessage(r.restarted > 0 ? `${r.restarted} generálás újraindítva.` : "Nem volt mit újraindítani.");
|
||||
setAsking(false);
|
||||
} catch (e) {
|
||||
setError(errorText(e));
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
}
|
||||
|
||||
const changed = (jobs ?? []).filter((j) => j.model !== j.next_model).length;
|
||||
return (
|
||||
<section className="card model-card" aria-labelledby="jobs-title">
|
||||
<div className="model-head">
|
||||
<div>
|
||||
<h2 id="jobs-title">Futó generálások</h2>
|
||||
<p className="hint">
|
||||
Az éppen készülő és a sorban álló elemzések és válaszok. Újraindításkor mindegyik azzal a modellel indul újra,
|
||||
ami most be van állítva; a még futó régi kérést a worker eldobja. Senkitől nem von le újabb egységet.
|
||||
</p>
|
||||
</div>
|
||||
{jobs && jobs.length > 0 && (
|
||||
<div className="regenerate">
|
||||
{asking ? (
|
||||
<>
|
||||
<span className="hint">
|
||||
{jobs.length} generálás indul újra{changed > 0 ? `, ebből ${changed} másik modellel` : ""}.
|
||||
</span>
|
||||
<button type="button" className="btn btn-sm" onClick={() => restart()} disabled={busy}>
|
||||
{busy ? "Újraindítás…" : "Igen, mind"}
|
||||
</button>
|
||||
<button type="button" className="btn btn-sm btn-ghost" onClick={() => setAsking(false)} disabled={busy}>
|
||||
Mégse
|
||||
</button>
|
||||
</>
|
||||
) : (
|
||||
<button type="button" className="btn btn-sm" onClick={() => setAsking(true)}>
|
||||
↻ Mind újraindítása ({jobs.length})
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
{message && (
|
||||
<p className="msg" role="status">
|
||||
{message}
|
||||
</p>
|
||||
)}
|
||||
{error && (
|
||||
<p className="msg msg-error" role="alert">
|
||||
{error}
|
||||
</p>
|
||||
)}
|
||||
{jobs === null ? (
|
||||
<p className="muted">Betöltés…</p>
|
||||
) : jobs.length === 0 ? (
|
||||
<p className="muted">Most semmi nem készül és semmi nem vár a sorban.</p>
|
||||
) : (
|
||||
<ul className="model-options">
|
||||
{jobs.map((j) => (
|
||||
<li key={j.id} className="job-row">
|
||||
<span>
|
||||
<strong>{KINDS[j.kind]}</strong>
|
||||
<span className="hint">
|
||||
{j.status === "running" && j.started_at
|
||||
? `fut, ${since(j.started_at)} indult`
|
||||
: `vár (kérve ${since(j.created_at)})`}
|
||||
{j.attempts > 0 ? ` · ${j.attempts}. kísérlet` : ""}
|
||||
{j.on_fallback ? " · tartalék modellel" : ""}
|
||||
</span>
|
||||
{j.error && <span className="hint">utolsó hiba: {j.error}</span>}
|
||||
</span>
|
||||
<span className="model-price">
|
||||
<code>{j.model}</code>
|
||||
{j.model !== j.next_model ? <strong>→ {j.next_model}</strong> : <span>újraindítva is ez</span>}
|
||||
</span>
|
||||
<button type="button" className="btn btn-sm btn-ghost" onClick={() => restart([j.id])} disabled={busy}>
|
||||
↻
|
||||
<span className="sr-only"> Újraindítás</span>
|
||||
</button>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
)}
|
||||
</section>
|
||||
);
|
||||
}
|
||||
@@ -644,3 +644,6 @@ a.pair-row:hover strong { color: var(--gold); }
|
||||
.qa-form { display: flex; flex-direction: column; gap: 8px; }
|
||||
.qa-form textarea { resize: vertical; min-height: 84px; }
|
||||
.hint.warn { color: var(--gold); }
|
||||
.job-row { display: grid; grid-template-columns: minmax(0, 1fr) auto auto; gap: 12px; align-items: center; padding: 8px 12px; border: 1px solid var(--line); border-radius: 8px; }
|
||||
.job-row > span:first-child { display: flex; flex-direction: column; min-width: 0; }
|
||||
@media (max-width: 640px) { .job-row { grid-template-columns: minmax(0, 1fr) auto; } .job-row .model-price { grid-column: 1 / -1; align-items: flex-start; } }
|
||||
|
||||
@@ -266,6 +266,25 @@ export interface ModelFunction {
|
||||
finished_30d: number;
|
||||
}
|
||||
|
||||
/** Admin: an analysis or answer being written or waiting in the queue. */
|
||||
export interface AdminJob {
|
||||
id: string;
|
||||
type: "analysis" | "question";
|
||||
kind: "summary" | "natal" | "synastry" | "forecast" | "question";
|
||||
status: "pending" | "running";
|
||||
/** The model of the current attempt. */
|
||||
model: string;
|
||||
on_fallback: boolean;
|
||||
/** What a restart would use now. */
|
||||
next_model: string;
|
||||
next_fallback: string | null;
|
||||
attempts: number;
|
||||
started_at: string | null;
|
||||
created_at: string;
|
||||
next_attempt_at: string;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
/** A model OpenRouter offers; prices in USD per million tokens. */
|
||||
export interface CatalogModel {
|
||||
id: string;
|
||||
|
||||
@@ -47,6 +47,9 @@ pub struct Job {
|
||||
/// The default model, when the chosen one is another (see the module docs).
|
||||
pub fallback_model: Option<String>,
|
||||
pub on_fallback: bool,
|
||||
/// When this attempt started; also its token: an admin restart clears it,
|
||||
/// so a superseded attempt neither writes its result nor keeps running.
|
||||
pub started_at: time::OffsetDateTime,
|
||||
}
|
||||
|
||||
impl Job {
|
||||
@@ -125,7 +128,7 @@ impl Analyzer {
|
||||
WHERE id = (SELECT id FROM analyses WHERE status = 'pending' AND next_attempt_at <= now() \
|
||||
ORDER BY next_attempt_at LIMIT 1 FOR UPDATE SKIP LOCKED) \
|
||||
RETURNING id, kind, chart_id, synastry_id, forecast_id, language, model, prompt_version, attempts, \
|
||||
fallback_model, on_fallback",
|
||||
fallback_model, on_fallback, started_at",
|
||||
)
|
||||
.fetch_optional(&self.db)
|
||||
.await?;
|
||||
@@ -273,12 +276,21 @@ impl Analyzer {
|
||||
/// Writes one claimed analysis and records the outcome.
|
||||
pub async fn handle(&self, job: Job) {
|
||||
let started = Instant::now();
|
||||
let result = match self.prompt_for(&job).await {
|
||||
Ok(prompt) => self.llm.complete(job.active_model(), &prompt).await.map(|mut c| {
|
||||
prompt::polish(&job.language, &mut c.content);
|
||||
c
|
||||
}),
|
||||
Err(e) => Err(e),
|
||||
let work = async {
|
||||
match self.prompt_for(&job).await {
|
||||
Ok(prompt) => self.llm.complete(job.active_model(), &prompt).await.map(|mut c| {
|
||||
prompt::polish(&job.language, &mut c.content);
|
||||
c
|
||||
}),
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
};
|
||||
let result = tokio::select! {
|
||||
result = work => result,
|
||||
() = self.superseded("analyses", job.id, job.started_at) => {
|
||||
tracing::info!(id = %job.id, "analysis restarted by an admin; this attempt is dropped");
|
||||
return;
|
||||
}
|
||||
};
|
||||
if let Err(e) = self.finish(&job, result, started.elapsed()).await {
|
||||
tracing::error!(id = %job.id, error = %e, "saving the analysis result failed");
|
||||
@@ -292,7 +304,7 @@ impl Analyzer {
|
||||
sqlx::query(
|
||||
"UPDATE analyses SET status = 'done', content = $2, model_used = $3, input_tokens = $4, \
|
||||
output_tokens = $5, cost_usd = $6, error = NULL, completed_at = now() \
|
||||
WHERE id = $1 AND status = 'running'",
|
||||
WHERE id = $1 AND status = 'running' AND started_at = $7",
|
||||
)
|
||||
.bind(job.id)
|
||||
.bind(&c.content)
|
||||
@@ -300,6 +312,7 @@ impl Analyzer {
|
||||
.bind(c.input_tokens)
|
||||
.bind(c.output_tokens)
|
||||
.bind(c.cost_usd)
|
||||
.bind(job.started_at)
|
||||
.execute(&self.db)
|
||||
.await?;
|
||||
tracing::info!(
|
||||
@@ -313,11 +326,13 @@ impl Analyzer {
|
||||
Next::Retry(wait) => {
|
||||
sqlx::query(
|
||||
"UPDATE analyses SET status = 'pending', error = $2, \
|
||||
next_attempt_at = now() + make_interval(secs => $3) WHERE id = $1 AND status = 'running'",
|
||||
next_attempt_at = now() + make_interval(secs => $3) \
|
||||
WHERE id = $1 AND status = 'running' AND started_at = $4",
|
||||
)
|
||||
.bind(job.id)
|
||||
.bind(e.message())
|
||||
.bind(wait.as_secs_f64())
|
||||
.bind(job.started_at)
|
||||
.execute(&self.db)
|
||||
.await?;
|
||||
tracing::warn!(id = %job.id, attempts = job.attempts, retry_in_secs = wait.as_secs(), error = %e.message(), "analysis will be retried");
|
||||
@@ -325,10 +340,11 @@ impl Analyzer {
|
||||
Next::Fallback => {
|
||||
sqlx::query(
|
||||
"UPDATE analyses SET status = 'pending', error = $2, on_fallback = true, attempts = 0, \
|
||||
next_attempt_at = now() WHERE id = $1 AND status = 'running'",
|
||||
next_attempt_at = now() WHERE id = $1 AND status = 'running' AND started_at = $3",
|
||||
)
|
||||
.bind(job.id)
|
||||
.bind(e.message())
|
||||
.bind(job.started_at)
|
||||
.execute(&self.db)
|
||||
.await?;
|
||||
tracing::warn!(
|
||||
@@ -337,7 +353,7 @@ impl Analyzer {
|
||||
);
|
||||
}
|
||||
Next::Fail => {
|
||||
self.fail(job.id, e.message()).await?;
|
||||
self.fail(job.id, job.started_at, e.message()).await?;
|
||||
tracing::error!(id = %job.id, attempts = job.attempts, error = %e.message(), "analysis failed");
|
||||
}
|
||||
}
|
||||
@@ -347,14 +363,15 @@ impl Analyzer {
|
||||
}
|
||||
|
||||
/// Marks the analysis failed and gives its unit back (once).
|
||||
async fn fail(&self, id: Uuid, message: &str) -> Result<(), sqlx::Error> {
|
||||
async fn fail(&self, id: Uuid, started_at: time::OffsetDateTime, message: &str) -> Result<(), sqlx::Error> {
|
||||
let mut tx = self.db.begin().await?;
|
||||
let row: Option<(Option<Uuid>, i32, String)> = sqlx::query_as(
|
||||
"UPDATE analyses SET status = 'failed', error = $2, completed_at = now() \
|
||||
WHERE id = $1 AND status = 'running' RETURNING requested_by, credits_charged, kind",
|
||||
WHERE id = $1 AND status = 'running' AND started_at = $3 RETURNING requested_by, credits_charged, kind",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(message)
|
||||
.bind(started_at)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await?;
|
||||
if let Some((Some(user), units, kind)) = row
|
||||
@@ -407,6 +424,21 @@ impl Analyzer {
|
||||
Ok(result.rows_affected())
|
||||
}
|
||||
|
||||
/// Resolves when this attempt is no longer the current one (an admin
|
||||
/// restarted it); checked every 10 seconds, never on a database error.
|
||||
async fn superseded(&self, table: &str, id: Uuid, started_at: time::OffsetDateTime) {
|
||||
let query = format!("SELECT status = 'running' AND started_at = $2 FROM {table} WHERE id = $1");
|
||||
loop {
|
||||
tokio::time::sleep(Duration::from_secs(10)).await;
|
||||
match sqlx::query_scalar::<_, Option<bool>>(&query).bind(id).bind(started_at).fetch_optional(&self.db).await
|
||||
{
|
||||
Ok(Some(Some(true))) => {}
|
||||
Ok(_) => return,
|
||||
Err(_) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// On shutdown: analyses still being written go back to `pending` at once.
|
||||
async fn release_running(&self) {
|
||||
let ids: Vec<Uuid> = self.running.lock().unwrap().drain().collect();
|
||||
|
||||
@@ -27,6 +27,8 @@ pub struct QuestionJob {
|
||||
pub created_at: time::OffsetDateTime,
|
||||
pub fallback_model: Option<String>,
|
||||
pub on_fallback: bool,
|
||||
/// The token of this attempt (see `Job::started_at`).
|
||||
pub started_at: time::OffsetDateTime,
|
||||
}
|
||||
|
||||
impl Analyzer {
|
||||
@@ -37,7 +39,7 @@ impl Analyzer {
|
||||
WHERE id = (SELECT id FROM questions WHERE status = 'pending' AND next_attempt_at <= now() \
|
||||
ORDER BY next_attempt_at LIMIT 1 FOR UPDATE SKIP LOCKED) \
|
||||
RETURNING id, analysis_id, question, language, model, prompt_version, attempts, created_at, \
|
||||
fallback_model, on_fallback",
|
||||
fallback_model, on_fallback, started_at",
|
||||
)
|
||||
.fetch_optional(&self.db)
|
||||
.await?;
|
||||
@@ -91,15 +93,24 @@ impl Analyzer {
|
||||
/// Answers one claimed question and records the outcome.
|
||||
pub async fn handle_question(&self, job: QuestionJob) {
|
||||
let started = Instant::now();
|
||||
let result = match self.question_prompt(&job).await {
|
||||
Ok(p) => {
|
||||
let model = active_model(&job.model, job.fallback_model.as_deref(), job.on_fallback);
|
||||
self.llm.complete(model, &p).await.map(|mut c| {
|
||||
prompt::polish(&job.language, &mut c.content);
|
||||
c
|
||||
})
|
||||
let work = async {
|
||||
match self.question_prompt(&job).await {
|
||||
Ok(p) => {
|
||||
let model = active_model(&job.model, job.fallback_model.as_deref(), job.on_fallback);
|
||||
self.llm.complete(model, &p).await.map(|mut c| {
|
||||
prompt::polish(&job.language, &mut c.content);
|
||||
c
|
||||
})
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
};
|
||||
let result = tokio::select! {
|
||||
result = work => result,
|
||||
() = self.superseded("questions", job.id, job.started_at) => {
|
||||
tracing::info!(id = %job.id, "answer restarted by an admin; this attempt is dropped");
|
||||
return;
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
};
|
||||
if let Err(e) = self.finish_question(&job, result, started.elapsed()).await {
|
||||
tracing::error!(id = %job.id, error = %e, "saving the answer failed");
|
||||
@@ -119,7 +130,7 @@ impl Analyzer {
|
||||
sqlx::query(
|
||||
"UPDATE questions SET status = 'done', answer = $2, model_used = $3, input_tokens = $4, \
|
||||
output_tokens = $5, cost_usd = $6, error = NULL, completed_at = now() \
|
||||
WHERE id = $1 AND status = 'running'",
|
||||
WHERE id = $1 AND status = 'running' AND started_at = $7",
|
||||
)
|
||||
.bind(job.id)
|
||||
.bind(&answer)
|
||||
@@ -127,6 +138,7 @@ impl Analyzer {
|
||||
.bind(c.input_tokens)
|
||||
.bind(c.output_tokens)
|
||||
.bind(c.cost_usd)
|
||||
.bind(job.started_at)
|
||||
.execute(&self.db)
|
||||
.await?;
|
||||
tracing::info!(
|
||||
@@ -140,11 +152,13 @@ impl Analyzer {
|
||||
Next::Retry(wait) => {
|
||||
sqlx::query(
|
||||
"UPDATE questions SET status = 'pending', error = $2, \
|
||||
next_attempt_at = now() + make_interval(secs => $3) WHERE id = $1 AND status = 'running'",
|
||||
next_attempt_at = now() + make_interval(secs => $3) \
|
||||
WHERE id = $1 AND status = 'running' AND started_at = $4",
|
||||
)
|
||||
.bind(job.id)
|
||||
.bind(e.message())
|
||||
.bind(wait.as_secs_f64())
|
||||
.bind(job.started_at)
|
||||
.execute(&self.db)
|
||||
.await?;
|
||||
tracing::warn!(id = %job.id, attempts = job.attempts, error = %e.message(), "answer will be retried");
|
||||
@@ -152,10 +166,11 @@ impl Analyzer {
|
||||
Next::Fallback => {
|
||||
sqlx::query(
|
||||
"UPDATE questions SET status = 'pending', error = $2, on_fallback = true, attempts = 0, \
|
||||
next_attempt_at = now() WHERE id = $1 AND status = 'running'",
|
||||
next_attempt_at = now() WHERE id = $1 AND status = 'running' AND started_at = $3",
|
||||
)
|
||||
.bind(job.id)
|
||||
.bind(e.message())
|
||||
.bind(job.started_at)
|
||||
.execute(&self.db)
|
||||
.await?;
|
||||
tracing::warn!(
|
||||
@@ -164,7 +179,7 @@ impl Analyzer {
|
||||
);
|
||||
}
|
||||
Next::Fail => {
|
||||
self.fail_question(job.id, e.message()).await?;
|
||||
self.fail_question(job.id, job.started_at, e.message()).await?;
|
||||
tracing::error!(id = %job.id, attempts = job.attempts, error = %e.message(), "answer failed");
|
||||
}
|
||||
}
|
||||
@@ -174,14 +189,20 @@ impl Analyzer {
|
||||
}
|
||||
|
||||
/// Marks the question failed and gives its unit back (once).
|
||||
async fn fail_question(&self, id: Uuid, message: &str) -> Result<(), sqlx::Error> {
|
||||
async fn fail_question(
|
||||
&self,
|
||||
id: Uuid,
|
||||
started_at: time::OffsetDateTime,
|
||||
message: &str,
|
||||
) -> Result<(), sqlx::Error> {
|
||||
let mut tx = self.db.begin().await?;
|
||||
let row: Option<(Option<Uuid>, i32)> = sqlx::query_as(
|
||||
"UPDATE questions SET status = 'failed', error = $2, completed_at = now() \
|
||||
WHERE id = $1 AND status = 'running' RETURNING asked_by, credits_charged",
|
||||
WHERE id = $1 AND status = 'running' AND started_at = $3 RETURNING asked_by, credits_charged",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(message)
|
||||
.bind(started_at)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await?;
|
||||
if let Some((Some(user), units)) = row
|
||||
|
||||
@@ -781,4 +781,28 @@ mod analyses {
|
||||
fake.requests.lock().unwrap().iter().map(|b| b["model"].as_str().unwrap().to_string()).collect();
|
||||
assert_eq!(models, ["cheap/model", "cheap/model", "test/model"]);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrator = "db::MIGRATOR")]
|
||||
async fn a_restarted_attempt_does_not_write_its_result(db: PgPool) {
|
||||
let s = setup(&db).await;
|
||||
let id = request_analysis(&db, &s).await;
|
||||
let (_, url) = fake_openrouter(vec![reply(&content(Kind::Natal))]).await;
|
||||
let a = analyzer(&db, &url);
|
||||
let job = a.claim().await.unwrap().expect("a job");
|
||||
// an admin restarts it while the worker is still writing it
|
||||
sqlx::query("UPDATE analyses SET status = 'pending', started_at = NULL, attempts = 0 WHERE id = $1")
|
||||
.bind(id)
|
||||
.execute(&db)
|
||||
.await
|
||||
.unwrap();
|
||||
a.handle(job).await;
|
||||
let (status, content): (String, Option<Value>) =
|
||||
sqlx::query_as("SELECT status, content FROM analyses WHERE id = $1").bind(id).fetch_one(&db).await.unwrap();
|
||||
assert_eq!((status.as_str(), content), ("pending", None));
|
||||
// the new attempt writes it
|
||||
assert!(run_once(&a).await);
|
||||
let status: String =
|
||||
sqlx::query_scalar("SELECT status FROM analyses WHERE id = $1").bind(id).fetch_one(&db).await.unwrap();
|
||||
assert_eq!(status, "done");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user