From 769b4791aebb598e7644e95f32ef170768c81415 Mon Sep 17 00:00:00 2001 From: feco Date: Wed, 18 Nov 2026 21:11:08 +0000 Subject: [PATCH] Models: one for analyses and forecasts, one for answers; Opus 5.5 default and fallback - Admin Modellek: summary, analyses + forecast (shared), question-answer (own model) - Every default is Opus 5.5; when the chosen model fails twice (permanent: once), the default model takes over with fresh attempts before the unit is refunded - The page shows how often the fallback was needed in the last 30 days Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012vLHzh5qUhFtmJhzkcTbXL --- .../0019_model_functions_fallback.sql | 18 ++++ docs/kerdesek.md | 2 +- services/backend/README.md | 7 +- services/backend/src/config.rs | 10 +- services/backend/src/models.rs | 68 ++++++++++--- services/backend/src/routes/analyses.rs | 14 ++- services/backend/src/routes/models.rs | 59 +++++++----- services/backend/src/routes/questions.rs | 39 ++++---- services/backend/src/tests.rs | 42 ++++---- .../src/app/admin/modellek/AdminModels.tsx | 22 +++-- .../src/app/admin/promptok/AdminPrompts.tsx | 2 +- services/frontend/src/app/csomagok/page.tsx | 2 +- services/frontend/src/app/globals.css | 1 + services/frontend/src/lib/types.ts | 7 +- services/worker/README.md | 2 +- services/worker/src/analysis/mod.rs | 96 +++++++++++++++---- services/worker/src/analysis/questions.rs | 72 +++++++++----- services/worker/src/tests.rs | 45 +++++++++ 18 files changed, 370 insertions(+), 138 deletions(-) create mode 100644 crates/db/migrations/0019_model_functions_fallback.sql diff --git a/crates/db/migrations/0019_model_functions_fallback.sql b/crates/db/migrations/0019_model_functions_fallback.sql new file mode 100644 index 0000000..0ef7209 --- /dev/null +++ b/crates/db/migrations/0019_model_functions_fallback.sql @@ -0,0 +1,18 @@ +-- One model for all analyses (natal, synastry, forecast) and a separate one +-- for answering questions; the short summary keeps its own. +ALTER TABLE model_settings DROP CONSTRAINT model_settings_kind_check; +INSERT INTO model_settings (kind, model, updated_by, updated_at) +SELECT 'analysis', model, updated_by, updated_at FROM model_settings +WHERE kind IN ('natal', 'synastry', 'forecast') ORDER BY updated_at DESC LIMIT 1; +DELETE FROM model_settings WHERE kind IN ('natal', 'synastry', 'forecast'); +ALTER TABLE model_settings ADD CONSTRAINT model_settings_kind_check + CHECK (kind IN ('summary', 'analysis', 'question')); + +-- When the chosen model fails, the default model takes over (`on_fallback`). +-- NULL: the chosen model is the default, there is nothing to fall back to. +ALTER TABLE analyses + ADD COLUMN fallback_model text, + ADD COLUMN on_fallback boolean NOT NULL DEFAULT false; +ALTER TABLE questions + ADD COLUMN fallback_model text, + ADD COLUMN on_fallback boolean NOT NULL DEFAULT false; diff --git a/docs/kerdesek.md b/docs/kerdesek.md index 0ce2c3b..0535459 100644 --- a/docs/kerdesek.md +++ b/docs/kerdesek.md @@ -4,7 +4,7 @@ A kész részletes képletelemzés és éves előrejelzés alatt a felhasználó ## Hogyan készül a válasz -- **Modell:** ugyanaz, amelyik az elemzést írta (`analyses.model`), nem a Modellek oldalon beállított. +- **Modell:** a Modellek oldalon külön beállítható kérdés-válasz modell (alapból Opus 5.5). Ha a választott modell hibára fut, az alapmodell válaszol (tartalék). - **Amit a modell kap:** a képlet tömör adatsorát (előrejelzésnél az év adatait is), magát az elemzést, az ugyanerre az elemzésre feltett korábbi legfeljebb 6 kérdést a válasszal, és az új kérdést. Születési dátum, hely, idő és név nem megy ki. - **Prompt:** az admin felületen szerkeszthető (Promptok → Kérdés-válasz). A szóhatár (`QUESTION_MAX_WORDS` = 250) a felhasználói üzenetben is benne van, így a prompt átírásától sem vész el. A kérdés legfeljebb 600 karakter. - **Szabályok:** csak a képletből és az elemzésből válaszol, nem jósol betegséget, halált, balesetet, terhességet vagy biztos eseményt, nem ad orvosi, jogi, pénzügyi tanácsot; a nem asztrológiai kérdést udvariasan elhárítja; a kérdésben lévő utasításokat nem követi. diff --git a/services/backend/README.md b/services/backend/README.md index 8d99e2a..0205dd6 100644 --- a/services/backend/README.md +++ b/services/backend/README.md @@ -20,8 +20,8 @@ Felhasználók, belépés, mentett születési képletek, szinasztriák és megh | `GOOGLE_CLIENT_IDS` | – | Google OAuth kliens-azonosító(k); üresen a Google-belépés ki van kapcsolva | | `BOOTSTRAP_ADMIN_EMAILS` | – | ezek a címek regisztrációkor vagy belépéskor admin szerepet kapnak (így lesz meg az első admin) | | `SUPPORTED_LOCALES` | `hu,en` | felület- és elemzésnyelvek; az első az alapértelmezett | -| `ANALYSIS_MODEL` | `anthropic/claude-opus-5.5` | alapból ez írja a fizetős részletes elemzéseket (az admin felületen funkciónként felülírható) | -| `SUMMARY_MODEL` | `anthropic/claude-sonnet-5.5` | alapból ez írja minden mentett képlet ingyenes rövid összefoglalóját (kb. 1 cent darabonként) | +| `ANALYSIS_MODEL` | `anthropic/claude-opus-5.5` | az elemzések, előrejelzések és válaszok alapmodellje, és a tartalék, ha a választott modell hibára fut | +| `SUMMARY_MODEL` | `anthropic/claude-opus-5.5` | a rövid összefoglaló alapmodellje (az admin felületen olcsóbbra állítható) | | `OPENROUTER_MODELS_URL` | `https://openrouter.ai/api/v1/models` | az admin modellválasztó listája árakkal (óránként egyszer kérdezi le) | | `ANALYSIS_PROMPT_VERSION` | `v3` | a prompt verziója (az elemzés-gyorsítótár kulcsának része) | | `MAX_CHARTS_PER_USER` | `1000` | biztonsági felső korlát a képletek számára | @@ -125,7 +125,8 @@ Hibák formátuma: `{"error": {"code", "message"}}`. A születési adatok hibái - **Ingyenes rész:** minden mentett képlethez elkészül egy rövid összefoglaló (`summary`, `SUMMARY_MODEL`), ezért nem kell fizetni. - **Fizetős rész:** a részletes elemzés darabra megy. A képletelemzés egy `natal`, a szinasztria-elemzés egy `synastry` egységet von le (`credit_ledger.unit`). Az egységek csomagvásárlásból (Stripe, később) vagy admin jóváírásból jönnek. Havi ingyenes keret nincs. -- **Modell:** funkciónként (összefoglaló, képletelemzés, kapcsolati elemzés, előrejelzés) az admin felületen választható az OpenRouter listájából (`model_settings` tábla); ha nincs beállítva, az `ANALYSIS_MODEL` / `SUMMARY_MODEL` (alapból Opus 5.5, a 2026-11-es összevetés legjobbja, lásd `services/worker/README.md`). Csak kötött JSON-választ adó modell választható. A kliens nem választ modellt, és a modell neve felhasználónak sehol nem jelenik meg. +- **Modell:** három beállítás az admin felületen, az OpenRouter listájából (`model_settings` tábla): az elemzések és az éves előrejelzés (képlet-, kapcsolati elemzés, előrejelzés) közös modellje, a kérdés-válasz modellje és a rövid összefoglalóé. Ha nincs beállítva, az `ANALYSIS_MODEL` / `SUMMARY_MODEL` (alapból mindkettő Opus 5.5, a 2026-11-es összevetés legjobbja, lásd `services/worker/README.md`). Csak kötött JSON-választ adó modell választható. A kliens nem választ modellt, és a modell neve felhasználónak sehol nem jelenik meg. +- **Tartalék:** ha a választott modell kétszer hibára fut (végleges hibánál egyszer), a worker az alapmodellel próbálja tovább (`fallback_model`, `on_fallback`); csak ha az is elbukik, jár vissza az egység. A Modellek oldal mutatja, hány szövegnél kellett az elmúlt 30 napban. - **Újragenerálás:** admin minden elemzést egyenként újraírathat (`POST /v1/admin/analyses/{id}/regenerate`): ugyanaz a sor (az azonosító és a megosztások maradnak), a jelenlegi prompttal és modellel, egységlevonás nélkül. - **Újraszámolás:** számítási változás után a `backend recalculate` parancs minden mentett képletet, összevetést és előrejelzést újraszámol, majd minden elemzést újra sorba állít (egységlevonás nélkül). - **Kérés:** a `POST .../analyses` (`{language?}`) létrehoz egy `pending` elemzést, és levon egy egységet; a szöveget a worker írja meg, általában 1–2 perc alatt. A kliens a `GET /v1/analyses/{id}` végponton kérdezi le az állapotát. Ha nincs egység, 402 `insufficient_credits` a válasz (`unit`, `balance` részletekkel). Hiba esetén a worker visszaadja az egységet. diff --git a/services/backend/src/config.rs b/services/backend/src/config.rs index f3031f7..308e89b 100644 --- a/services/backend/src/config.rs +++ b/services/backend/src/config.rs @@ -55,11 +55,13 @@ pub struct Config { pub bootstrap_admin_emails: Vec, /// `SUPPORTED_LOCALES` (default `hu,en`); the first is the default. pub supported_locales: Vec, - /// `ANALYSIS_MODEL`: writes the paid detailed analyses (default + /// `ANALYSIS_MODEL`: the default model of the analyses, forecasts and + /// answers, and the fallback when a chosen model fails (default /// `anthropic/claude-opus-5.5`, the best of the 2026-11 comparison). pub analysis_model: String, - /// `SUMMARY_MODEL`: writes the free short summary of every saved chart - /// (default `anthropic/claude-sonnet-5.5`, about 1 US cent each). + /// `SUMMARY_MODEL`: the default model of the free short summary of every + /// saved chart (default `anthropic/claude-opus-5.5`; the admin page can + /// choose a cheaper one). pub summary_model: String, /// `OPENROUTER_MODELS_URL` (default `https://openrouter.ai/api/v1/models`): /// the model list with prices for the admin page. @@ -123,7 +125,7 @@ impl Config { bootstrap_admin_emails: list("BOOTSTRAP_ADMIN_EMAILS").into_iter().map(|e| e.to_lowercase()).collect(), supported_locales, analysis_model: var("ANALYSIS_MODEL").unwrap_or_else(|| "anthropic/claude-opus-5.5".into()), - summary_model: var("SUMMARY_MODEL").unwrap_or_else(|| "anthropic/claude-sonnet-5.5".into()), + summary_model: var("SUMMARY_MODEL").unwrap_or_else(|| "anthropic/claude-opus-5.5".into()), openrouter_models_url: var("OPENROUTER_MODELS_URL") .unwrap_or_else(|| "https://openrouter.ai/api/v1/models".into()), prompt_version: var("ANALYSIS_PROMPT_VERSION").unwrap_or_else(|| "v3".into()), diff --git a/services/backend/src/models.rs b/services/backend/src/models.rs index 6e7f058..01b5f23 100644 --- a/services/backend/src/models.rs +++ b/services/backend/src/models.rs @@ -2,8 +2,9 @@ //! offers (with prices) for the admin page. //! //! The choice is in `model_settings`; without a row the configured default -//! is used (`SUMMARY_MODEL` for the short summary, `ANALYSIS_MODEL` else). -//! The model is never shown to users (a business secret). +//! is used (`SUMMARY_MODEL` for the short summary, `ANALYSIS_MODEL` else, +//! both Opus 5.5 by default). When the chosen model fails, the worker falls +//! back to the default. The model is never shown to users (a business secret). use std::sync::Arc; use std::time::{Duration, Instant}; @@ -18,21 +19,66 @@ use crate::AppState; /// The OpenRouter list is fetched at most this often. const CATALOG_TTL: Duration = Duration::from_secs(3600); -/// The configured default model of a kind. -pub fn default_model(state: &AppState, kind: Kind) -> &str { - match kind { - Kind::Summary => &state.config.summary_model, - _ => &state.config.analysis_model, +/// The model settings of the admin page: one model writes all analyses and +/// forecasts, another the answers to questions, a third the short summary. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Function { + Summary, + Analysis, + Question, +} + +impl Function { + pub const ALL: [Function; 3] = [Function::Summary, Function::Analysis, Function::Question]; + + pub fn key(self) -> &'static str { + match self { + Function::Summary => "summary", + Function::Analysis => "analysis", + Function::Question => "question", + } + } + + pub fn parse(key: &str) -> Option { + Function::ALL.into_iter().find(|f| f.key() == key) + } + + pub fn of(kind: Kind) -> Function { + match kind { + Kind::Summary => Function::Summary, + Kind::Question => Function::Question, + Kind::Natal | Kind::Synastry | Kind::Forecast => Function::Analysis, + } } } -/// The model new analyses of this kind are written with. -pub async fn model_for(state: &AppState, kind: Kind) -> Result { +/// The configured default model of a function; also the fallback when the +/// chosen model fails. +pub fn default_model(state: &AppState, function: Function) -> &str { + match function { + Function::Summary => &state.config.summary_model, + Function::Analysis | Function::Question => &state.config.analysis_model, + } +} + +/// The model new texts of this kind are written with, and the fallback model +/// (`None` when the chosen model is the default). +pub async fn models_for(state: &AppState, kind: Kind) -> Result<(String, Option), sqlx::Error> { + let function = Function::of(kind); let chosen: Option = sqlx::query_scalar("SELECT model FROM model_settings WHERE kind = $1") - .bind(kind.as_str()) + .bind(function.key()) .fetch_optional(&state.db) .await?; - Ok(chosen.unwrap_or_else(|| default_model(state, kind).to_owned())) + let default = default_model(state, function); + Ok(match chosen { + Some(model) if model != default => (model, Some(default.to_owned())), + _ => (default.to_owned(), None), + }) +} + +/// The model new texts of this kind are written with. +pub async fn model_for(state: &AppState, kind: Kind) -> Result { + Ok(models_for(state, kind).await?.0) } /// One model OpenRouter offers, with its price in USD per million tokens. diff --git a/services/backend/src/routes/analyses.rs b/services/backend/src/routes/analyses.rs index 2a3b254..7eb205f 100644 --- a/services/backend/src/routes/analyses.rs +++ b/services/backend/src/routes/analyses.rs @@ -175,7 +175,7 @@ pub(super) async fn queue( Target::Synastry { synastry_id, hash } => ("synastry", hash, None, Some(*synastry_id), None), Target::Forecast { forecast_id, hash } => ("forecast", hash, None, None, Some(*forecast_id)), }; - let model = &crate::models::model_for(state, Kind::parse(kind).expect("known kind")).await?; + let (model, fallback) = &crate::models::models_for(state, Kind::parse(kind).expect("known kind")).await?; let (cache_key, prompt_version) = &versions(state, kind, hash, model, language).await?; // Already requested for this chart/synastry: return it. @@ -197,8 +197,8 @@ pub(super) async fn queue( let insert = format!( "INSERT INTO analyses (id, kind, chart_id, synastry_id, requested_by, language, model, prompt_version, \ cache_key, status, content, input_tokens, output_tokens, credits_charged, completed_at, \ - forecast_id) \ - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16) RETURNING {ANALYSIS_COLUMNS}" + forecast_id, fallback_model) \ + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17) RETURNING {ANALYSIS_COLUMNS}" ); let id = Uuid::now_v7(); @@ -229,6 +229,7 @@ pub(super) async fn queue( .bind(0) .bind(OffsetDateTime::now_utc()) .bind(forecast_id) + .bind(fallback) .fetch_one(&state.db) .await?; return Ok((StatusCode::CREATED, row)); @@ -253,6 +254,7 @@ pub(super) async fn queue( .bind(i32::from(unit.is_some())) .bind(None::) .bind(forecast_id) + .bind(fallback) .fetch_one(&mut *tx) .await?; if let Some(unit) = unit { @@ -391,19 +393,21 @@ pub async fn regenerate( } }; let hash = hash.ok_or(ApiError::NotFound("analysis"))?; - let model = crate::models::model_for(&state, kind).await?; + let (model, fallback) = crate::models::models_for(&state, kind).await?; let (cache_key, prompt_version) = versions(&state, kind.as_str(), &hash, &model, &row.language).await?; let mut tx = state.db.begin().await?; let row: AnalysisRow = sqlx::query_as(&format!( "UPDATE analyses SET status = 'pending', content = NULL, error = NULL, model_used = NULL, \ input_tokens = NULL, output_tokens = NULL, cost_usd = NULL, attempts = 0, next_attempt_at = now(), \ - started_at = NULL, completed_at = NULL, model = $2, prompt_version = $3, cache_key = $4 \ + started_at = NULL, completed_at = NULL, model = $2, prompt_version = $3, cache_key = $4, \ + fallback_model = $5, on_fallback = false \ WHERE id = $1 AND status IN ('done', 'failed') RETURNING {ANALYSIS_COLUMNS}" )) .bind(id) .bind(&model) .bind(&prompt_version) .bind(&cache_key) + .bind(&fallback) .fetch_optional(&mut *tx) .await? .ok_or_else(|| ApiError::conflict("analysis_in_progress", "Ez az elemzés éppen készül."))?; diff --git a/services/backend/src/routes/models.rs b/services/backend/src/routes/models.rs index 923902b..dd45668 100644 --- a/services/backend/src/routes/models.rs +++ b/services/backend/src/routes/models.rs @@ -1,7 +1,7 @@ -//! Admin: which model writes which kind of analysis, chosen from the models -//! OpenRouter offers. A change applies to analyses requested after it. +//! Admin: which model writes the analyses and forecasts, the answers to +//! questions and the short summary, chosen from the models OpenRouter offers. +//! A change applies to texts requested after it. -use analysis_prompt::Kind; use axum::Json; use axum::extract::{Path, State}; use axum_extra::extract::WithRejection; @@ -13,7 +13,7 @@ use super::{Body, St}; use crate::AppState; use crate::auth::AdminUser; use crate::error::{ApiError, ApiResult}; -use crate::models::default_model; +use crate::models::{Function, default_model}; use crate::views::rfc3339; fn catalog_error(e: String) -> ApiError { @@ -23,27 +23,38 @@ fn catalog_error(e: String) -> ApiError { } } -async fn entry(state: &AppState, kind: Kind) -> Result { +async fn entry(state: &AppState, function: Function) -> Result { let chosen: Option<(String, OffsetDateTime, Option)> = sqlx::query_as( "SELECT m.model, m.updated_at, u.email FROM model_settings m LEFT JOIN users u ON u.id = m.updated_by \ WHERE m.kind = $1", ) - .bind(kind.as_str()) + .bind(function.key()) .fetch_optional(&state.db) .await?; - // Typical size of this kind of analysis, from the latest ones: for the cost estimate. - let (input, output, samples): (Option, Option, i64) = sqlx::query_as( + // Typical size of the latest texts, for the cost estimate, and how often + // the default model had to take over in the last 30 days. + let (table, kinds) = match function { + Function::Summary => ("analyses", "kind = 'summary'"), + Function::Analysis => ("analyses", "kind IN ('natal', 'synastry', 'forecast')"), + Function::Question => ("questions", "true"), + }; + let (input, output, samples): (Option, Option, i64) = sqlx::query_as(&format!( "SELECT avg(input_tokens)::float8, avg(output_tokens)::float8, count(*) FROM ( \ - SELECT input_tokens, output_tokens FROM analyses \ - WHERE kind = $1 AND status = 'done' AND input_tokens IS NOT NULL AND output_tokens IS NOT NULL \ - ORDER BY completed_at DESC LIMIT 50) recent", - ) - .bind(kind.as_str()) + SELECT input_tokens, output_tokens FROM {table} \ + WHERE {kinds} AND status = 'done' AND input_tokens IS NOT NULL AND output_tokens IS NOT NULL \ + ORDER BY completed_at DESC LIMIT 50) recent" + )) .fetch_one(&state.db) .await?; - let default = default_model(state, kind); + let (fallbacks, finished): (i64, i64) = sqlx::query_as(&format!( + "SELECT count(*) FILTER (WHERE on_fallback), count(*) FROM {table} \ + WHERE {kinds} AND status IN ('done', 'failed') AND completed_at > now() - interval '30 days'" + )) + .fetch_one(&state.db) + .await?; + let default = default_model(state, function); Ok(json!({ - "kind": kind.as_str(), + "kind": function.key(), "model": chosen.as_ref().map_or(default, |c| c.0.as_str()), "default": default, "custom": chosen.is_some(), @@ -52,13 +63,15 @@ async fn entry(state: &AppState, kind: Kind) -> Result { "avg_input_tokens": input, "avg_output_tokens": output, "samples": samples, + "fallbacks_30d": fallbacks, + "finished_30d": finished, })) } pub async fn list(State(state): St, _admin: AdminUser) -> ApiResult> { let mut functions = Vec::new(); - for kind in Kind::ANALYSES { - functions.push(entry(&state, kind).await?); + for function in Function::ALL { + functions.push(entry(&state, function).await?); } Ok(Json(json!({ "functions": functions }))) } @@ -81,9 +94,9 @@ pub async fn save( WithRejection(Path(kind), _): WithRejection, ApiError>, WithRejection(Json(req), _): Body, ) -> ApiResult> { - let kind = Kind::parse(&kind).ok_or(ApiError::NotFound("function"))?; + let function = Function::parse(&kind).ok_or(ApiError::NotFound("function"))?; match req.model.as_deref().map(str::trim) { - Some(model) if model != default_model(&state, kind) => { + Some(model) if model != default_model(&state, function) => { let models = state.models.models().await.map_err(catalog_error)?; if !models.iter().any(|m| m.id == model) { return Err(ApiError::BadRequest(format!( @@ -95,16 +108,16 @@ pub async fn save( ON CONFLICT (kind) DO UPDATE SET model = EXCLUDED.model, updated_by = EXCLUDED.updated_by, \ updated_at = now()", ) - .bind(kind.as_str()) + .bind(function.key()) .bind(model) .bind(admin.id()) .execute(&state.db) .await?; } _ => { - sqlx::query("DELETE FROM model_settings WHERE kind = $1").bind(kind.as_str()).execute(&state.db).await?; + sqlx::query("DELETE FROM model_settings WHERE kind = $1").bind(function.key()).execute(&state.db).await?; } } - tracing::info!(admin = %admin.id(), kind = kind.as_str(), "analysis model changed"); - Ok(Json(entry(&state, kind).await?)) + tracing::info!(admin = %admin.id(), function = function.key(), "model changed"); + Ok(Json(entry(&state, function).await?)) } diff --git a/services/backend/src/routes/questions.rs b/services/backend/src/routes/questions.rs index 2dc5177..afa4147 100644 --- a/services/backend/src/routes/questions.rs +++ b/services/backend/src/routes/questions.rs @@ -1,7 +1,7 @@ //! Questions about a finished detailed analysis: a natal chart analysis or a //! yearly forecast. Each question uses a `question` unit (admins ask for -//! free). The worker answers with the model that wrote the analysis, seeing -//! the chart data, the analysis and the earlier questions about it. +//! free). The worker answers with the question model of the admin page, +//! seeing the chart data, the analysis and the earlier questions about it. use analysis_prompt::{Kind, QUESTION_MAX_CHARS}; use axum::Json; @@ -64,21 +64,19 @@ async fn find_question(db: &PgPool, id: Uuid) -> Result, sql sqlx::query_as(&format!("SELECT {QUESTION_COLUMNS} FROM questions WHERE id = $1")).bind(id).fetch_optional(db).await } -/// The model of the analysis (questions are answered by the same model) and -/// the prompt version of the question prompt in force. +/// The question model (and its fallback) and the version of the question +/// prompt in force. async fn model_and_version( state: &crate::AppState, - analysis_id: Uuid, language: &str, -) -> Result<(String, String), sqlx::Error> { - let model: String = - sqlx::query_scalar("SELECT model FROM analyses WHERE id = $1").bind(analysis_id).fetch_one(&state.db).await?; +) -> Result<(String, Option, String), sqlx::Error> { + let (model, fallback) = crate::models::models_for(state, Kind::Question).await?; let base = &state.config.prompt_version; let version = match super::prompts::current_revision(&state.db, Kind::Question.as_str(), language).await? { Some(id) => format!("{base}-r{id}"), None => base.clone(), }; - Ok((model, version)) + Ok((model, fallback, version)) } pub async fn list(State(state): St, auth: AuthUser, WithRejection(Path(id), _): IdPath) -> ApiResult> { @@ -141,12 +139,13 @@ pub async fn ask( } let admin = auth.user.has_role(Role::Admin); - let (model, prompt_version) = model_and_version(&state, id, &analysis.language).await?; + let (model, fallback, prompt_version) = model_and_version(&state, &analysis.language).await?; let question_id = Uuid::now_v7(); let mut tx = state.db.begin().await?; let row: QuestionRow = sqlx::query_as(&format!( - "INSERT INTO questions (id, analysis_id, asked_by, question, language, model, prompt_version, credits_charged) \ - VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING {QUESTION_COLUMNS}" + "INSERT INTO questions (id, analysis_id, asked_by, question, language, model, prompt_version, credits_charged, \ + fallback_model) \ + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) RETURNING {QUESTION_COLUMNS}" )) .bind(question_id) .bind(id) @@ -156,6 +155,7 @@ pub async fn ask( .bind(&model) .bind(&prompt_version) .bind(i32::from(!admin)) + .bind(&fallback) .fetch_one(&mut *tx) .await?; if !admin { @@ -175,26 +175,29 @@ pub async fn get_one(State(state): St, auth: AuthUser, WithRejection(Path(id), _ } /// Admin: answers a question again in place, with the current question prompt -/// and the analysis's model. Nobody is charged. +/// and model. Nobody is charged. pub async fn regenerate( State(state): St, _admin: AdminUser, WithRejection(Path(id), _): IdPath, ) -> ApiResult> { - let row = find_question(&state.db, id).await?.ok_or(ApiError::NotFound("question"))?; - let language: String = - sqlx::query_scalar("SELECT language FROM questions WHERE id = $1").bind(id).fetch_one(&state.db).await?; - let (model, prompt_version) = model_and_version(&state, row.analysis_id, &language).await?; + let language: String = sqlx::query_scalar("SELECT language FROM questions WHERE id = $1") + .bind(id) + .fetch_optional(&state.db) + .await? + .ok_or(ApiError::NotFound("question"))?; + let (model, fallback, prompt_version) = model_and_version(&state, &language).await?; let mut tx = state.db.begin().await?; let row: QuestionRow = sqlx::query_as(&format!( "UPDATE questions SET status = 'pending', answer = NULL, error = NULL, model_used = NULL, input_tokens = NULL, \ output_tokens = NULL, cost_usd = NULL, attempts = 0, next_attempt_at = now(), started_at = NULL, \ - completed_at = NULL, model = $2, prompt_version = $3 \ + completed_at = NULL, model = $2, prompt_version = $3, fallback_model = $4, on_fallback = false \ WHERE id = $1 AND status IN ('done', 'failed') RETURNING {QUESTION_COLUMNS}" )) .bind(id) .bind(&model) .bind(&prompt_version) + .bind(&fallback) .fetch_optional(&mut *tx) .await? .ok_or_else(|| ApiError::conflict("question_in_progress", "Erre a kérdésre éppen készül a válasz."))?; diff --git a/services/backend/src/tests.rs b/services/backend/src/tests.rs index df03f90..20df951 100644 --- a/services/backend/src/tests.rs +++ b/services/backend/src/tests.rs @@ -1499,9 +1499,8 @@ async fn admins_choose_the_model_of_each_function(pool: PgPool) { models, [ ("summary", "anthropic/claude-summary", false), - ("natal", "anthropic/claude-test", false), - ("synastry", "anthropic/claude-test", false), - ("forecast", "anthropic/claude-test", false), + ("analysis", "anthropic/claude-test", false), + ("question", "anthropic/claude-test", false), ] ); @@ -1516,9 +1515,9 @@ async fn admins_choose_the_model_of_each_function(pool: PgPool) { let (app, admin) = (&app, &admin); async move { app.call("PUT", &uri, Some(admin), Some(json!({ "model": model }))).await } }; - assert_eq!(put("natal", json!("vendor/plain")).await.status, StatusCode::BAD_REQUEST); - assert_eq!(put("composite", json!("openai/gpt-x")).await.status, StatusCode::NOT_FOUND); - let r = put("natal", json!("openai/gpt-x")).await; + assert_eq!(put("analysis", json!("vendor/plain")).await.status, StatusCode::BAD_REQUEST); + assert_eq!(put("natal", json!("openai/gpt-x")).await.status, StatusCode::NOT_FOUND); + let r = put("analysis", json!("openai/gpt-x")).await; assert_eq!((r.status, &r.body["model"], &r.body["custom"]), (StatusCode::OK, &json!("openai/gpt-x"), &json!(true))); // new analyses use it; regenerating writes it again in place with the current model @@ -1527,9 +1526,13 @@ async fn admins_choose_the_model_of_each_function(pool: PgPool) { let r = app.post(&format!("/v1/charts/{chart_id}/analyses"), Some(&admin), json!({})).await; assert_eq!(r.status, StatusCode::ACCEPTED, "{}", r.body); let id = r.body["id"].as_str().unwrap().to_string(); - let model = - || sqlx::query_scalar::<_, String>("SELECT model FROM analyses WHERE id = $1::uuid").bind(&id).fetch_one(&pool); - assert_eq!(model().await.unwrap(), "openai/gpt-x"); + let model = || { + sqlx::query_as::<_, (String, Option)>("SELECT model, fallback_model FROM analyses WHERE id = $1::uuid") + .bind(&id) + .fetch_one(&pool) + }; + // the default model is kept as the fallback + assert_eq!(model().await.unwrap(), ("openai/gpt-x".into(), Some("anthropic/claude-test".into()))); let regenerate_uri = format!("/v1/admin/analyses/{id}/regenerate"); let regenerate = || app.post(®enerate_uri, Some(&admin), json!({})); assert_eq!(regenerate().await.status, StatusCode::CONFLICT); // still being written @@ -1537,7 +1540,7 @@ async fn admins_choose_the_model_of_each_function(pool: PgPool) { .execute(&pool) .await .unwrap(); - assert_eq!(put("natal", Value::Null).await.body["model"], "anthropic/claude-test"); + assert_eq!(put("analysis", Value::Null).await.body["model"], "anthropic/claude-test"); assert_eq!( app.post(&format!("/v1/admin/analyses/{id}/regenerate"), Some(&anna), json!({})).await.status, StatusCode::FORBIDDEN @@ -1548,7 +1551,7 @@ async fn admins_choose_the_model_of_each_function(pool: PgPool) { (StatusCode::OK, Some("pending"), &Value::Null) ); assert_eq!(r.body["id"].as_str(), Some(id.as_str())); - assert_eq!(model().await.unwrap(), "anthropic/claude-test"); + assert_eq!(model().await.unwrap(), ("anthropic/claude-test".into(), None)); let r = app.get(&format!("/v1/charts/{chart_id}/analyses"), Some(&admin)).await; assert_eq!(r.body["analyses"].as_array().unwrap().iter().filter(|a| a["kind"] == "natal").count(), 1); } @@ -1587,15 +1590,14 @@ async fn questions_about_an_analysis_use_question_units(pool: PgPool) { assert!(r.body.get("model").is_none(), "the model is a secret"); assert_eq!(left().await, json!(2)); let question = r.body["id"].as_str().unwrap().to_string(); - // the same model as the analysis answers - let (q_model, a_model): (String, String) = sqlx::query_as( - "SELECT q.model, a.model FROM questions q JOIN analyses a ON a.id = q.analysis_id WHERE q.id = $1::uuid", - ) - .bind(&question) - .fetch_one(&pool) - .await - .unwrap(); - assert_eq!(q_model, a_model); + // the question model of the admin page answers (here the default) + let (q_model, fallback): (String, Option) = + sqlx::query_as("SELECT model, fallback_model FROM questions WHERE id = $1::uuid") + .bind(&question) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!((q_model.as_str(), fallback), ("anthropic/claude-test", None)); // one at a time let r = ask(&anna, "És a munkában?").await; assert_eq!((r.status, r.body["error"]["code"].as_str()), (StatusCode::CONFLICT, Some("question_in_progress"))); diff --git a/services/frontend/src/app/admin/modellek/AdminModels.tsx b/services/frontend/src/app/admin/modellek/AdminModels.tsx index d2491a2..84c5748 100644 --- a/services/frontend/src/app/admin/modellek/AdminModels.tsx +++ b/services/frontend/src/app/admin/modellek/AdminModels.tsx @@ -4,13 +4,15 @@ import { useEffect, useMemo, useState } from "react"; import { api, errorText } from "@/lib/api"; import { formatDate } from "@/lib/astro"; -import type { CatalogModel, ModelFunction, PromptKind } from "@/lib/types"; +import type { CatalogModel, ModelFunction } from "@/lib/types"; -const LABELS: Record, { title: string; about: string }> = { +const LABELS: Record = { summary: { title: "Tömör összefoglaló", about: "Ingyenes, minden új képlethez magától elkészül. Olcsó, gyors modell elég." }, - natal: { title: "Részletes képletelemzés", about: "Fizetős, hat fejezet." }, - synastry: { title: "Kapcsolati elemzés", about: "Fizetős, öt fejezet két képletről." }, - forecast: { title: "Éves előrejelzés", about: "Fizetős, öt fejezet, a leghosszabb adatsorral." }, + analysis: { + title: "Elemzések és éves előrejelzés", + about: "Fizetős szövegek: részletes képletelemzés, kapcsolati elemzés és éves előrejelzés.", + }, + question: { title: "Kérdés-válasz", about: "A kész elemzésekre és előrejelzésekre feltett kérdések válasza (legfeljebb 250 szó)." }, }; /** How many models the picker lists at once (narrow it with the search). */ @@ -58,7 +60,8 @@ export function AdminModels() {

)}

- Az elemzésekre feltett kérdésekre mindig ugyanaz a modell válaszol, amelyik az adott elemzést írta. + Tartalék: ha a választott modell kétszer hibára fut (vagy egyszer végleges hibával), az alapértelmezett modell írja + meg a szöveget. Csak ha az is elbukik, kapja vissza a felhasználó az egységét.

{functions.map((f) => ( @@ -82,7 +85,7 @@ function FunctionModel({ const [error, setError] = useState(null); const current = catalog?.find((m) => m.id === fn.model); const cost = estimate(fn, current); - const label = LABELS[fn.kind as keyof typeof LABELS]; + const label = LABELS[fn.kind]; const searchId = `model-search-${fn.kind}`; const matches = useMemo(() => { @@ -134,6 +137,11 @@ function FunctionModel({ ? `Módosítva: ${fn.updated_at ? formatDate(fn.updated_at, true) : ""}${fn.updated_by ? `, ${fn.updated_by}` : ""}` : "Alapértelmezett"} + {fn.custom && ( + 0 ? "hint warn" : "hint"}> + Tartalék: {fn.default} · az elmúlt 30 napban {fn.fallbacks_30d} / {fn.finished_30d} szövegnél kellett + + )}
diff --git a/services/frontend/src/app/admin/promptok/AdminPrompts.tsx b/services/frontend/src/app/admin/promptok/AdminPrompts.tsx index 84b6301..96785bc 100644 --- a/services/frontend/src/app/admin/promptok/AdminPrompts.tsx +++ b/services/frontend/src/app/admin/promptok/AdminPrompts.tsx @@ -40,7 +40,7 @@ const KINDS: { kind: PromptKind; id: string; label: string; about: string }[] = id: "kerdesek", label: "Kérdés-válasz", about: - "A kész képletelemzésre vagy előrejelzésre feltett kérdések válasza (legfeljebb 250 szó). A modell kapja a képlet adatait, az elemzést és a korábbi kérdéseket; ugyanaz a modell válaszol, amelyik az elemzést írta.", + "A kész képletelemzésre vagy előrejelzésre feltett kérdések válasza (legfeljebb 250 szó). A modell kapja a képlet adatait, az elemzést és a korábbi kérdéseket; a Modellek oldalon beállított kérdés-válasz modell válaszol.", }, ]; diff --git a/services/frontend/src/app/csomagok/page.tsx b/services/frontend/src/app/csomagok/page.tsx index 9cb5307..4269fe9 100644 --- a/services/frontend/src/app/csomagok/page.tsx +++ b/services/frontend/src/app/csomagok/page.tsx @@ -22,7 +22,7 @@ const FAQ: [string, string][] = [ ], [ "Mit jelent a kérdés?", - "A kész részletes képletelemzésed vagy éves előrejelzésed alatt kérdezhetsz róla, például hogy mit jelent egy fényszög a munkádra nézve, vagy mire figyelj egy időszakban. A választ ugyanaz a modell írja, amelyik az elemzést, a képleted és az elemzés alapján, legfeljebb 250 szóban. Minden csomagban 3 kérdés van, és külön is vehetsz még.", + "A kész részletes képletelemzésed vagy éves előrejelzésed alatt kérdezhetsz róla, például hogy mit jelent egy fényszög a munkádra nézve, vagy mire figyelj egy időszakban. A válasz a képleted és az elemzés alapján készül, legfeljebb 250 szóban. Minden csomagban 3 kérdés van, és külön is vehetsz még.", ], [ "Mire kell a személyhely?", diff --git a/services/frontend/src/app/globals.css b/services/frontend/src/app/globals.css index 50ea2db..4d94d75 100644 --- a/services/frontend/src/app/globals.css +++ b/services/frontend/src/app/globals.css @@ -643,3 +643,4 @@ a.pair-row:hover strong { color: var(--gold); } .qa-answer p { margin: 0 0 8px; } .qa-form { display: flex; flex-direction: column; gap: 8px; } .qa-form textarea { resize: vertical; min-height: 84px; } +.hint.warn { color: var(--gold); } diff --git a/services/frontend/src/lib/types.ts b/services/frontend/src/lib/types.ts index 15a4954..3fddfa3 100644 --- a/services/frontend/src/lib/types.ts +++ b/services/frontend/src/lib/types.ts @@ -248,9 +248,9 @@ export interface AdminUser { export type PromptKind = "summary" | "natal" | "synastry" | "forecast" | "question"; -/** Admin: the model writing one kind of analysis. */ +/** Admin: the model of one function (all analyses and forecasts, answers, the short summary). */ export interface ModelFunction { - kind: PromptKind; + kind: "summary" | "analysis" | "question"; model: string; /** The configured default (environment). */ default: string; @@ -261,6 +261,9 @@ export interface ModelFunction { avg_input_tokens: number | null; avg_output_tokens: number | null; samples: number; + /** In the last 30 days: texts the default model had to write, of all finished. */ + fallbacks_30d: number; + finished_30d: number; } /** A model OpenRouter offers; prices in USD per million tokens. */ diff --git a/services/worker/README.md b/services/worker/README.md index 082ab99..e49cc40 100644 --- a/services/worker/README.md +++ b/services/worker/README.md @@ -45,7 +45,7 @@ A 2026-11-15-i összevetésen két képletet hét modell írt meg; az egyiket a A MiniMax M3 olcsó, de az asztrológus szerint többször felcserélt dolgokat. -Az ingyenes rövid összefoglalót (2026-11-16) a Haiku és a Sonnet négy képleten írta meg. Mindkettő tényszerűen helyes volt, de a Sonnet jobban válogat és fogalmaz, ezért ez az alapértelmezett (`SUMMARY_MODEL`, kb. 0,009 dollár, a Haiku kb. 0,001). A fizetős részletes elemzés az Opus 5.5-tel készül. +Az ingyenes rövid összefoglalót (2026-11-16) a Haiku és a Sonnet négy képleten írta meg. Mindkettő tényszerűen helyes volt, de a Sonnet jobban válogat és fogalmaz (kb. 0,009 dollár, a Haiku kb. 0,001). 2026-11-18 óta minden funkció alapmodellje az Opus 5.5 (`SUMMARY_MODEL`, `ANALYSIS_MODEL`); olcsóbb modell az admin Modellek oldalán választható, hiba esetén az Opus 5.5 a tartalék. ```bash OPENROUTER_API_KEY=... worker compare-models --chart natal.json \ diff --git a/services/worker/src/analysis/mod.rs b/services/worker/src/analysis/mod.rs index cd6ee3f..a22bded 100644 --- a/services/worker/src/analysis/mod.rs +++ b/services/worker/src/analysis/mod.rs @@ -8,6 +8,10 @@ //! //! Admin test runs of an edited prompt (`prompt_previews`) go through the //! same loop when no analysis is due: one attempt, no credits, no retries. +//! +//! Fallback: when the model chosen on the admin page fails twice (or once with +//! a permanent error), the default model (`fallback_model`) takes over with a +//! fresh set of attempts; only when that fails too is the unit refunded. pub mod compact; pub mod openrouter; @@ -40,6 +44,45 @@ pub struct Job { pub model: String, pub prompt_version: String, pub attempts: i32, + /// The default model, when the chosen one is another (see the module docs). + pub fallback_model: Option, + pub on_fallback: bool, +} + +impl Job { + /// The model of this attempt. + pub fn active_model(&self) -> &str { + active_model(&self.model, self.fallback_model.as_deref(), self.on_fallback) + } +} + +pub fn active_model<'a>(model: &'a str, fallback: Option<&'a str>, on_fallback: bool) -> &'a str { + match fallback { + Some(f) if on_fallback => f, + _ => model, + } +} + +/// What happens after a failed attempt. +#[derive(Debug, PartialEq, Eq)] +pub enum Next { + Retry(Duration), + /// Switch to the fallback model, with a fresh set of attempts, at once. + Fallback, + /// Give up (and refund). + Fail, +} + +/// The chosen model gets two attempts (one after a permanent error), then the +/// fallback model takes over with its own `max_attempts`. +pub fn next_step(error: &LlmError, attempts: i32, max_attempts: i32, can_fall_back: bool) -> Next { + if can_fall_back && (matches!(error, LlmError::Permanent(_)) || attempts >= 2) { + return Next::Fallback; + } + match error { + LlmError::Transient(_) if attempts < max_attempts => Next::Retry(backoff(attempts)), + _ => Next::Fail, + } } /// A test run of an edited system prompt on a chart. @@ -81,7 +124,8 @@ impl Analyzer { "UPDATE analyses SET status = 'running', started_at = now(), attempts = attempts + 1 \ WHERE id = (SELECT id FROM analyses WHERE status = 'pending' AND next_attempt_at <= now() \ ORDER BY next_attempt_at LIMIT 1 FOR UPDATE SKIP LOCKED) \ - RETURNING id, kind, chart_id, synastry_id, forecast_id, language, model, prompt_version, attempts", + RETURNING id, kind, chart_id, synastry_id, forecast_id, language, model, prompt_version, attempts, \ + fallback_model, on_fallback", ) .fetch_optional(&self.db) .await?; @@ -230,7 +274,7 @@ impl Analyzer { pub async fn handle(&self, job: Job) { let started = Instant::now(); let result = match self.prompt_for(&job).await { - Ok(prompt) => self.llm.complete(&job.model, &prompt).await.map(|mut c| { + Ok(prompt) => self.llm.complete(job.active_model(), &prompt).await.map(|mut c| { prompt::polish(&job.language, &mut c.content); c }), @@ -263,22 +307,40 @@ impl Analyzer { cost_usd = c.cost_usd, secs = took.as_secs(), "analysis done" ); } - Err(LlmError::Transient(message)) if job.attempts < self.max_attempts => { - let wait = backoff(job.attempts); - sqlx::query( - "UPDATE analyses SET status = 'pending', error = $2, \ - next_attempt_at = now() + make_interval(secs => $3) WHERE id = $1 AND status = 'running'", - ) - .bind(job.id) - .bind(&message) - .bind(wait.as_secs_f64()) - .execute(&self.db) - .await?; - tracing::warn!(id = %job.id, attempts = job.attempts, retry_in_secs = wait.as_secs(), error = %message, "analysis will be retried"); - } Err(e) => { - self.fail(job.id, e.message()).await?; - tracing::error!(id = %job.id, attempts = job.attempts, error = %e.message(), "analysis failed"); + let can_fall_back = !job.on_fallback && job.fallback_model.is_some(); + match next_step(&e, job.attempts, self.max_attempts, can_fall_back) { + Next::Retry(wait) => { + sqlx::query( + "UPDATE analyses SET status = 'pending', error = $2, \ + next_attempt_at = now() + make_interval(secs => $3) WHERE id = $1 AND status = 'running'", + ) + .bind(job.id) + .bind(e.message()) + .bind(wait.as_secs_f64()) + .execute(&self.db) + .await?; + tracing::warn!(id = %job.id, attempts = job.attempts, retry_in_secs = wait.as_secs(), error = %e.message(), "analysis will be retried"); + } + Next::Fallback => { + sqlx::query( + "UPDATE analyses SET status = 'pending', error = $2, on_fallback = true, attempts = 0, \ + next_attempt_at = now() WHERE id = $1 AND status = 'running'", + ) + .bind(job.id) + .bind(e.message()) + .execute(&self.db) + .await?; + tracing::warn!( + id = %job.id, model = %job.model, fallback = ?job.fallback_model, error = %e.message(), + "the chosen model failed, the default model takes over" + ); + } + Next::Fail => { + self.fail(job.id, e.message()).await?; + tracing::error!(id = %job.id, attempts = job.attempts, error = %e.message(), "analysis failed"); + } + } } } Ok(()) diff --git a/services/worker/src/analysis/questions.rs b/services/worker/src/analysis/questions.rs index ab88254..ea6623c 100644 --- a/services/worker/src/analysis/questions.rs +++ b/services/worker/src/analysis/questions.rs @@ -1,7 +1,7 @@ //! Answers to questions about a finished analysis (`questions` rows). The -//! model of the analysis answers, seeing the chart data the analysis was -//! written from, the analysis itself and the earlier questions about it. -//! Retries and refunds work as for analyses. +//! question model answers, seeing the chart data the analysis was written +//! from, the analysis itself and the earlier questions about it. Retries, +//! fallback and refunds work as for analyses. use std::time::{Duration, Instant}; @@ -10,7 +10,7 @@ use uuid::Uuid; use super::openrouter::{Completion, LlmError}; use super::prompt::{self, Kind}; -use super::{Analyzer, backoff}; +use super::{Analyzer, Next, active_model, next_step}; /// The earlier questions and answers given to the model, at most this many. const HISTORY: i64 = 6; @@ -25,6 +25,8 @@ pub struct QuestionJob { pub prompt_version: String, pub attempts: i32, pub created_at: time::OffsetDateTime, + pub fallback_model: Option, + pub on_fallback: bool, } impl Analyzer { @@ -34,7 +36,8 @@ impl Analyzer { "UPDATE questions SET status = 'running', started_at = now(), attempts = attempts + 1 \ WHERE id = (SELECT id FROM questions WHERE status = 'pending' AND next_attempt_at <= now() \ ORDER BY next_attempt_at LIMIT 1 FOR UPDATE SKIP LOCKED) \ - RETURNING id, analysis_id, question, language, model, prompt_version, attempts, created_at", + RETURNING id, analysis_id, question, language, model, prompt_version, attempts, created_at, \ + fallback_model, on_fallback", ) .fetch_optional(&self.db) .await?; @@ -89,10 +92,13 @@ impl Analyzer { pub async fn handle_question(&self, job: QuestionJob) { let started = Instant::now(); let result = match self.question_prompt(&job).await { - Ok(p) => self.llm.complete(&job.model, &p).await.map(|mut c| { - prompt::polish(&job.language, &mut c.content); - c - }), + Ok(p) => { + let model = active_model(&job.model, job.fallback_model.as_deref(), job.on_fallback); + self.llm.complete(model, &p).await.map(|mut c| { + prompt::polish(&job.language, &mut c.content); + c + }) + } Err(e) => Err(e), }; if let Err(e) = self.finish_question(&job, result, started.elapsed()).await { @@ -128,22 +134,40 @@ impl Analyzer { secs = took.as_secs(), "answer done" ); } - Err(LlmError::Transient(message)) if job.attempts < self.max_attempts => { - let wait = backoff(job.attempts); - sqlx::query( - "UPDATE questions SET status = 'pending', error = $2, \ - next_attempt_at = now() + make_interval(secs => $3) WHERE id = $1 AND status = 'running'", - ) - .bind(job.id) - .bind(&message) - .bind(wait.as_secs_f64()) - .execute(&self.db) - .await?; - tracing::warn!(id = %job.id, attempts = job.attempts, error = %message, "answer will be retried"); - } Err(e) => { - self.fail_question(job.id, e.message()).await?; - tracing::error!(id = %job.id, attempts = job.attempts, error = %e.message(), "answer failed"); + let can_fall_back = !job.on_fallback && job.fallback_model.is_some(); + match next_step(&e, job.attempts, self.max_attempts, can_fall_back) { + Next::Retry(wait) => { + sqlx::query( + "UPDATE questions SET status = 'pending', error = $2, \ + next_attempt_at = now() + make_interval(secs => $3) WHERE id = $1 AND status = 'running'", + ) + .bind(job.id) + .bind(e.message()) + .bind(wait.as_secs_f64()) + .execute(&self.db) + .await?; + tracing::warn!(id = %job.id, attempts = job.attempts, error = %e.message(), "answer will be retried"); + } + Next::Fallback => { + sqlx::query( + "UPDATE questions SET status = 'pending', error = $2, on_fallback = true, attempts = 0, \ + next_attempt_at = now() WHERE id = $1 AND status = 'running'", + ) + .bind(job.id) + .bind(e.message()) + .execute(&self.db) + .await?; + tracing::warn!( + id = %job.id, model = %job.model, error = %e.message(), + "the chosen model failed, the default model answers" + ); + } + Next::Fail => { + self.fail_question(job.id, e.message()).await?; + tracing::error!(id = %job.id, attempts = job.attempts, error = %e.message(), "answer failed"); + } + } } } Ok(()) diff --git a/services/worker/src/tests.rs b/services/worker/src/tests.rs index 15eb2da..bf4c9c4 100644 --- a/services/worker/src/tests.rs +++ b/services/worker/src/tests.rs @@ -736,4 +736,49 @@ mod analyses { .unwrap(); assert_eq!(refunds, vec![("question".to_string(), 1)]); } + + #[test] + fn the_chosen_model_gets_two_attempts_then_the_default_takes_over() { + use crate::analysis::openrouter::LlmError; + use crate::analysis::{Next, next_step}; + let t = || LlmError::Transient("overloaded".into()); + let p = || LlmError::Permanent("no such model".into()); + assert_eq!(next_step(&t(), 1, 3, true), Next::Retry(backoff(1))); + assert_eq!(next_step(&t(), 2, 3, true), Next::Fallback); + assert_eq!(next_step(&p(), 1, 3, true), Next::Fallback); + // without a fallback (or already on it): retries, then failure + assert_eq!(next_step(&t(), 2, 3, false), Next::Retry(backoff(2))); + assert_eq!(next_step(&t(), 3, 3, false), Next::Fail); + assert_eq!(next_step(&p(), 1, 3, false), Next::Fail); + } + + #[sqlx::test(migrator = "db::MIGRATOR")] + async fn the_default_model_writes_when_the_chosen_one_fails(db: PgPool) { + let s = setup(&db).await; + let id = request_analysis(&db, &s).await; + sqlx::query("UPDATE analyses SET model = 'cheap/model', fallback_model = 'test/model' WHERE id = $1") + .bind(id) + .execute(&db) + .await + .unwrap(); + let broken = json!({"title": "Csak cím"}); + let (fake, url) = fake_openrouter(vec![reply(&broken), reply(&broken), reply(&content(Kind::Natal))]).await; + let a = analyzer(&db, &url); + + assert!(run_once(&a).await); + make_due(&db, id).await; + assert!(run_once(&a).await); // second failure: the default model takes over at once + assert!(run_once(&a).await); + let (status, on_fallback, credits): (String, bool, i32) = + sqlx::query_as("SELECT status, on_fallback, credits_charged FROM analyses WHERE id = $1") + .bind(id) + .fetch_one(&db) + .await + .unwrap(); + assert_eq!((status.as_str(), on_fallback, credits), ("done", true, 1)); + assert!(refunds(&db, s.user).await.is_empty()); + let models: Vec = + fake.requests.lock().unwrap().iter().map(|b| b["model"].as_str().unwrap().to_string()).collect(); + assert_eq!(models, ["cheap/model", "cheap/model", "test/model"]); + } }