Models: one for analyses and forecasts, one for answers; Opus 5.5 default and fallback
CI / test (push) Successful in 1m1s
CI / build (push) Successful in 4m56s
CI / deploy (push) Skipped

- Admin Modellek: summary, analyses + forecast (shared), question-answer (own model)
- Every default is Opus 5.5; when the chosen model fails twice (permanent: once),
  the default model takes over with fresh attempts before the unit is refunded
- The page shows how often the fallback was needed in the last 30 days

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012vLHzh5qUhFtmJhzkcTbXL
This commit is contained in:
feco
2026-11-18 21:11:08 +00:00
co-authored by Claude Opus 5.5
parent 5af08ab689
commit 769b4791ae
18 changed files with 370 additions and 138 deletions
+9 -5
View File
@@ -175,7 +175,7 @@ pub(super) async fn queue(
Target::Synastry { synastry_id, hash } => ("synastry", hash, None, Some(*synastry_id), None),
Target::Forecast { forecast_id, hash } => ("forecast", hash, None, None, Some(*forecast_id)),
};
let model = &crate::models::model_for(state, Kind::parse(kind).expect("known kind")).await?;
let (model, fallback) = &crate::models::models_for(state, Kind::parse(kind).expect("known kind")).await?;
let (cache_key, prompt_version) = &versions(state, kind, hash, model, language).await?;
// Already requested for this chart/synastry: return it.
@@ -197,8 +197,8 @@ pub(super) async fn queue(
let insert = format!(
"INSERT INTO analyses (id, kind, chart_id, synastry_id, requested_by, language, model, prompt_version, \
cache_key, status, content, input_tokens, output_tokens, credits_charged, completed_at, \
forecast_id) \
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16) RETURNING {ANALYSIS_COLUMNS}"
forecast_id, fallback_model) \
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17) RETURNING {ANALYSIS_COLUMNS}"
);
let id = Uuid::now_v7();
@@ -229,6 +229,7 @@ pub(super) async fn queue(
.bind(0)
.bind(OffsetDateTime::now_utc())
.bind(forecast_id)
.bind(fallback)
.fetch_one(&state.db)
.await?;
return Ok((StatusCode::CREATED, row));
@@ -253,6 +254,7 @@ pub(super) async fn queue(
.bind(i32::from(unit.is_some()))
.bind(None::<OffsetDateTime>)
.bind(forecast_id)
.bind(fallback)
.fetch_one(&mut *tx)
.await?;
if let Some(unit) = unit {
@@ -391,19 +393,21 @@ pub async fn regenerate(
}
};
let hash = hash.ok_or(ApiError::NotFound("analysis"))?;
let model = crate::models::model_for(&state, kind).await?;
let (model, fallback) = crate::models::models_for(&state, kind).await?;
let (cache_key, prompt_version) = versions(&state, kind.as_str(), &hash, &model, &row.language).await?;
let mut tx = state.db.begin().await?;
let row: AnalysisRow = sqlx::query_as(&format!(
"UPDATE analyses SET status = 'pending', content = NULL, error = NULL, model_used = NULL, \
input_tokens = NULL, output_tokens = NULL, cost_usd = NULL, attempts = 0, next_attempt_at = now(), \
started_at = NULL, completed_at = NULL, model = $2, prompt_version = $3, cache_key = $4 \
started_at = NULL, completed_at = NULL, model = $2, prompt_version = $3, cache_key = $4, \
fallback_model = $5, on_fallback = false \
WHERE id = $1 AND status IN ('done', 'failed') RETURNING {ANALYSIS_COLUMNS}"
))
.bind(id)
.bind(&model)
.bind(&prompt_version)
.bind(&cache_key)
.bind(&fallback)
.fetch_optional(&mut *tx)
.await?
.ok_or_else(|| ApiError::conflict("analysis_in_progress", "Ez az elemzés éppen készül."))?;
+36 -23
View File
@@ -1,7 +1,7 @@
//! Admin: which model writes which kind of analysis, chosen from the models
//! OpenRouter offers. A change applies to analyses requested after it.
//! Admin: which model writes the analyses and forecasts, the answers to
//! questions and the short summary, chosen from the models OpenRouter offers.
//! A change applies to texts requested after it.
use analysis_prompt::Kind;
use axum::Json;
use axum::extract::{Path, State};
use axum_extra::extract::WithRejection;
@@ -13,7 +13,7 @@ use super::{Body, St};
use crate::AppState;
use crate::auth::AdminUser;
use crate::error::{ApiError, ApiResult};
use crate::models::default_model;
use crate::models::{Function, default_model};
use crate::views::rfc3339;
fn catalog_error(e: String) -> ApiError {
@@ -23,27 +23,38 @@ fn catalog_error(e: String) -> ApiError {
}
}
async fn entry(state: &AppState, kind: Kind) -> Result<Value, sqlx::Error> {
async fn entry(state: &AppState, function: Function) -> Result<Value, sqlx::Error> {
let chosen: Option<(String, OffsetDateTime, Option<String>)> = sqlx::query_as(
"SELECT m.model, m.updated_at, u.email FROM model_settings m LEFT JOIN users u ON u.id = m.updated_by \
WHERE m.kind = $1",
)
.bind(kind.as_str())
.bind(function.key())
.fetch_optional(&state.db)
.await?;
// Typical size of this kind of analysis, from the latest ones: for the cost estimate.
let (input, output, samples): (Option<f64>, Option<f64>, i64) = sqlx::query_as(
// Typical size of the latest texts, for the cost estimate, and how often
// the default model had to take over in the last 30 days.
let (table, kinds) = match function {
Function::Summary => ("analyses", "kind = 'summary'"),
Function::Analysis => ("analyses", "kind IN ('natal', 'synastry', 'forecast')"),
Function::Question => ("questions", "true"),
};
let (input, output, samples): (Option<f64>, Option<f64>, i64) = sqlx::query_as(&format!(
"SELECT avg(input_tokens)::float8, avg(output_tokens)::float8, count(*) FROM ( \
SELECT input_tokens, output_tokens FROM analyses \
WHERE kind = $1 AND status = 'done' AND input_tokens IS NOT NULL AND output_tokens IS NOT NULL \
ORDER BY completed_at DESC LIMIT 50) recent",
)
.bind(kind.as_str())
SELECT input_tokens, output_tokens FROM {table} \
WHERE {kinds} AND status = 'done' AND input_tokens IS NOT NULL AND output_tokens IS NOT NULL \
ORDER BY completed_at DESC LIMIT 50) recent"
))
.fetch_one(&state.db)
.await?;
let default = default_model(state, kind);
let (fallbacks, finished): (i64, i64) = sqlx::query_as(&format!(
"SELECT count(*) FILTER (WHERE on_fallback), count(*) FROM {table} \
WHERE {kinds} AND status IN ('done', 'failed') AND completed_at > now() - interval '30 days'"
))
.fetch_one(&state.db)
.await?;
let default = default_model(state, function);
Ok(json!({
"kind": kind.as_str(),
"kind": function.key(),
"model": chosen.as_ref().map_or(default, |c| c.0.as_str()),
"default": default,
"custom": chosen.is_some(),
@@ -52,13 +63,15 @@ async fn entry(state: &AppState, kind: Kind) -> Result<Value, sqlx::Error> {
"avg_input_tokens": input,
"avg_output_tokens": output,
"samples": samples,
"fallbacks_30d": fallbacks,
"finished_30d": finished,
}))
}
pub async fn list(State(state): St, _admin: AdminUser) -> ApiResult<Json<Value>> {
let mut functions = Vec::new();
for kind in Kind::ANALYSES {
functions.push(entry(&state, kind).await?);
for function in Function::ALL {
functions.push(entry(&state, function).await?);
}
Ok(Json(json!({ "functions": functions })))
}
@@ -81,9 +94,9 @@ pub async fn save(
WithRejection(Path(kind), _): WithRejection<Path<String>, ApiError>,
WithRejection(Json(req), _): Body<SaveModel>,
) -> ApiResult<Json<Value>> {
let kind = Kind::parse(&kind).ok_or(ApiError::NotFound("function"))?;
let function = Function::parse(&kind).ok_or(ApiError::NotFound("function"))?;
match req.model.as_deref().map(str::trim) {
Some(model) if model != default_model(&state, kind) => {
Some(model) if model != default_model(&state, function) => {
let models = state.models.models().await.map_err(catalog_error)?;
if !models.iter().any(|m| m.id == model) {
return Err(ApiError::BadRequest(format!(
@@ -95,16 +108,16 @@ pub async fn save(
ON CONFLICT (kind) DO UPDATE SET model = EXCLUDED.model, updated_by = EXCLUDED.updated_by, \
updated_at = now()",
)
.bind(kind.as_str())
.bind(function.key())
.bind(model)
.bind(admin.id())
.execute(&state.db)
.await?;
}
_ => {
sqlx::query("DELETE FROM model_settings WHERE kind = $1").bind(kind.as_str()).execute(&state.db).await?;
sqlx::query("DELETE FROM model_settings WHERE kind = $1").bind(function.key()).execute(&state.db).await?;
}
}
tracing::info!(admin = %admin.id(), kind = kind.as_str(), "analysis model changed");
Ok(Json(entry(&state, kind).await?))
tracing::info!(admin = %admin.id(), function = function.key(), "model changed");
Ok(Json(entry(&state, function).await?))
}
+21 -18
View File
@@ -1,7 +1,7 @@
//! Questions about a finished detailed analysis: a natal chart analysis or a
//! yearly forecast. Each question uses a `question` unit (admins ask for
//! free). The worker answers with the model that wrote the analysis, seeing
//! the chart data, the analysis and the earlier questions about it.
//! free). The worker answers with the question model of the admin page,
//! seeing the chart data, the analysis and the earlier questions about it.
use analysis_prompt::{Kind, QUESTION_MAX_CHARS};
use axum::Json;
@@ -64,21 +64,19 @@ async fn find_question(db: &PgPool, id: Uuid) -> Result<Option<QuestionRow>, sql
sqlx::query_as(&format!("SELECT {QUESTION_COLUMNS} FROM questions WHERE id = $1")).bind(id).fetch_optional(db).await
}
/// The model of the analysis (questions are answered by the same model) and
/// the prompt version of the question prompt in force.
/// The question model (and its fallback) and the version of the question
/// prompt in force.
async fn model_and_version(
state: &crate::AppState,
analysis_id: Uuid,
language: &str,
) -> Result<(String, String), sqlx::Error> {
let model: String =
sqlx::query_scalar("SELECT model FROM analyses WHERE id = $1").bind(analysis_id).fetch_one(&state.db).await?;
) -> Result<(String, Option<String>, String), sqlx::Error> {
let (model, fallback) = crate::models::models_for(state, Kind::Question).await?;
let base = &state.config.prompt_version;
let version = match super::prompts::current_revision(&state.db, Kind::Question.as_str(), language).await? {
Some(id) => format!("{base}-r{id}"),
None => base.clone(),
};
Ok((model, version))
Ok((model, fallback, version))
}
pub async fn list(State(state): St, auth: AuthUser, WithRejection(Path(id), _): IdPath) -> ApiResult<Json<Value>> {
@@ -141,12 +139,13 @@ pub async fn ask(
}
let admin = auth.user.has_role(Role::Admin);
let (model, prompt_version) = model_and_version(&state, id, &analysis.language).await?;
let (model, fallback, prompt_version) = model_and_version(&state, &analysis.language).await?;
let question_id = Uuid::now_v7();
let mut tx = state.db.begin().await?;
let row: QuestionRow = sqlx::query_as(&format!(
"INSERT INTO questions (id, analysis_id, asked_by, question, language, model, prompt_version, credits_charged) \
VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING {QUESTION_COLUMNS}"
"INSERT INTO questions (id, analysis_id, asked_by, question, language, model, prompt_version, credits_charged, \
fallback_model) \
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) RETURNING {QUESTION_COLUMNS}"
))
.bind(question_id)
.bind(id)
@@ -156,6 +155,7 @@ pub async fn ask(
.bind(&model)
.bind(&prompt_version)
.bind(i32::from(!admin))
.bind(&fallback)
.fetch_one(&mut *tx)
.await?;
if !admin {
@@ -175,26 +175,29 @@ pub async fn get_one(State(state): St, auth: AuthUser, WithRejection(Path(id), _
}
/// Admin: answers a question again in place, with the current question prompt
/// and the analysis's model. Nobody is charged.
/// and model. Nobody is charged.
pub async fn regenerate(
State(state): St,
_admin: AdminUser,
WithRejection(Path(id), _): IdPath,
) -> ApiResult<Json<Value>> {
let row = find_question(&state.db, id).await?.ok_or(ApiError::NotFound("question"))?;
let language: String =
sqlx::query_scalar("SELECT language FROM questions WHERE id = $1").bind(id).fetch_one(&state.db).await?;
let (model, prompt_version) = model_and_version(&state, row.analysis_id, &language).await?;
let language: String = sqlx::query_scalar("SELECT language FROM questions WHERE id = $1")
.bind(id)
.fetch_optional(&state.db)
.await?
.ok_or(ApiError::NotFound("question"))?;
let (model, fallback, prompt_version) = model_and_version(&state, &language).await?;
let mut tx = state.db.begin().await?;
let row: QuestionRow = sqlx::query_as(&format!(
"UPDATE questions SET status = 'pending', answer = NULL, error = NULL, model_used = NULL, input_tokens = NULL, \
output_tokens = NULL, cost_usd = NULL, attempts = 0, next_attempt_at = now(), started_at = NULL, \
completed_at = NULL, model = $2, prompt_version = $3 \
completed_at = NULL, model = $2, prompt_version = $3, fallback_model = $4, on_fallback = false \
WHERE id = $1 AND status IN ('done', 'failed') RETURNING {QUESTION_COLUMNS}"
))
.bind(id)
.bind(&model)
.bind(&prompt_version)
.bind(&fallback)
.fetch_optional(&mut *tx)
.await?
.ok_or_else(|| ApiError::conflict("question_in_progress", "Erre a kérdésre éppen készül a válasz."))?;