Models: one for analyses and forecasts, one for answers; Opus 5.5 default and fallback
CI / test (push) Successful in 1m1s
CI / build (push) Successful in 4m56s
CI / deploy (push) Skipped

- Admin Modellek: summary, analyses + forecast (shared), question-answer (own model)
- Every default is Opus 5.5; when the chosen model fails twice (permanent: once),
  the default model takes over with fresh attempts before the unit is refunded
- The page shows how often the fallback was needed in the last 30 days

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012vLHzh5qUhFtmJhzkcTbXL
This commit is contained in:
feco
2026-11-18 21:11:08 +00:00
co-authored by Claude Opus 5.5
parent 5af08ab689
commit 769b4791ae
18 changed files with 370 additions and 138 deletions
+22 -20
View File
@@ -1499,9 +1499,8 @@ async fn admins_choose_the_model_of_each_function(pool: PgPool) {
models,
[
("summary", "anthropic/claude-summary", false),
("natal", "anthropic/claude-test", false),
("synastry", "anthropic/claude-test", false),
("forecast", "anthropic/claude-test", false),
("analysis", "anthropic/claude-test", false),
("question", "anthropic/claude-test", false),
]
);
@@ -1516,9 +1515,9 @@ async fn admins_choose_the_model_of_each_function(pool: PgPool) {
let (app, admin) = (&app, &admin);
async move { app.call("PUT", &uri, Some(admin), Some(json!({ "model": model }))).await }
};
assert_eq!(put("natal", json!("vendor/plain")).await.status, StatusCode::BAD_REQUEST);
assert_eq!(put("composite", json!("openai/gpt-x")).await.status, StatusCode::NOT_FOUND);
let r = put("natal", json!("openai/gpt-x")).await;
assert_eq!(put("analysis", json!("vendor/plain")).await.status, StatusCode::BAD_REQUEST);
assert_eq!(put("natal", json!("openai/gpt-x")).await.status, StatusCode::NOT_FOUND);
let r = put("analysis", json!("openai/gpt-x")).await;
assert_eq!((r.status, &r.body["model"], &r.body["custom"]), (StatusCode::OK, &json!("openai/gpt-x"), &json!(true)));
// new analyses use it; regenerating writes it again in place with the current model
@@ -1527,9 +1526,13 @@ async fn admins_choose_the_model_of_each_function(pool: PgPool) {
let r = app.post(&format!("/v1/charts/{chart_id}/analyses"), Some(&admin), json!({})).await;
assert_eq!(r.status, StatusCode::ACCEPTED, "{}", r.body);
let id = r.body["id"].as_str().unwrap().to_string();
let model =
|| sqlx::query_scalar::<_, String>("SELECT model FROM analyses WHERE id = $1::uuid").bind(&id).fetch_one(&pool);
assert_eq!(model().await.unwrap(), "openai/gpt-x");
let model = || {
sqlx::query_as::<_, (String, Option<String>)>("SELECT model, fallback_model FROM analyses WHERE id = $1::uuid")
.bind(&id)
.fetch_one(&pool)
};
// the default model is kept as the fallback
assert_eq!(model().await.unwrap(), ("openai/gpt-x".into(), Some("anthropic/claude-test".into())));
let regenerate_uri = format!("/v1/admin/analyses/{id}/regenerate");
let regenerate = || app.post(&regenerate_uri, Some(&admin), json!({}));
assert_eq!(regenerate().await.status, StatusCode::CONFLICT); // still being written
@@ -1537,7 +1540,7 @@ async fn admins_choose_the_model_of_each_function(pool: PgPool) {
.execute(&pool)
.await
.unwrap();
assert_eq!(put("natal", Value::Null).await.body["model"], "anthropic/claude-test");
assert_eq!(put("analysis", Value::Null).await.body["model"], "anthropic/claude-test");
assert_eq!(
app.post(&format!("/v1/admin/analyses/{id}/regenerate"), Some(&anna), json!({})).await.status,
StatusCode::FORBIDDEN
@@ -1548,7 +1551,7 @@ async fn admins_choose_the_model_of_each_function(pool: PgPool) {
(StatusCode::OK, Some("pending"), &Value::Null)
);
assert_eq!(r.body["id"].as_str(), Some(id.as_str()));
assert_eq!(model().await.unwrap(), "anthropic/claude-test");
assert_eq!(model().await.unwrap(), ("anthropic/claude-test".into(), None));
let r = app.get(&format!("/v1/charts/{chart_id}/analyses"), Some(&admin)).await;
assert_eq!(r.body["analyses"].as_array().unwrap().iter().filter(|a| a["kind"] == "natal").count(), 1);
}
@@ -1587,15 +1590,14 @@ async fn questions_about_an_analysis_use_question_units(pool: PgPool) {
assert!(r.body.get("model").is_none(), "the model is a secret");
assert_eq!(left().await, json!(2));
let question = r.body["id"].as_str().unwrap().to_string();
// the same model as the analysis answers
let (q_model, a_model): (String, String) = sqlx::query_as(
"SELECT q.model, a.model FROM questions q JOIN analyses a ON a.id = q.analysis_id WHERE q.id = $1::uuid",
)
.bind(&question)
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(q_model, a_model);
// the question model of the admin page answers (here the default)
let (q_model, fallback): (String, Option<String>) =
sqlx::query_as("SELECT model, fallback_model FROM questions WHERE id = $1::uuid")
.bind(&question)
.fetch_one(&pool)
.await
.unwrap();
assert_eq!((q_model.as_str(), fallback), ("anthropic/claude-test", None));
// one at a time
let r = ask(&anna, "És a munkában?").await;
assert_eq!((r.status, r.body["error"]["code"].as_str()), (StatusCode::CONFLICT, Some("question_in_progress")));