Models: one for analyses and forecasts, one for answers; Opus 5.5 default and fallback
CI / test (push) Successful in 1m1s
CI / build (push) Successful in 4m56s
CI / deploy (push) Skipped

- Admin Modellek: summary, analyses + forecast (shared), question-answer (own model)
- Every default is Opus 5.5; when the chosen model fails twice (permanent: once),
  the default model takes over with fresh attempts before the unit is refunded
- The page shows how often the fallback was needed in the last 30 days

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012vLHzh5qUhFtmJhzkcTbXL
This commit is contained in:
feco
2026-11-18 21:11:08 +00:00
co-authored by Claude Opus 5.5
parent 5af08ab689
commit 769b4791ae
18 changed files with 370 additions and 138 deletions
+45
View File
@@ -736,4 +736,49 @@ mod analyses {
.unwrap();
assert_eq!(refunds, vec![("question".to_string(), 1)]);
}
#[test]
fn the_chosen_model_gets_two_attempts_then_the_default_takes_over() {
use crate::analysis::openrouter::LlmError;
use crate::analysis::{Next, next_step};
let t = || LlmError::Transient("overloaded".into());
let p = || LlmError::Permanent("no such model".into());
assert_eq!(next_step(&t(), 1, 3, true), Next::Retry(backoff(1)));
assert_eq!(next_step(&t(), 2, 3, true), Next::Fallback);
assert_eq!(next_step(&p(), 1, 3, true), Next::Fallback);
// without a fallback (or already on it): retries, then failure
assert_eq!(next_step(&t(), 2, 3, false), Next::Retry(backoff(2)));
assert_eq!(next_step(&t(), 3, 3, false), Next::Fail);
assert_eq!(next_step(&p(), 1, 3, false), Next::Fail);
}
#[sqlx::test(migrator = "db::MIGRATOR")]
async fn the_default_model_writes_when_the_chosen_one_fails(db: PgPool) {
let s = setup(&db).await;
let id = request_analysis(&db, &s).await;
sqlx::query("UPDATE analyses SET model = 'cheap/model', fallback_model = 'test/model' WHERE id = $1")
.bind(id)
.execute(&db)
.await
.unwrap();
let broken = json!({"title": "Csak cím"});
let (fake, url) = fake_openrouter(vec![reply(&broken), reply(&broken), reply(&content(Kind::Natal))]).await;
let a = analyzer(&db, &url);
assert!(run_once(&a).await);
make_due(&db, id).await;
assert!(run_once(&a).await); // second failure: the default model takes over at once
assert!(run_once(&a).await);
let (status, on_fallback, credits): (String, bool, i32) =
sqlx::query_as("SELECT status, on_fallback, credits_charged FROM analyses WHERE id = $1")
.bind(id)
.fetch_one(&db)
.await
.unwrap();
assert_eq!((status.as_str(), on_fallback, credits), ("done", true, 1));
assert!(refunds(&db, s.user).await.is_empty());
let models: Vec<String> =
fake.requests.lock().unwrap().iter().map(|b| b["model"].as_str().unwrap().to_string()).collect();
assert_eq!(models, ["cheap/model", "cheap/model", "test/model"]);
}
}