Models: one for analyses and forecasts, one for answers; Opus 5.5 default and fallback
- Admin Modellek: summary, analyses + forecast (shared), question-answer (own model) - Every default is Opus 5.5; when the chosen model fails twice (permanent: once), the default model takes over with fresh attempts before the unit is refunded - The page shows how often the fallback was needed in the last 30 days Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012vLHzh5qUhFtmJhzkcTbXL
This commit is contained in:
@@ -736,4 +736,49 @@ mod analyses {
|
||||
.unwrap();
|
||||
assert_eq!(refunds, vec![("question".to_string(), 1)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_chosen_model_gets_two_attempts_then_the_default_takes_over() {
|
||||
use crate::analysis::openrouter::LlmError;
|
||||
use crate::analysis::{Next, next_step};
|
||||
let t = || LlmError::Transient("overloaded".into());
|
||||
let p = || LlmError::Permanent("no such model".into());
|
||||
assert_eq!(next_step(&t(), 1, 3, true), Next::Retry(backoff(1)));
|
||||
assert_eq!(next_step(&t(), 2, 3, true), Next::Fallback);
|
||||
assert_eq!(next_step(&p(), 1, 3, true), Next::Fallback);
|
||||
// without a fallback (or already on it): retries, then failure
|
||||
assert_eq!(next_step(&t(), 2, 3, false), Next::Retry(backoff(2)));
|
||||
assert_eq!(next_step(&t(), 3, 3, false), Next::Fail);
|
||||
assert_eq!(next_step(&p(), 1, 3, false), Next::Fail);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrator = "db::MIGRATOR")]
|
||||
async fn the_default_model_writes_when_the_chosen_one_fails(db: PgPool) {
|
||||
let s = setup(&db).await;
|
||||
let id = request_analysis(&db, &s).await;
|
||||
sqlx::query("UPDATE analyses SET model = 'cheap/model', fallback_model = 'test/model' WHERE id = $1")
|
||||
.bind(id)
|
||||
.execute(&db)
|
||||
.await
|
||||
.unwrap();
|
||||
let broken = json!({"title": "Csak cím"});
|
||||
let (fake, url) = fake_openrouter(vec![reply(&broken), reply(&broken), reply(&content(Kind::Natal))]).await;
|
||||
let a = analyzer(&db, &url);
|
||||
|
||||
assert!(run_once(&a).await);
|
||||
make_due(&db, id).await;
|
||||
assert!(run_once(&a).await); // second failure: the default model takes over at once
|
||||
assert!(run_once(&a).await);
|
||||
let (status, on_fallback, credits): (String, bool, i32) =
|
||||
sqlx::query_as("SELECT status, on_fallback, credits_charged FROM analyses WHERE id = $1")
|
||||
.bind(id)
|
||||
.fetch_one(&db)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!((status.as_str(), on_fallback, credits), ("done", true, 1));
|
||||
assert!(refunds(&db, s.user).await.is_empty());
|
||||
let models: Vec<String> =
|
||||
fake.requests.lock().unwrap().iter().map(|b| b["model"].as_str().unwrap().to_string()).collect();
|
||||
assert_eq!(models, ["cheap/model", "cheap/model", "test/model"]);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user