Models: one for analyses and forecasts, one for answers; Opus 5.5 default and fallback
- Admin Modellek: summary, analyses + forecast (shared), question-answer (own model) - Every default is Opus 5.5; when the chosen model fails twice (permanent: once), the default model takes over with fresh attempts before the unit is refunded - The page shows how often the fallback was needed in the last 30 days Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012vLHzh5qUhFtmJhzkcTbXL
This commit is contained in:
@@ -8,6 +8,10 @@
|
||||
//!
|
||||
//! Admin test runs of an edited prompt (`prompt_previews`) go through the
|
||||
//! same loop when no analysis is due: one attempt, no credits, no retries.
|
||||
//!
|
||||
//! Fallback: when the model chosen on the admin page fails twice (or once with
|
||||
//! a permanent error), the default model (`fallback_model`) takes over with a
|
||||
//! fresh set of attempts; only when that fails too is the unit refunded.
|
||||
|
||||
pub mod compact;
|
||||
pub mod openrouter;
|
||||
@@ -40,6 +44,45 @@ pub struct Job {
|
||||
pub model: String,
|
||||
pub prompt_version: String,
|
||||
pub attempts: i32,
|
||||
/// The default model, when the chosen one is another (see the module docs).
|
||||
pub fallback_model: Option<String>,
|
||||
pub on_fallback: bool,
|
||||
}
|
||||
|
||||
impl Job {
|
||||
/// The model of this attempt.
|
||||
pub fn active_model(&self) -> &str {
|
||||
active_model(&self.model, self.fallback_model.as_deref(), self.on_fallback)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn active_model<'a>(model: &'a str, fallback: Option<&'a str>, on_fallback: bool) -> &'a str {
|
||||
match fallback {
|
||||
Some(f) if on_fallback => f,
|
||||
_ => model,
|
||||
}
|
||||
}
|
||||
|
||||
/// What happens after a failed attempt.
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub enum Next {
|
||||
Retry(Duration),
|
||||
/// Switch to the fallback model, with a fresh set of attempts, at once.
|
||||
Fallback,
|
||||
/// Give up (and refund).
|
||||
Fail,
|
||||
}
|
||||
|
||||
/// The chosen model gets two attempts (one after a permanent error), then the
|
||||
/// fallback model takes over with its own `max_attempts`.
|
||||
pub fn next_step(error: &LlmError, attempts: i32, max_attempts: i32, can_fall_back: bool) -> Next {
|
||||
if can_fall_back && (matches!(error, LlmError::Permanent(_)) || attempts >= 2) {
|
||||
return Next::Fallback;
|
||||
}
|
||||
match error {
|
||||
LlmError::Transient(_) if attempts < max_attempts => Next::Retry(backoff(attempts)),
|
||||
_ => Next::Fail,
|
||||
}
|
||||
}
|
||||
|
||||
/// A test run of an edited system prompt on a chart.
|
||||
@@ -81,7 +124,8 @@ impl Analyzer {
|
||||
"UPDATE analyses SET status = 'running', started_at = now(), attempts = attempts + 1 \
|
||||
WHERE id = (SELECT id FROM analyses WHERE status = 'pending' AND next_attempt_at <= now() \
|
||||
ORDER BY next_attempt_at LIMIT 1 FOR UPDATE SKIP LOCKED) \
|
||||
RETURNING id, kind, chart_id, synastry_id, forecast_id, language, model, prompt_version, attempts",
|
||||
RETURNING id, kind, chart_id, synastry_id, forecast_id, language, model, prompt_version, attempts, \
|
||||
fallback_model, on_fallback",
|
||||
)
|
||||
.fetch_optional(&self.db)
|
||||
.await?;
|
||||
@@ -230,7 +274,7 @@ impl Analyzer {
|
||||
pub async fn handle(&self, job: Job) {
|
||||
let started = Instant::now();
|
||||
let result = match self.prompt_for(&job).await {
|
||||
Ok(prompt) => self.llm.complete(&job.model, &prompt).await.map(|mut c| {
|
||||
Ok(prompt) => self.llm.complete(job.active_model(), &prompt).await.map(|mut c| {
|
||||
prompt::polish(&job.language, &mut c.content);
|
||||
c
|
||||
}),
|
||||
@@ -263,22 +307,40 @@ impl Analyzer {
|
||||
cost_usd = c.cost_usd, secs = took.as_secs(), "analysis done"
|
||||
);
|
||||
}
|
||||
Err(LlmError::Transient(message)) if job.attempts < self.max_attempts => {
|
||||
let wait = backoff(job.attempts);
|
||||
sqlx::query(
|
||||
"UPDATE analyses SET status = 'pending', error = $2, \
|
||||
next_attempt_at = now() + make_interval(secs => $3) WHERE id = $1 AND status = 'running'",
|
||||
)
|
||||
.bind(job.id)
|
||||
.bind(&message)
|
||||
.bind(wait.as_secs_f64())
|
||||
.execute(&self.db)
|
||||
.await?;
|
||||
tracing::warn!(id = %job.id, attempts = job.attempts, retry_in_secs = wait.as_secs(), error = %message, "analysis will be retried");
|
||||
}
|
||||
Err(e) => {
|
||||
self.fail(job.id, e.message()).await?;
|
||||
tracing::error!(id = %job.id, attempts = job.attempts, error = %e.message(), "analysis failed");
|
||||
let can_fall_back = !job.on_fallback && job.fallback_model.is_some();
|
||||
match next_step(&e, job.attempts, self.max_attempts, can_fall_back) {
|
||||
Next::Retry(wait) => {
|
||||
sqlx::query(
|
||||
"UPDATE analyses SET status = 'pending', error = $2, \
|
||||
next_attempt_at = now() + make_interval(secs => $3) WHERE id = $1 AND status = 'running'",
|
||||
)
|
||||
.bind(job.id)
|
||||
.bind(e.message())
|
||||
.bind(wait.as_secs_f64())
|
||||
.execute(&self.db)
|
||||
.await?;
|
||||
tracing::warn!(id = %job.id, attempts = job.attempts, retry_in_secs = wait.as_secs(), error = %e.message(), "analysis will be retried");
|
||||
}
|
||||
Next::Fallback => {
|
||||
sqlx::query(
|
||||
"UPDATE analyses SET status = 'pending', error = $2, on_fallback = true, attempts = 0, \
|
||||
next_attempt_at = now() WHERE id = $1 AND status = 'running'",
|
||||
)
|
||||
.bind(job.id)
|
||||
.bind(e.message())
|
||||
.execute(&self.db)
|
||||
.await?;
|
||||
tracing::warn!(
|
||||
id = %job.id, model = %job.model, fallback = ?job.fallback_model, error = %e.message(),
|
||||
"the chosen model failed, the default model takes over"
|
||||
);
|
||||
}
|
||||
Next::Fail => {
|
||||
self.fail(job.id, e.message()).await?;
|
||||
tracing::error!(id = %job.id, attempts = job.attempts, error = %e.message(), "analysis failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
//! Answers to questions about a finished analysis (`questions` rows). The
|
||||
//! model of the analysis answers, seeing the chart data the analysis was
|
||||
//! written from, the analysis itself and the earlier questions about it.
|
||||
//! Retries and refunds work as for analyses.
|
||||
//! question model answers, seeing the chart data the analysis was written
|
||||
//! from, the analysis itself and the earlier questions about it. Retries,
|
||||
//! fallback and refunds work as for analyses.
|
||||
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
@@ -10,7 +10,7 @@ use uuid::Uuid;
|
||||
|
||||
use super::openrouter::{Completion, LlmError};
|
||||
use super::prompt::{self, Kind};
|
||||
use super::{Analyzer, backoff};
|
||||
use super::{Analyzer, Next, active_model, next_step};
|
||||
|
||||
/// The earlier questions and answers given to the model, at most this many.
|
||||
const HISTORY: i64 = 6;
|
||||
@@ -25,6 +25,8 @@ pub struct QuestionJob {
|
||||
pub prompt_version: String,
|
||||
pub attempts: i32,
|
||||
pub created_at: time::OffsetDateTime,
|
||||
pub fallback_model: Option<String>,
|
||||
pub on_fallback: bool,
|
||||
}
|
||||
|
||||
impl Analyzer {
|
||||
@@ -34,7 +36,8 @@ impl Analyzer {
|
||||
"UPDATE questions SET status = 'running', started_at = now(), attempts = attempts + 1 \
|
||||
WHERE id = (SELECT id FROM questions WHERE status = 'pending' AND next_attempt_at <= now() \
|
||||
ORDER BY next_attempt_at LIMIT 1 FOR UPDATE SKIP LOCKED) \
|
||||
RETURNING id, analysis_id, question, language, model, prompt_version, attempts, created_at",
|
||||
RETURNING id, analysis_id, question, language, model, prompt_version, attempts, created_at, \
|
||||
fallback_model, on_fallback",
|
||||
)
|
||||
.fetch_optional(&self.db)
|
||||
.await?;
|
||||
@@ -89,10 +92,13 @@ impl Analyzer {
|
||||
pub async fn handle_question(&self, job: QuestionJob) {
|
||||
let started = Instant::now();
|
||||
let result = match self.question_prompt(&job).await {
|
||||
Ok(p) => self.llm.complete(&job.model, &p).await.map(|mut c| {
|
||||
prompt::polish(&job.language, &mut c.content);
|
||||
c
|
||||
}),
|
||||
Ok(p) => {
|
||||
let model = active_model(&job.model, job.fallback_model.as_deref(), job.on_fallback);
|
||||
self.llm.complete(model, &p).await.map(|mut c| {
|
||||
prompt::polish(&job.language, &mut c.content);
|
||||
c
|
||||
})
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
};
|
||||
if let Err(e) = self.finish_question(&job, result, started.elapsed()).await {
|
||||
@@ -128,22 +134,40 @@ impl Analyzer {
|
||||
secs = took.as_secs(), "answer done"
|
||||
);
|
||||
}
|
||||
Err(LlmError::Transient(message)) if job.attempts < self.max_attempts => {
|
||||
let wait = backoff(job.attempts);
|
||||
sqlx::query(
|
||||
"UPDATE questions SET status = 'pending', error = $2, \
|
||||
next_attempt_at = now() + make_interval(secs => $3) WHERE id = $1 AND status = 'running'",
|
||||
)
|
||||
.bind(job.id)
|
||||
.bind(&message)
|
||||
.bind(wait.as_secs_f64())
|
||||
.execute(&self.db)
|
||||
.await?;
|
||||
tracing::warn!(id = %job.id, attempts = job.attempts, error = %message, "answer will be retried");
|
||||
}
|
||||
Err(e) => {
|
||||
self.fail_question(job.id, e.message()).await?;
|
||||
tracing::error!(id = %job.id, attempts = job.attempts, error = %e.message(), "answer failed");
|
||||
let can_fall_back = !job.on_fallback && job.fallback_model.is_some();
|
||||
match next_step(&e, job.attempts, self.max_attempts, can_fall_back) {
|
||||
Next::Retry(wait) => {
|
||||
sqlx::query(
|
||||
"UPDATE questions SET status = 'pending', error = $2, \
|
||||
next_attempt_at = now() + make_interval(secs => $3) WHERE id = $1 AND status = 'running'",
|
||||
)
|
||||
.bind(job.id)
|
||||
.bind(e.message())
|
||||
.bind(wait.as_secs_f64())
|
||||
.execute(&self.db)
|
||||
.await?;
|
||||
tracing::warn!(id = %job.id, attempts = job.attempts, error = %e.message(), "answer will be retried");
|
||||
}
|
||||
Next::Fallback => {
|
||||
sqlx::query(
|
||||
"UPDATE questions SET status = 'pending', error = $2, on_fallback = true, attempts = 0, \
|
||||
next_attempt_at = now() WHERE id = $1 AND status = 'running'",
|
||||
)
|
||||
.bind(job.id)
|
||||
.bind(e.message())
|
||||
.execute(&self.db)
|
||||
.await?;
|
||||
tracing::warn!(
|
||||
id = %job.id, model = %job.model, error = %e.message(),
|
||||
"the chosen model failed, the default model answers"
|
||||
);
|
||||
}
|
||||
Next::Fail => {
|
||||
self.fail_question(job.id, e.message()).await?;
|
||||
tracing::error!(id = %job.id, attempts = job.attempts, error = %e.message(), "answer failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
|
||||
@@ -736,4 +736,49 @@ mod analyses {
|
||||
.unwrap();
|
||||
assert_eq!(refunds, vec![("question".to_string(), 1)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_chosen_model_gets_two_attempts_then_the_default_takes_over() {
|
||||
use crate::analysis::openrouter::LlmError;
|
||||
use crate::analysis::{Next, next_step};
|
||||
let t = || LlmError::Transient("overloaded".into());
|
||||
let p = || LlmError::Permanent("no such model".into());
|
||||
assert_eq!(next_step(&t(), 1, 3, true), Next::Retry(backoff(1)));
|
||||
assert_eq!(next_step(&t(), 2, 3, true), Next::Fallback);
|
||||
assert_eq!(next_step(&p(), 1, 3, true), Next::Fallback);
|
||||
// without a fallback (or already on it): retries, then failure
|
||||
assert_eq!(next_step(&t(), 2, 3, false), Next::Retry(backoff(2)));
|
||||
assert_eq!(next_step(&t(), 3, 3, false), Next::Fail);
|
||||
assert_eq!(next_step(&p(), 1, 3, false), Next::Fail);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrator = "db::MIGRATOR")]
|
||||
async fn the_default_model_writes_when_the_chosen_one_fails(db: PgPool) {
|
||||
let s = setup(&db).await;
|
||||
let id = request_analysis(&db, &s).await;
|
||||
sqlx::query("UPDATE analyses SET model = 'cheap/model', fallback_model = 'test/model' WHERE id = $1")
|
||||
.bind(id)
|
||||
.execute(&db)
|
||||
.await
|
||||
.unwrap();
|
||||
let broken = json!({"title": "Csak cím"});
|
||||
let (fake, url) = fake_openrouter(vec![reply(&broken), reply(&broken), reply(&content(Kind::Natal))]).await;
|
||||
let a = analyzer(&db, &url);
|
||||
|
||||
assert!(run_once(&a).await);
|
||||
make_due(&db, id).await;
|
||||
assert!(run_once(&a).await); // second failure: the default model takes over at once
|
||||
assert!(run_once(&a).await);
|
||||
let (status, on_fallback, credits): (String, bool, i32) =
|
||||
sqlx::query_as("SELECT status, on_fallback, credits_charged FROM analyses WHERE id = $1")
|
||||
.bind(id)
|
||||
.fetch_one(&db)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!((status.as_str(), on_fallback, credits), ("done", true, 1));
|
||||
assert!(refunds(&db, s.user).await.is_empty());
|
||||
let models: Vec<String> =
|
||||
fake.requests.lock().unwrap().iter().map(|b| b["model"].as_str().unwrap().to_string()).collect();
|
||||
assert_eq!(models, ["cheap/model", "cheap/model", "test/model"]);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user