Models: one for analyses and forecasts, one for answers; Opus 5.5 default and fallback
CI / test (push) Successful in 1m1s
CI / build (push) Successful in 4m56s
CI / deploy (push) Skipped

- Admin Modellek: summary, analyses + forecast (shared), question-answer (own model)
- Every default is Opus 5.5; when the chosen model fails twice (permanent: once),
  the default model takes over with fresh attempts before the unit is refunded
- The page shows how often the fallback was needed in the last 30 days

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012vLHzh5qUhFtmJhzkcTbXL
This commit is contained in:
feco
2026-11-18 21:11:08 +00:00
co-authored by Claude Opus 5.5
parent 5af08ab689
commit 769b4791ae
18 changed files with 370 additions and 138 deletions
+79 -17
View File
@@ -8,6 +8,10 @@
//!
//! Admin test runs of an edited prompt (`prompt_previews`) go through the
//! same loop when no analysis is due: one attempt, no credits, no retries.
//!
//! Fallback: when the model chosen on the admin page fails twice (or once with
//! a permanent error), the default model (`fallback_model`) takes over with a
//! fresh set of attempts; only when that fails too is the unit refunded.
pub mod compact;
pub mod openrouter;
@@ -40,6 +44,45 @@ pub struct Job {
pub model: String,
pub prompt_version: String,
pub attempts: i32,
/// The default model, when the chosen one is another (see the module docs).
pub fallback_model: Option<String>,
pub on_fallback: bool,
}
impl Job {
/// The model of this attempt.
pub fn active_model(&self) -> &str {
active_model(&self.model, self.fallback_model.as_deref(), self.on_fallback)
}
}
pub fn active_model<'a>(model: &'a str, fallback: Option<&'a str>, on_fallback: bool) -> &'a str {
match fallback {
Some(f) if on_fallback => f,
_ => model,
}
}
/// What happens after a failed attempt.
#[derive(Debug, PartialEq, Eq)]
pub enum Next {
Retry(Duration),
/// Switch to the fallback model, with a fresh set of attempts, at once.
Fallback,
/// Give up (and refund).
Fail,
}
/// The chosen model gets two attempts (one after a permanent error), then the
/// fallback model takes over with its own `max_attempts`.
pub fn next_step(error: &LlmError, attempts: i32, max_attempts: i32, can_fall_back: bool) -> Next {
if can_fall_back && (matches!(error, LlmError::Permanent(_)) || attempts >= 2) {
return Next::Fallback;
}
match error {
LlmError::Transient(_) if attempts < max_attempts => Next::Retry(backoff(attempts)),
_ => Next::Fail,
}
}
/// A test run of an edited system prompt on a chart.
@@ -81,7 +124,8 @@ impl Analyzer {
"UPDATE analyses SET status = 'running', started_at = now(), attempts = attempts + 1 \
WHERE id = (SELECT id FROM analyses WHERE status = 'pending' AND next_attempt_at <= now() \
ORDER BY next_attempt_at LIMIT 1 FOR UPDATE SKIP LOCKED) \
RETURNING id, kind, chart_id, synastry_id, forecast_id, language, model, prompt_version, attempts",
RETURNING id, kind, chart_id, synastry_id, forecast_id, language, model, prompt_version, attempts, \
fallback_model, on_fallback",
)
.fetch_optional(&self.db)
.await?;
@@ -230,7 +274,7 @@ impl Analyzer {
pub async fn handle(&self, job: Job) {
let started = Instant::now();
let result = match self.prompt_for(&job).await {
Ok(prompt) => self.llm.complete(&job.model, &prompt).await.map(|mut c| {
Ok(prompt) => self.llm.complete(job.active_model(), &prompt).await.map(|mut c| {
prompt::polish(&job.language, &mut c.content);
c
}),
@@ -263,22 +307,40 @@ impl Analyzer {
cost_usd = c.cost_usd, secs = took.as_secs(), "analysis done"
);
}
Err(LlmError::Transient(message)) if job.attempts < self.max_attempts => {
let wait = backoff(job.attempts);
sqlx::query(
"UPDATE analyses SET status = 'pending', error = $2, \
next_attempt_at = now() + make_interval(secs => $3) WHERE id = $1 AND status = 'running'",
)
.bind(job.id)
.bind(&message)
.bind(wait.as_secs_f64())
.execute(&self.db)
.await?;
tracing::warn!(id = %job.id, attempts = job.attempts, retry_in_secs = wait.as_secs(), error = %message, "analysis will be retried");
}
Err(e) => {
self.fail(job.id, e.message()).await?;
tracing::error!(id = %job.id, attempts = job.attempts, error = %e.message(), "analysis failed");
let can_fall_back = !job.on_fallback && job.fallback_model.is_some();
match next_step(&e, job.attempts, self.max_attempts, can_fall_back) {
Next::Retry(wait) => {
sqlx::query(
"UPDATE analyses SET status = 'pending', error = $2, \
next_attempt_at = now() + make_interval(secs => $3) WHERE id = $1 AND status = 'running'",
)
.bind(job.id)
.bind(e.message())
.bind(wait.as_secs_f64())
.execute(&self.db)
.await?;
tracing::warn!(id = %job.id, attempts = job.attempts, retry_in_secs = wait.as_secs(), error = %e.message(), "analysis will be retried");
}
Next::Fallback => {
sqlx::query(
"UPDATE analyses SET status = 'pending', error = $2, on_fallback = true, attempts = 0, \
next_attempt_at = now() WHERE id = $1 AND status = 'running'",
)
.bind(job.id)
.bind(e.message())
.execute(&self.db)
.await?;
tracing::warn!(
id = %job.id, model = %job.model, fallback = ?job.fallback_model, error = %e.message(),
"the chosen model failed, the default model takes over"
);
}
Next::Fail => {
self.fail(job.id, e.message()).await?;
tracing::error!(id = %job.id, attempts = job.attempts, error = %e.message(), "analysis failed");
}
}
}
}
Ok(())