Error log, retry notice, a waiting counter that keeps counting
CI / test (push) Successful in 1m8s
CI / build (push) Successful in 5m13s
CI / deploy (push) Skipped

- generation_errors: every failed attempt (model, attempt, outcome, duration); admin
  Hibanapló page and GET /v1/admin/errors; kept 90 days
- The waiting page counts from when the text was asked for (requested_at), not from
  the page load, so a reload no longer restarts it
- It says when an attempt failed and is retried, and when a failed text's unit was
  given back; admins see the error

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012vLHzh5qUhFtmJhzkcTbXL
This commit is contained in:
feco
2026-11-18 21:47:37 +00:00
co-authored by Claude Opus 5.5
parent cf2fb9d454
commit 02e4e225f0
22 changed files with 569 additions and 25 deletions
+52 -1
View File
@@ -253,6 +253,8 @@ impl Analyzer {
.await
}
Err(e) => {
self.log_error("preview", job.id, &job.kind, &job.model, 1, "failed", e.message(), started.elapsed())
.await;
sqlx::query(
"UPDATE prompt_previews SET status = 'failed', error = $2, completed_at = now() \
WHERE id = $1 AND status = 'running'",
@@ -322,7 +324,24 @@ impl Analyzer {
}
Err(e) => {
let can_fall_back = !job.on_fallback && job.fallback_model.is_some();
match next_step(&e, job.attempts, self.max_attempts, can_fall_back) {
let next = next_step(&e, job.attempts, self.max_attempts, can_fall_back);
let outcome = match next {
Next::Retry(_) => "retry",
Next::Fallback => "fallback",
Next::Fail => "failed",
};
self.log_error(
"analysis",
job.id,
&job.kind,
job.active_model(),
job.attempts,
outcome,
e.message(),
took,
)
.await;
match next {
Next::Retry(wait) => {
sqlx::query(
"UPDATE analyses SET status = 'pending', error = $2, \
@@ -424,6 +443,38 @@ impl Analyzer {
Ok(result.rows_affected())
}
/// Records a failed attempt in the admin error log (a failure here is only logged).
#[allow(clippy::too_many_arguments)]
pub(crate) async fn log_error(
&self,
source: &str,
item_id: Uuid,
kind: &str,
model: &str,
attempt: i32,
outcome: &str,
error: &str,
took: Duration,
) {
let result = sqlx::query(
"INSERT INTO generation_errors (source, item_id, kind, model, attempt, outcome, error, seconds) \
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)",
)
.bind(source)
.bind(item_id)
.bind(kind)
.bind(model)
.bind(attempt)
.bind(outcome)
.bind(error.chars().take(2000).collect::<String>())
.bind(i32::try_from(took.as_secs()).unwrap_or(i32::MAX))
.execute(&self.db)
.await;
if let Err(e) = result {
tracing::error!(error = %e, "writing the error log");
}
}
/// Resolves when this attempt is no longer the current one (an admin
/// restarted it); checked every 10 seconds, never on a database error.
async fn superseded(&self, table: &str, id: Uuid, started_at: time::OffsetDateTime) {
+9 -1
View File
@@ -148,7 +148,15 @@ impl Analyzer {
}
Err(e) => {
let can_fall_back = !job.on_fallback && job.fallback_model.is_some();
match next_step(&e, job.attempts, self.max_attempts, can_fall_back) {
let next = next_step(&e, job.attempts, self.max_attempts, can_fall_back);
let outcome = match next {
Next::Retry(_) => "retry",
Next::Fallback => "fallback",
Next::Fail => "failed",
};
let model = active_model(&job.model, job.fallback_model.as_deref(), job.on_fallback);
self.log_error("question", job.id, "question", model, job.attempts, outcome, e.message(), took).await;
match next {
Next::Retry(wait) => {
sqlx::query(
"UPDATE questions SET status = 'pending', error = $2, \
+2 -1
View File
@@ -8,7 +8,7 @@ use tokio::sync::watch;
const INTERVAL: Duration = Duration::from_secs(3600);
/// (what, statement) pairs; each returns the number of deleted rows.
const STATEMENTS: [(&str, &str); 6] = [
const STATEMENTS: [(&str, &str); 7] = [
("sessions", "DELETE FROM sessions WHERE expires_at < now()"),
("auth_tokens", "DELETE FROM auth_tokens WHERE expires_at < now()"),
// Sent e-mails contain links with tokens; keep them a week for support.
@@ -26,6 +26,7 @@ const STATEMENTS: [(&str, &str); 6] = [
OR (accepted_at IS NOT NULL AND accepted_by IS NULL)",
),
("prompt_previews", "DELETE FROM prompt_previews WHERE created_at < now() - interval '30 days'"),
("generation_errors", "DELETE FROM generation_errors WHERE created_at < now() - interval '90 days'"),
];
pub async fn run_once(db: &PgPool) -> Result<Vec<(&'static str, u64)>, sqlx::Error> {
+24 -1
View File
@@ -212,7 +212,8 @@ async fn cleanup_removes_only_expired_data(db: PgPool) {
("emails", 1),
("invites", 0),
("invitations", 0),
("prompt_previews", 0)
("prompt_previews", 0),
("generation_errors", 0)
]
);
let (left,): (Vec<Uuid>,) =
@@ -533,6 +534,18 @@ mod analyses {
let (status, attempts, ..) = row(&db, id).await;
assert_eq!((status.as_str(), attempts), ("failed", 3));
assert_eq!(refunds(&db, s.user).await, vec![1]);
// every failed attempt is in the admin error log
let log: Vec<(String, i32, String, String)> = sqlx::query_as(
"SELECT source, attempt, outcome, model FROM generation_errors WHERE item_id = $1 ORDER BY id",
)
.bind(id)
.fetch_all(&db)
.await
.unwrap();
let expected = |attempt: i32, outcome: &str| {
("analysis".to_string(), attempt, outcome.to_string(), "test/model".to_string())
};
assert_eq!(log, vec![expected(1, "retry"), expected(2, "retry"), expected(3, "failed")]);
}
#[sqlx::test(migrator = "db::MIGRATOR")]
@@ -780,6 +793,16 @@ mod analyses {
let models: Vec<String> =
fake.requests.lock().unwrap().iter().map(|b| b["model"].as_str().unwrap().to_string()).collect();
assert_eq!(models, ["cheap/model", "cheap/model", "test/model"]);
let outcomes: Vec<(String, String)> =
sqlx::query_as("SELECT outcome, model FROM generation_errors WHERE item_id = $1 ORDER BY id")
.bind(id)
.fetch_all(&db)
.await
.unwrap();
assert_eq!(
outcomes,
[("retry".to_string(), "cheap/model".to_string()), ("fallback".to_string(), "cheap/model".to_string())]
);
}
#[sqlx::test(migrator = "db::MIGRATOR")]