Error log, retry notice, a waiting counter that keeps counting
- generation_errors: every failed attempt (model, attempt, outcome, duration); admin Hibanapló page and GET /v1/admin/errors; kept 90 days - The waiting page counts from when the text was asked for (requested_at), not from the page load, so a reload no longer restarts it - It says when an attempt failed and is retried, and when a failed text's unit was given back; admins see the error Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012vLHzh5qUhFtmJhzkcTbXL
This commit is contained in:
@@ -253,6 +253,8 @@ impl Analyzer {
|
||||
.await
|
||||
}
|
||||
Err(e) => {
|
||||
self.log_error("preview", job.id, &job.kind, &job.model, 1, "failed", e.message(), started.elapsed())
|
||||
.await;
|
||||
sqlx::query(
|
||||
"UPDATE prompt_previews SET status = 'failed', error = $2, completed_at = now() \
|
||||
WHERE id = $1 AND status = 'running'",
|
||||
@@ -322,7 +324,24 @@ impl Analyzer {
|
||||
}
|
||||
Err(e) => {
|
||||
let can_fall_back = !job.on_fallback && job.fallback_model.is_some();
|
||||
match next_step(&e, job.attempts, self.max_attempts, can_fall_back) {
|
||||
let next = next_step(&e, job.attempts, self.max_attempts, can_fall_back);
|
||||
let outcome = match next {
|
||||
Next::Retry(_) => "retry",
|
||||
Next::Fallback => "fallback",
|
||||
Next::Fail => "failed",
|
||||
};
|
||||
self.log_error(
|
||||
"analysis",
|
||||
job.id,
|
||||
&job.kind,
|
||||
job.active_model(),
|
||||
job.attempts,
|
||||
outcome,
|
||||
e.message(),
|
||||
took,
|
||||
)
|
||||
.await;
|
||||
match next {
|
||||
Next::Retry(wait) => {
|
||||
sqlx::query(
|
||||
"UPDATE analyses SET status = 'pending', error = $2, \
|
||||
@@ -424,6 +443,38 @@ impl Analyzer {
|
||||
Ok(result.rows_affected())
|
||||
}
|
||||
|
||||
/// Records a failed attempt in the admin error log (a failure here is only logged).
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) async fn log_error(
|
||||
&self,
|
||||
source: &str,
|
||||
item_id: Uuid,
|
||||
kind: &str,
|
||||
model: &str,
|
||||
attempt: i32,
|
||||
outcome: &str,
|
||||
error: &str,
|
||||
took: Duration,
|
||||
) {
|
||||
let result = sqlx::query(
|
||||
"INSERT INTO generation_errors (source, item_id, kind, model, attempt, outcome, error, seconds) \
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)",
|
||||
)
|
||||
.bind(source)
|
||||
.bind(item_id)
|
||||
.bind(kind)
|
||||
.bind(model)
|
||||
.bind(attempt)
|
||||
.bind(outcome)
|
||||
.bind(error.chars().take(2000).collect::<String>())
|
||||
.bind(i32::try_from(took.as_secs()).unwrap_or(i32::MAX))
|
||||
.execute(&self.db)
|
||||
.await;
|
||||
if let Err(e) = result {
|
||||
tracing::error!(error = %e, "writing the error log");
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolves when this attempt is no longer the current one (an admin
|
||||
/// restarted it); checked every 10 seconds, never on a database error.
|
||||
async fn superseded(&self, table: &str, id: Uuid, started_at: time::OffsetDateTime) {
|
||||
|
||||
@@ -148,7 +148,15 @@ impl Analyzer {
|
||||
}
|
||||
Err(e) => {
|
||||
let can_fall_back = !job.on_fallback && job.fallback_model.is_some();
|
||||
match next_step(&e, job.attempts, self.max_attempts, can_fall_back) {
|
||||
let next = next_step(&e, job.attempts, self.max_attempts, can_fall_back);
|
||||
let outcome = match next {
|
||||
Next::Retry(_) => "retry",
|
||||
Next::Fallback => "fallback",
|
||||
Next::Fail => "failed",
|
||||
};
|
||||
let model = active_model(&job.model, job.fallback_model.as_deref(), job.on_fallback);
|
||||
self.log_error("question", job.id, "question", model, job.attempts, outcome, e.message(), took).await;
|
||||
match next {
|
||||
Next::Retry(wait) => {
|
||||
sqlx::query(
|
||||
"UPDATE questions SET status = 'pending', error = $2, \
|
||||
|
||||
@@ -8,7 +8,7 @@ use tokio::sync::watch;
|
||||
const INTERVAL: Duration = Duration::from_secs(3600);
|
||||
|
||||
/// (what, statement) pairs; each returns the number of deleted rows.
|
||||
const STATEMENTS: [(&str, &str); 6] = [
|
||||
const STATEMENTS: [(&str, &str); 7] = [
|
||||
("sessions", "DELETE FROM sessions WHERE expires_at < now()"),
|
||||
("auth_tokens", "DELETE FROM auth_tokens WHERE expires_at < now()"),
|
||||
// Sent e-mails contain links with tokens; keep them a week for support.
|
||||
@@ -26,6 +26,7 @@ const STATEMENTS: [(&str, &str); 6] = [
|
||||
OR (accepted_at IS NOT NULL AND accepted_by IS NULL)",
|
||||
),
|
||||
("prompt_previews", "DELETE FROM prompt_previews WHERE created_at < now() - interval '30 days'"),
|
||||
("generation_errors", "DELETE FROM generation_errors WHERE created_at < now() - interval '90 days'"),
|
||||
];
|
||||
|
||||
pub async fn run_once(db: &PgPool) -> Result<Vec<(&'static str, u64)>, sqlx::Error> {
|
||||
|
||||
@@ -212,7 +212,8 @@ async fn cleanup_removes_only_expired_data(db: PgPool) {
|
||||
("emails", 1),
|
||||
("invites", 0),
|
||||
("invitations", 0),
|
||||
("prompt_previews", 0)
|
||||
("prompt_previews", 0),
|
||||
("generation_errors", 0)
|
||||
]
|
||||
);
|
||||
let (left,): (Vec<Uuid>,) =
|
||||
@@ -533,6 +534,18 @@ mod analyses {
|
||||
let (status, attempts, ..) = row(&db, id).await;
|
||||
assert_eq!((status.as_str(), attempts), ("failed", 3));
|
||||
assert_eq!(refunds(&db, s.user).await, vec![1]);
|
||||
// every failed attempt is in the admin error log
|
||||
let log: Vec<(String, i32, String, String)> = sqlx::query_as(
|
||||
"SELECT source, attempt, outcome, model FROM generation_errors WHERE item_id = $1 ORDER BY id",
|
||||
)
|
||||
.bind(id)
|
||||
.fetch_all(&db)
|
||||
.await
|
||||
.unwrap();
|
||||
let expected = |attempt: i32, outcome: &str| {
|
||||
("analysis".to_string(), attempt, outcome.to_string(), "test/model".to_string())
|
||||
};
|
||||
assert_eq!(log, vec![expected(1, "retry"), expected(2, "retry"), expected(3, "failed")]);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrator = "db::MIGRATOR")]
|
||||
@@ -780,6 +793,16 @@ mod analyses {
|
||||
let models: Vec<String> =
|
||||
fake.requests.lock().unwrap().iter().map(|b| b["model"].as_str().unwrap().to_string()).collect();
|
||||
assert_eq!(models, ["cheap/model", "cheap/model", "test/model"]);
|
||||
let outcomes: Vec<(String, String)> =
|
||||
sqlx::query_as("SELECT outcome, model FROM generation_errors WHERE item_id = $1 ORDER BY id")
|
||||
.bind(id)
|
||||
.fetch_all(&db)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
outcomes,
|
||||
[("retry".to_string(), "cheap/model".to_string()), ("fallback".to_string(), "cheap/model".to_string())]
|
||||
);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrator = "db::MIGRATOR")]
|
||||
|
||||
Reference in New Issue
Block a user