Error log, retry notice, a waiting counter that keeps counting
CI / test (push) Successful in 1m8s
CI / build (push) Successful in 5m13s
CI / deploy (push) Skipped

- generation_errors: every failed attempt (model, attempt, outcome, duration); admin
  Hibanapló page and GET /v1/admin/errors; kept 90 days
- The waiting page counts from when the text was asked for (requested_at), not from
  the page load, so a reload no longer restarts it
- It says when an attempt failed and is retried, and when a failed text's unit was
  given back; admins see the error

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012vLHzh5qUhFtmJhzkcTbXL
This commit is contained in:
feco
2026-11-18 21:47:37 +00:00
co-authored by Claude Opus 5.5
parent cf2fb9d454
commit 02e4e225f0
22 changed files with 569 additions and 25 deletions
+10 -2
View File
@@ -49,11 +49,16 @@ pub struct AnalysisRow {
pub credits_charged: i32,
pub created_at: OffsetDateTime,
pub completed_at: Option<OffsetDateTime>,
/// Attempts so far; above 1 the page says it is being tried again.
pub attempts: i32,
pub next_attempt_at: OffsetDateTime,
/// When the current text was asked for (set again by a regeneration).
pub requested_at: OffsetDateTime,
}
const ANALYSIS_COLUMNS: &str = "id, kind, chart_id, synastry_id, forecast_id, language, prompt_version, status, \
content, error, \
credits_charged, created_at, completed_at";
credits_charged, created_at, completed_at, attempts, next_attempt_at, requested_at";
impl AnalysisRow {
pub fn view(&self) -> Value {
@@ -71,6 +76,9 @@ impl AnalysisRow {
"credits_charged": self.credits_charged,
"created_at": rfc3339(self.created_at),
"completed_at": rfc3339_opt(self.completed_at),
"attempts": self.attempts,
"next_attempt_at": rfc3339(self.next_attempt_at),
"requested_at": rfc3339(self.requested_at),
})
}
}
@@ -391,7 +399,7 @@ pub async fn regenerate(
"UPDATE analyses SET status = 'pending', content = NULL, error = NULL, model_used = NULL, \
input_tokens = NULL, output_tokens = NULL, cost_usd = NULL, attempts = 0, next_attempt_at = now(), \
started_at = NULL, completed_at = NULL, model = $2, prompt_version = $3, cache_key = $4, \
fallback_model = $5, on_fallback = false \
fallback_model = $5, on_fallback = false, requested_at = now() \
WHERE id = $1 AND status IN ('done', 'failed') RETURNING {ANALYSIS_COLUMNS}"
))
.bind(id)
+73
View File
@@ -0,0 +1,73 @@
//! Admin: the error log of the analyses, answers and prompt test runs (every
//! failed attempt, written by the worker), newest first.
use axum::Json;
use axum::extract::{Query, State};
use axum_extra::extract::WithRejection;
use serde::Deserialize;
use serde_json::{Value, json};
use time::OffsetDateTime;
use uuid::Uuid;
use super::St;
use crate::auth::AdminUser;
use crate::error::{ApiError, ApiResult};
use crate::views::rfc3339;
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ErrorQuery {
limit: Option<i64>,
offset: Option<i64>,
/// `analysis`, `question` or `preview`.
source: Option<String>,
/// `retry`, `fallback` or `failed`.
outcome: Option<String>,
}
type Row = (i64, OffsetDateTime, String, Uuid, String, String, i32, String, String, Option<i32>, Option<String>);
pub async fn list(
State(state): St,
_admin: AdminUser,
WithRejection(Query(query), _): WithRejection<Query<ErrorQuery>, ApiError>,
) -> ApiResult<Json<Value>> {
let limit = query.limit.unwrap_or(100).clamp(1, 500);
let offset = query.offset.unwrap_or(0).max(0);
// Whose text it was: the requester of the analysis or the asker.
let rows: Vec<Row> = sqlx::query_as(
"SELECT e.id, e.created_at, e.source, e.item_id, e.kind, e.model, e.attempt, e.outcome, e.error, e.seconds, \
u.email \
FROM generation_errors e \
LEFT JOIN analyses a ON e.source = 'analysis' AND a.id = e.item_id \
LEFT JOIN questions q ON e.source = 'question' AND q.id = e.item_id \
LEFT JOIN prompt_previews p ON e.source = 'preview' AND p.id = e.item_id \
LEFT JOIN users u ON u.id = coalesce(a.requested_by, q.asked_by, p.created_by) \
WHERE ($1::text IS NULL OR e.source = $1) AND ($2::text IS NULL OR e.outcome = $2) \
ORDER BY e.id DESC LIMIT $3 OFFSET $4",
)
.bind(&query.source)
.bind(&query.outcome)
.bind(limit)
.bind(offset)
.fetch_all(&state.db)
.await?;
let (total, last_day): (i64, i64) = sqlx::query_as(
"SELECT count(*), count(*) FILTER (WHERE created_at > now() - interval '1 day') FROM generation_errors \
WHERE ($1::text IS NULL OR source = $1) AND ($2::text IS NULL OR outcome = $2)",
)
.bind(&query.source)
.bind(&query.outcome)
.fetch_one(&state.db)
.await?;
let errors: Vec<Value> = rows
.into_iter()
.map(|(id, at, source, item, kind, model, attempt, outcome, error, seconds, email)| {
json!({
"id": id, "created_at": rfc3339(at), "source": source, "item_id": item, "kind": kind, "model": model,
"attempt": attempt, "outcome": outcome, "error": error, "seconds": seconds, "user": email,
})
})
.collect();
Ok(Json(json!({ "errors": errors, "total": total, "last_day": last_day, "limit": limit, "offset": offset })))
}
+3 -2
View File
@@ -109,7 +109,8 @@ pub async fn restart(
let (model, fallback, prompt_version) = super::questions::model_and_version(&state, &language).await?;
sqlx::query(
"UPDATE questions SET status = 'pending', model = $2, fallback_model = $3, on_fallback = false, \
prompt_version = $4, attempts = 0, next_attempt_at = now(), started_at = NULL, error = NULL \
prompt_version = $4, attempts = 0, next_attempt_at = now(), started_at = NULL, error = NULL, \
requested_at = now() \
WHERE id = $1 AND status IN ('pending', 'running')",
)
.bind(id)
@@ -127,7 +128,7 @@ pub async fn restart(
sqlx::query(
"UPDATE analyses SET status = 'pending', model = $2, fallback_model = $3, on_fallback = false, \
cache_key = $4, prompt_version = $5, attempts = 0, next_attempt_at = now(), started_at = NULL, \
error = NULL \
error = NULL, requested_at = now() \
WHERE id = $1 AND status IN ('pending', 'running')",
)
.bind(id)
+2
View File
@@ -10,6 +10,7 @@ mod admin;
mod analyses;
mod auth;
mod charts;
mod errors;
mod forecasts;
mod invitations;
mod jobs;
@@ -117,6 +118,7 @@ pub fn router(state: Arc<AppState>) -> Router {
.route("/v1/admin/prompts/{kind}/{language}", axum::routing::put(prompts::save))
.route("/v1/admin/prompts/{kind}/{language}/preview", post(prompts::start_preview))
.route("/v1/admin/previews/{id}", get(prompts::get_preview))
.route("/v1/admin/errors", get(errors::list))
.route("/v1/admin/jobs", get(jobs::list))
.route("/v1/admin/jobs/restart", post(jobs::restart))
.route("/v1/admin/models", get(models::list))
+10 -3
View File
@@ -39,10 +39,13 @@ pub struct QuestionRow {
pub credits_charged: i32,
pub created_at: OffsetDateTime,
pub completed_at: Option<OffsetDateTime>,
pub attempts: i32,
pub next_attempt_at: OffsetDateTime,
pub requested_at: OffsetDateTime,
}
const QUESTION_COLUMNS: &str =
"id, analysis_id, question, status, answer, error, credits_charged, created_at, completed_at";
const QUESTION_COLUMNS: &str = "id, analysis_id, question, status, answer, error, credits_charged, created_at, \
completed_at, attempts, next_attempt_at, requested_at";
impl QuestionRow {
pub fn view(&self) -> Value {
@@ -56,6 +59,9 @@ impl QuestionRow {
"credits_charged": self.credits_charged,
"created_at": rfc3339(self.created_at),
"completed_at": rfc3339_opt(self.completed_at),
"attempts": self.attempts,
"next_attempt_at": rfc3339(self.next_attempt_at),
"requested_at": rfc3339(self.requested_at),
})
}
}
@@ -191,7 +197,8 @@ pub async fn regenerate(
let row: QuestionRow = sqlx::query_as(&format!(
"UPDATE questions SET status = 'pending', answer = NULL, error = NULL, model_used = NULL, input_tokens = NULL, \
output_tokens = NULL, cost_usd = NULL, attempts = 0, next_attempt_at = now(), started_at = NULL, \
completed_at = NULL, model = $2, prompt_version = $3, fallback_model = $4, on_fallback = false \
completed_at = NULL, model = $2, prompt_version = $3, fallback_model = $4, on_fallback = false, \
requested_at = now() \
WHERE id = $1 AND status IN ('done', 'failed') RETURNING {QUESTION_COLUMNS}"
))
.bind(id)