Try draft prompts on own charts before saving
CI / test (push) Successful in 46s
CI / build (push) Successful in 4m16s
CI / deploy (push) Skipped

Under each prompt on /admin/promptok, "Kipróbálás" writes the summary or
detailed analysis with the text in the editor (saved or not) on one of the
admin's own charts. Test runs live in prompt_previews: never shown with the
chart, not cached, no credits, one attempt, at most three open per admin,
removed after 30 days. The worker writes them when no analysis is due. The
result shows the model, tokens, cost and time, and the draft can be loaded back.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012vLHzh5qUhFtmJhzkcTbXL
This commit is contained in:
feco
2026-11-17 14:41:12 +00:00
co-authored by Claude Opus 5.5
parent 86d5dc5503
commit b77dafdb26
13 changed files with 697 additions and 24 deletions
+1
View File
@@ -27,6 +27,7 @@ Háttérmunkák a közös PostgreSQL-adatbázis fölött: e-mail-küldés, szöv
- a rövid összefoglalónál (`summary`) csak `title` és egy 3–4 mondatos `summary`.
A worker a választ maga is ellenőrzi a sémán. Magyar szövegben az egyenes idézőjelpárokat („…”) alakra cseréli.
- Ha nincs esedékes elemzés, a worker az admin próbafuttatásait írja (`prompt_previews`): a mentetlen system prompttal, egyetlen próbálkozással, egység nélkül. 30 nap után a takarítás törli őket.
- Eltárolja a ténylegesen válaszoló modellt (`model_used`; az `openrouter/auto` mögött ez más lehet), a tokenszámot és a költséget (`cost_usd`).
- **Átmeneti hiba** (hálózat, 429, 5xx, elfogyott OpenRouter-egyenleg, hibás vagy csonka válasz): újrapróbálja 30 mp, 2 perc, majd 8 perc múlva.
- **Végleges hiba** (például ismeretlen modell), vagy ha elfogyott a `MAX_ANALYSIS_ATTEMPTS` próbálkozás: az elemzés `failed` lesz, és az egység visszajár (`refund`, elemzésenként legfeljebb egyszer).
+126 -11
View File
@@ -5,6 +5,9 @@
//! through OpenRouter and stores the structured answer (`done`). Temporary
//! failures are retried with backoff; after the last attempt, or on a
//! permanent error, the analysis is `failed` and the unit is refunded.
//!
//! Admin test runs of an edited prompt (`prompt_previews`) go through the
//! same loop when no analysis is due: one attempt, no credits, no retries.
pub mod compact;
pub mod openrouter;
@@ -37,12 +40,23 @@ pub struct Job {
pub attempts: i32,
}
/// A test run of an edited system prompt on a chart.
#[derive(Debug, sqlx::FromRow)]
pub struct PreviewJob {
pub id: Uuid,
pub kind: String,
pub chart_id: Uuid,
pub language: String,
pub model: String,
pub system: String,
}
pub struct Analyzer {
pub db: PgPool,
pub llm: OpenRouter,
pub max_attempts: i32,
pub concurrency: usize,
/// Analyses being written right now (put back to `pending` on shutdown).
/// Analyses and previews being written right now (released on shutdown).
running: Mutex<HashSet<Uuid>>,
}
@@ -79,15 +93,7 @@ impl Analyzer {
let db_err = |e: sqlx::Error| LlmError::Transient(format!("database: {e}"));
let gone = || LlmError::Permanent("the chart was deleted".into());
let text = match kind {
Kind::Natal | Kind::Summary => {
let (chart,): (Value,) = sqlx::query_as("SELECT result FROM charts WHERE id = $1")
.bind(job.chart_id)
.fetch_optional(&self.db)
.await
.map_err(db_err)?
.ok_or_else(gone)?;
compact::natal(&chart)
}
Kind::Natal | Kind::Summary => self.natal_text(job.chart_id).await?,
Kind::Synastry => {
let (result, chart1, chart2): (Value, Value, Value) = sqlx::query_as(
"SELECT s.result, c1.result, c2.result FROM synastries s \
@@ -118,6 +124,83 @@ impl Analyzer {
Ok(prompt::build(kind, &job.language, &text, system.as_deref()))
}
/// The compact text of a stored natal chart.
async fn natal_text(&self, chart_id: Option<Uuid>) -> Result<String, LlmError> {
let (chart,): (Value,) = sqlx::query_as("SELECT result FROM charts WHERE id = $1")
.bind(chart_id)
.fetch_optional(&self.db)
.await
.map_err(|e| LlmError::Transient(format!("database: {e}")))?
.ok_or_else(|| LlmError::Permanent("the chart was deleted".into()))?;
Ok(compact::natal(&chart))
}
/// Takes the oldest waiting prompt test run, if any.
pub async fn claim_preview(&self) -> Result<Option<PreviewJob>, sqlx::Error> {
let job: Option<PreviewJob> = sqlx::query_as(
"UPDATE prompt_previews SET status = 'running', started_at = now() \
WHERE id = (SELECT id FROM prompt_previews WHERE status = 'pending' \
ORDER BY created_at LIMIT 1 FOR UPDATE SKIP LOCKED) \
RETURNING id, kind, chart_id, language, model, system",
)
.fetch_optional(&self.db)
.await?;
if let Some(job) = &job {
self.running.lock().unwrap().insert(job.id);
}
Ok(job)
}
/// Writes one test run with the edited system prompt (a single attempt).
pub async fn handle_preview(&self, job: PreviewJob) {
let started = Instant::now();
let result = async {
let kind = Kind::parse(&job.kind)
.filter(|k| *k != Kind::Synastry)
.ok_or_else(|| LlmError::Permanent(format!("unknown kind '{}'", job.kind)))?;
let text = self.natal_text(Some(job.chart_id)).await?;
let built = prompt::build(kind, &job.language, &text, Some(&job.system));
let mut c = self.llm.complete(&job.model, &built).await?;
prompt::polish(&job.language, &mut c.content);
Ok::<_, LlmError>(c)
}
.await;
let saved = match &result {
Ok(c) => {
sqlx::query(
"UPDATE prompt_previews SET status = 'done', content = $2, model_used = $3, input_tokens = $4, \
output_tokens = $5, cost_usd = $6, completed_at = now() WHERE id = $1 AND status = 'running'",
)
.bind(job.id)
.bind(&c.content)
.bind(&c.model)
.bind(c.input_tokens)
.bind(c.output_tokens)
.bind(c.cost_usd)
.execute(&self.db)
.await
}
Err(e) => {
sqlx::query(
"UPDATE prompt_previews SET status = 'failed', error = $2, completed_at = now() \
WHERE id = $1 AND status = 'running'",
)
.bind(job.id)
.bind(e.message())
.execute(&self.db)
.await
}
};
match (&result, saved) {
(_, Err(e)) => tracing::error!(id = %job.id, error = %e, "saving the prompt preview failed"),
(Ok(c), Ok(_)) => tracing::info!(
id = %job.id, model = %c.model, cost_usd = c.cost_usd, secs = started.elapsed().as_secs(), "prompt preview done"
),
(Err(e), Ok(_)) => tracing::warn!(id = %job.id, error = %e.message(), "prompt preview failed"),
}
self.running.lock().unwrap().remove(&job.id);
}
/// Writes one claimed analysis and records the outcome.
pub async fn handle(&self, job: Job) {
let started = Instant::now();
@@ -215,6 +298,14 @@ impl Analyzer {
.bind(STUCK_AFTER_MINUTES)
.execute(&self.db)
.await?;
// A test run is not retried: the admin can start it again.
sqlx::query(
"UPDATE prompt_previews SET status = 'failed', error = 'interrupted', completed_at = now() \
WHERE status = 'running' AND started_at < now() - make_interval(mins => $1)",
)
.bind(STUCK_AFTER_MINUTES)
.execute(&self.db)
.await?;
Ok(result.rows_affected())
}
@@ -235,6 +326,16 @@ impl Analyzer {
Ok(r) => tracing::info!(released = r.rows_affected(), "unfinished analyses put back in the queue"),
Err(e) => tracing::error!(error = %e, "releasing unfinished analyses"),
}
let result = sqlx::query(
"UPDATE prompt_previews SET status = 'failed', error = 'interrupted', completed_at = now() \
WHERE id = ANY($1) AND status = 'running'",
)
.bind(&ids)
.execute(&self.db)
.await;
if let Err(e) = result {
tracing::error!(error = %e, "releasing unfinished prompt previews");
}
}
}
@@ -265,7 +366,21 @@ pub async fn run(analyzer: Arc<Analyzer>, poll: Duration, stop: watch::Receiver<
drop(permit);
});
}
Ok(None) => break,
// No analysis is due: a prompt test run, if one waits.
Ok(None) => match analyzer.claim_preview().await {
Ok(Some(job)) => {
let analyzer = analyzer.clone();
tasks.spawn(async move {
analyzer.handle_preview(job).await;
drop(permit);
});
}
Ok(None) => break,
Err(e) => {
tracing::error!(error = %e, "claiming a prompt preview");
break;
}
},
Err(e) => {
tracing::error!(error = %e, "claiming an analysis");
break;
+2 -1
View File
@@ -8,7 +8,7 @@ use tokio::sync::watch;
const INTERVAL: Duration = Duration::from_secs(3600);
/// (what, statement) pairs; each returns the number of deleted rows.
const STATEMENTS: [(&str, &str); 4] = [
const STATEMENTS: [(&str, &str); 5] = [
("sessions", "DELETE FROM sessions WHERE expires_at < now()"),
("auth_tokens", "DELETE FROM auth_tokens WHERE expires_at < now()"),
// Sent e-mails contain links with tokens; keep them a week for support.
@@ -18,6 +18,7 @@ const STATEMENTS: [(&str, &str); 4] = [
OR failed_at < now() - interval '30 days'",
),
("invites", "DELETE FROM synastry_invites WHERE accepted_at IS NULL AND expires_at < now() - interval '30 days'"),
("prompt_previews", "DELETE FROM prompt_previews WHERE created_at < now() - interval '30 days'"),
];
pub async fn run_once(db: &PgPool) -> Result<Vec<(&'static str, u64)>, sqlx::Error> {
+76 -1
View File
@@ -175,7 +175,10 @@ async fn cleanup_removes_only_expired_data(db: PgPool) {
sqlx::query("UPDATE email_outbox SET sent_at = now() WHERE id = $1").bind(recent).execute(&db).await.unwrap();
let deleted = crate::cleanup::run_once(&db).await.unwrap();
assert_eq!(deleted, vec![("sessions", 1), ("auth_tokens", 0), ("emails", 1), ("invites", 0)]);
assert_eq!(
deleted,
vec![("sessions", 1), ("auth_tokens", 0), ("emails", 1), ("invites", 0), ("prompt_previews", 0)]
);
let (left,): (Vec<Uuid>,) =
sqlx::query_as("SELECT array_agg(id ORDER BY id) FROM email_outbox").fetch_one(&db).await.unwrap();
assert_eq!(left, vec![recent, unsent]);
@@ -398,6 +401,78 @@ mod analyses {
assert!(requests[0]["messages"][1]["content"].as_str().unwrap().contains("- personality: "));
}
#[sqlx::test(migrator = "db::MIGRATOR")]
async fn prompt_previews_run_after_analyses(db: PgPool) {
let s = setup(&db).await;
let preview = Uuid::now_v7();
sqlx::query(
"INSERT INTO prompt_previews (id, kind, chart_id, language, model, system, created_by) \
VALUES ($1, 'summary', $2, 'hu', 'test/summary', 'Próba szabályok.', $3)",
)
.bind(preview)
.bind(s.chart)
.bind(s.user)
.execute(&db)
.await
.unwrap();
let analysis = request_analysis(&db, &s).await;
let (fake, url) = fake_openrouter(vec![
reply(&content(Kind::Natal)),
reply(&json!({"title": "Érzékeny mag", "summary": "Rövid összefoglaló szöveg."})),
(500, json!({})),
])
.await;
let a = analyzer(&db, &url);
// the analysis goes first; the preview only when nothing else is due
let job = a.claim().await.unwrap().expect("the analysis");
assert_eq!(job.id, analysis);
a.handle(job).await;
assert!(a.claim().await.unwrap().is_none());
let job = a.claim_preview().await.unwrap().expect("the preview");
a.handle_preview(job).await;
let (status, title, model, credits): (String, Option<String>, Option<String>, i64) = sqlx::query_as(
"SELECT status, content->>'title', model_used, \
(SELECT count(*) FROM credit_ledger WHERE user_id = p.created_by) \
FROM prompt_previews p WHERE id = $1",
)
.bind(preview)
.fetch_one(&db)
.await
.unwrap();
assert_eq!((status.as_str(), title.is_some(), model.as_deref()), ("done", true, Some("test/model-2026")));
let before = credits;
{
let requests = fake.requests.lock().unwrap();
assert_eq!(requests[1]["model"], "test/summary");
assert_eq!(requests[1]["messages"][0]["content"], "Próba szabályok.");
}
// one attempt only: an error fails the preview at once, without refunds
let second = Uuid::now_v7();
sqlx::query(
"INSERT INTO prompt_previews (id, kind, chart_id, language, model, system, created_by) \
VALUES ($1, 'natal', $2, 'hu', 'test/model', 'Másik.', $3)",
)
.bind(second)
.bind(s.chart)
.bind(s.user)
.execute(&db)
.await
.unwrap();
a.handle_preview(a.claim_preview().await.unwrap().unwrap()).await;
let (status, error, credits): (String, Option<String>, i64) = sqlx::query_as(
"SELECT status, error, (SELECT count(*) FROM credit_ledger WHERE user_id = p.created_by) \
FROM prompt_previews p WHERE id = $1",
)
.bind(second)
.fetch_one(&db)
.await
.unwrap();
assert_eq!((status.as_str(), error.is_some(), credits), ("failed", true, before));
assert!(a.claim_preview().await.unwrap().is_none());
}
#[sqlx::test(migrator = "db::MIGRATOR")]
async fn retries_bad_answers_then_fails_and_refunds(db: PgPool) {
let s = setup(&db).await;