Under each prompt on /admin/promptok, "Kipróbálás" writes the summary or detailed analysis with the text in the editor (saved or not) on one of the admin's own charts. Test runs live in prompt_previews: never shown with the chart, not cached, no credits, one attempt, at most three open per admin, removed after 30 days. The worker writes them when no analysis is due. The result shows the model, tokens, cost and time, and the draft can be loaded back. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012vLHzh5qUhFtmJhzkcTbXL
404 lines
16 KiB
Rust
404 lines
16 KiB
Rust
//! Writes the requested text analyses (`analyses` rows in `pending` state).
|
|
//!
|
|
//! The backend inserts the row and charges the credits; this job claims it
|
|
//! (`running`), builds the prompt from the stored chart data, asks the model
|
|
//! through OpenRouter and stores the structured answer (`done`). Temporary
|
|
//! failures are retried with backoff; after the last attempt, or on a
|
|
//! permanent error, the analysis is `failed` and the unit is refunded.
|
|
//!
|
|
//! Admin test runs of an edited prompt (`prompt_previews`) go through the
|
|
//! same loop when no analysis is due: one attempt, no credits, no retries.
|
|
|
|
pub mod compact;
|
|
pub mod openrouter;
|
|
pub mod prompt;
|
|
|
|
use std::collections::HashSet;
|
|
use std::sync::{Arc, Mutex};
|
|
use std::time::{Duration, Instant};
|
|
|
|
use serde_json::Value;
|
|
use sqlx::PgPool;
|
|
use tokio::sync::{Semaphore, watch};
|
|
use uuid::Uuid;
|
|
|
|
use self::openrouter::{Completion, LlmError, OpenRouter};
|
|
use self::prompt::{Kind, PROMPT_VERSION, Prompt};
|
|
|
|
/// A `running` analysis older than this was left behind by a stopped worker.
|
|
const STUCK_AFTER_MINUTES: i32 = 15;
|
|
|
|
#[derive(Debug, sqlx::FromRow)]
|
|
pub struct Job {
|
|
pub id: Uuid,
|
|
pub kind: String,
|
|
pub chart_id: Option<Uuid>,
|
|
pub synastry_id: Option<Uuid>,
|
|
pub language: String,
|
|
pub model: String,
|
|
pub prompt_version: String,
|
|
pub attempts: i32,
|
|
}
|
|
|
|
/// A test run of an edited system prompt on a chart.
|
|
#[derive(Debug, sqlx::FromRow)]
|
|
pub struct PreviewJob {
|
|
pub id: Uuid,
|
|
pub kind: String,
|
|
pub chart_id: Uuid,
|
|
pub language: String,
|
|
pub model: String,
|
|
pub system: String,
|
|
}
|
|
|
|
pub struct Analyzer {
|
|
pub db: PgPool,
|
|
pub llm: OpenRouter,
|
|
pub max_attempts: i32,
|
|
pub concurrency: usize,
|
|
/// Analyses and previews being written right now (released on shutdown).
|
|
running: Mutex<HashSet<Uuid>>,
|
|
}
|
|
|
|
/// Wait before the next attempt: 30 s, 2 min, 8 min, ... at most 1 hour.
|
|
pub fn backoff(attempts: i32) -> Duration {
|
|
let exp = attempts.clamp(1, 6).saturating_sub(1) as u32;
|
|
Duration::from_secs((30 * 4u64.pow(exp)).min(3600))
|
|
}
|
|
|
|
impl Analyzer {
|
|
pub fn new(db: PgPool, llm: OpenRouter, max_attempts: i32, concurrency: usize) -> Self {
|
|
Self { db, llm, max_attempts, concurrency: concurrency.max(1), running: Mutex::default() }
|
|
}
|
|
|
|
/// Takes the next due analysis, if any.
|
|
pub async fn claim(&self) -> Result<Option<Job>, sqlx::Error> {
|
|
let job: Option<Job> = sqlx::query_as(
|
|
"UPDATE analyses SET status = 'running', started_at = now(), attempts = attempts + 1 \
|
|
WHERE id = (SELECT id FROM analyses WHERE status = 'pending' AND next_attempt_at <= now() \
|
|
ORDER BY next_attempt_at LIMIT 1 FOR UPDATE SKIP LOCKED) \
|
|
RETURNING id, kind, chart_id, synastry_id, language, model, prompt_version, attempts",
|
|
)
|
|
.fetch_optional(&self.db)
|
|
.await?;
|
|
if let Some(job) = &job {
|
|
self.running.lock().unwrap().insert(job.id);
|
|
}
|
|
Ok(job)
|
|
}
|
|
|
|
/// Builds the prompt from the stored chart or synastry.
|
|
pub async fn prompt_for(&self, job: &Job) -> Result<Prompt, LlmError> {
|
|
let kind = Kind::parse(&job.kind).ok_or_else(|| LlmError::Permanent(format!("unknown kind '{}'", job.kind)))?;
|
|
let db_err = |e: sqlx::Error| LlmError::Transient(format!("database: {e}"));
|
|
let gone = || LlmError::Permanent("the chart was deleted".into());
|
|
let text = match kind {
|
|
Kind::Natal | Kind::Summary => self.natal_text(job.chart_id).await?,
|
|
Kind::Synastry => {
|
|
let (result, chart1, chart2): (Value, Value, Value) = sqlx::query_as(
|
|
"SELECT s.result, c1.result, c2.result FROM synastries s \
|
|
JOIN charts c1 ON c1.id = s.chart1_id JOIN charts c2 ON c2.id = s.chart2_id WHERE s.id = $1",
|
|
)
|
|
.bind(job.synastry_id)
|
|
.fetch_optional(&self.db)
|
|
.await
|
|
.map_err(db_err)?
|
|
.ok_or_else(gone)?;
|
|
compact::synastry(&chart1, &chart2, &result)
|
|
}
|
|
};
|
|
let (base, revision) = prompt::split_version(&job.prompt_version);
|
|
if base != PROMPT_VERSION {
|
|
tracing::warn!(id = %job.id, requested = %job.prompt_version, current = PROMPT_VERSION, "prompt version differs");
|
|
}
|
|
// An admin-edited system prompt; NULL (or a missing row) means the built-in one.
|
|
let system: Option<String> = match revision {
|
|
Some(id) => sqlx::query_scalar("SELECT system FROM prompt_revisions WHERE id = $1")
|
|
.bind(id)
|
|
.fetch_optional(&self.db)
|
|
.await
|
|
.map_err(db_err)?
|
|
.flatten(),
|
|
None => None,
|
|
};
|
|
Ok(prompt::build(kind, &job.language, &text, system.as_deref()))
|
|
}
|
|
|
|
/// The compact text of a stored natal chart.
|
|
async fn natal_text(&self, chart_id: Option<Uuid>) -> Result<String, LlmError> {
|
|
let (chart,): (Value,) = sqlx::query_as("SELECT result FROM charts WHERE id = $1")
|
|
.bind(chart_id)
|
|
.fetch_optional(&self.db)
|
|
.await
|
|
.map_err(|e| LlmError::Transient(format!("database: {e}")))?
|
|
.ok_or_else(|| LlmError::Permanent("the chart was deleted".into()))?;
|
|
Ok(compact::natal(&chart))
|
|
}
|
|
|
|
/// Takes the oldest waiting prompt test run, if any.
|
|
pub async fn claim_preview(&self) -> Result<Option<PreviewJob>, sqlx::Error> {
|
|
let job: Option<PreviewJob> = sqlx::query_as(
|
|
"UPDATE prompt_previews SET status = 'running', started_at = now() \
|
|
WHERE id = (SELECT id FROM prompt_previews WHERE status = 'pending' \
|
|
ORDER BY created_at LIMIT 1 FOR UPDATE SKIP LOCKED) \
|
|
RETURNING id, kind, chart_id, language, model, system",
|
|
)
|
|
.fetch_optional(&self.db)
|
|
.await?;
|
|
if let Some(job) = &job {
|
|
self.running.lock().unwrap().insert(job.id);
|
|
}
|
|
Ok(job)
|
|
}
|
|
|
|
/// Writes one test run with the edited system prompt (a single attempt).
|
|
pub async fn handle_preview(&self, job: PreviewJob) {
|
|
let started = Instant::now();
|
|
let result = async {
|
|
let kind = Kind::parse(&job.kind)
|
|
.filter(|k| *k != Kind::Synastry)
|
|
.ok_or_else(|| LlmError::Permanent(format!("unknown kind '{}'", job.kind)))?;
|
|
let text = self.natal_text(Some(job.chart_id)).await?;
|
|
let built = prompt::build(kind, &job.language, &text, Some(&job.system));
|
|
let mut c = self.llm.complete(&job.model, &built).await?;
|
|
prompt::polish(&job.language, &mut c.content);
|
|
Ok::<_, LlmError>(c)
|
|
}
|
|
.await;
|
|
let saved = match &result {
|
|
Ok(c) => {
|
|
sqlx::query(
|
|
"UPDATE prompt_previews SET status = 'done', content = $2, model_used = $3, input_tokens = $4, \
|
|
output_tokens = $5, cost_usd = $6, completed_at = now() WHERE id = $1 AND status = 'running'",
|
|
)
|
|
.bind(job.id)
|
|
.bind(&c.content)
|
|
.bind(&c.model)
|
|
.bind(c.input_tokens)
|
|
.bind(c.output_tokens)
|
|
.bind(c.cost_usd)
|
|
.execute(&self.db)
|
|
.await
|
|
}
|
|
Err(e) => {
|
|
sqlx::query(
|
|
"UPDATE prompt_previews SET status = 'failed', error = $2, completed_at = now() \
|
|
WHERE id = $1 AND status = 'running'",
|
|
)
|
|
.bind(job.id)
|
|
.bind(e.message())
|
|
.execute(&self.db)
|
|
.await
|
|
}
|
|
};
|
|
match (&result, saved) {
|
|
(_, Err(e)) => tracing::error!(id = %job.id, error = %e, "saving the prompt preview failed"),
|
|
(Ok(c), Ok(_)) => tracing::info!(
|
|
id = %job.id, model = %c.model, cost_usd = c.cost_usd, secs = started.elapsed().as_secs(), "prompt preview done"
|
|
),
|
|
(Err(e), Ok(_)) => tracing::warn!(id = %job.id, error = %e.message(), "prompt preview failed"),
|
|
}
|
|
self.running.lock().unwrap().remove(&job.id);
|
|
}
|
|
|
|
/// Writes one claimed analysis and records the outcome.
|
|
pub async fn handle(&self, job: Job) {
|
|
let started = Instant::now();
|
|
let result = match self.prompt_for(&job).await {
|
|
Ok(prompt) => self.llm.complete(&job.model, &prompt).await.map(|mut c| {
|
|
prompt::polish(&job.language, &mut c.content);
|
|
c
|
|
}),
|
|
Err(e) => Err(e),
|
|
};
|
|
if let Err(e) = self.finish(&job, result, started.elapsed()).await {
|
|
tracing::error!(id = %job.id, error = %e, "saving the analysis result failed");
|
|
}
|
|
self.running.lock().unwrap().remove(&job.id);
|
|
}
|
|
|
|
async fn finish(&self, job: &Job, result: Result<Completion, LlmError>, took: Duration) -> Result<(), sqlx::Error> {
|
|
match result {
|
|
Ok(c) => {
|
|
sqlx::query(
|
|
"UPDATE analyses SET status = 'done', content = $2, model_used = $3, input_tokens = $4, \
|
|
output_tokens = $5, cost_usd = $6, error = NULL, completed_at = now() \
|
|
WHERE id = $1 AND status = 'running'",
|
|
)
|
|
.bind(job.id)
|
|
.bind(&c.content)
|
|
.bind(&c.model)
|
|
.bind(c.input_tokens)
|
|
.bind(c.output_tokens)
|
|
.bind(c.cost_usd)
|
|
.execute(&self.db)
|
|
.await?;
|
|
tracing::info!(
|
|
id = %job.id, model = %c.model, input_tokens = c.input_tokens, output_tokens = c.output_tokens,
|
|
cost_usd = c.cost_usd, secs = took.as_secs(), "analysis done"
|
|
);
|
|
}
|
|
Err(LlmError::Transient(message)) if job.attempts < self.max_attempts => {
|
|
let wait = backoff(job.attempts);
|
|
sqlx::query(
|
|
"UPDATE analyses SET status = 'pending', error = $2, \
|
|
next_attempt_at = now() + make_interval(secs => $3) WHERE id = $1 AND status = 'running'",
|
|
)
|
|
.bind(job.id)
|
|
.bind(&message)
|
|
.bind(wait.as_secs_f64())
|
|
.execute(&self.db)
|
|
.await?;
|
|
tracing::warn!(id = %job.id, attempts = job.attempts, retry_in_secs = wait.as_secs(), error = %message, "analysis will be retried");
|
|
}
|
|
Err(e) => {
|
|
self.fail(job.id, e.message()).await?;
|
|
tracing::error!(id = %job.id, attempts = job.attempts, error = %e.message(), "analysis failed");
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Marks the analysis failed and gives its unit back (once).
|
|
async fn fail(&self, id: Uuid, message: &str) -> Result<(), sqlx::Error> {
|
|
let mut tx = self.db.begin().await?;
|
|
let row: Option<(Option<Uuid>, i32, String)> = sqlx::query_as(
|
|
"UPDATE analyses SET status = 'failed', error = $2, completed_at = now() \
|
|
WHERE id = $1 AND status = 'running' RETURNING requested_by, credits_charged, kind",
|
|
)
|
|
.bind(id)
|
|
.bind(message)
|
|
.fetch_optional(&mut *tx)
|
|
.await?;
|
|
if let Some((Some(user), units, kind)) = row
|
|
&& units > 0
|
|
{
|
|
let unit = if kind == "synastry" { "synastry" } else { "natal" };
|
|
sqlx::query(
|
|
"INSERT INTO credit_ledger (id, user_id, unit, delta, reason, analysis_id, note) \
|
|
VALUES ($1, $2, $3, $4, 'refund', $5, 'analysis failed') ON CONFLICT DO NOTHING",
|
|
)
|
|
.bind(Uuid::now_v7())
|
|
.bind(user)
|
|
.bind(unit)
|
|
.bind(units)
|
|
.bind(id)
|
|
.execute(&mut *tx)
|
|
.await?;
|
|
}
|
|
tx.commit().await
|
|
}
|
|
|
|
/// Puts analyses left `running` by a stopped worker back in the queue.
|
|
pub async fn requeue_stuck(&self) -> Result<u64, sqlx::Error> {
|
|
let result = sqlx::query(
|
|
"UPDATE analyses SET status = 'pending', error = 'interrupted', next_attempt_at = now() \
|
|
WHERE status = 'running' AND started_at < now() - make_interval(mins => $1)",
|
|
)
|
|
.bind(STUCK_AFTER_MINUTES)
|
|
.execute(&self.db)
|
|
.await?;
|
|
// A test run is not retried: the admin can start it again.
|
|
sqlx::query(
|
|
"UPDATE prompt_previews SET status = 'failed', error = 'interrupted', completed_at = now() \
|
|
WHERE status = 'running' AND started_at < now() - make_interval(mins => $1)",
|
|
)
|
|
.bind(STUCK_AFTER_MINUTES)
|
|
.execute(&self.db)
|
|
.await?;
|
|
Ok(result.rows_affected())
|
|
}
|
|
|
|
/// On shutdown: analyses still being written go back to `pending` at once.
|
|
async fn release_running(&self) {
|
|
let ids: Vec<Uuid> = self.running.lock().unwrap().drain().collect();
|
|
if ids.is_empty() {
|
|
return;
|
|
}
|
|
let result = sqlx::query(
|
|
"UPDATE analyses SET status = 'pending', error = 'interrupted', next_attempt_at = now(), \
|
|
attempts = greatest(attempts - 1, 0) WHERE id = ANY($1) AND status = 'running'",
|
|
)
|
|
.bind(&ids)
|
|
.execute(&self.db)
|
|
.await;
|
|
match result {
|
|
Ok(r) => tracing::info!(released = r.rows_affected(), "unfinished analyses put back in the queue"),
|
|
Err(e) => tracing::error!(error = %e, "releasing unfinished analyses"),
|
|
}
|
|
let result = sqlx::query(
|
|
"UPDATE prompt_previews SET status = 'failed', error = 'interrupted', completed_at = now() \
|
|
WHERE id = ANY($1) AND status = 'running'",
|
|
)
|
|
.bind(&ids)
|
|
.execute(&self.db)
|
|
.await;
|
|
if let Err(e) = result {
|
|
tracing::error!(error = %e, "releasing unfinished prompt previews");
|
|
}
|
|
}
|
|
}
|
|
|
|
pub async fn run(analyzer: Arc<Analyzer>, poll: Duration, stop: watch::Receiver<bool>) {
|
|
let slots = Arc::new(Semaphore::new(analyzer.concurrency));
|
|
let mut tasks = tokio::task::JoinSet::new();
|
|
let mut listener = crate::listen(&analyzer.db, "analyses").await;
|
|
let mut last_requeue: Option<Instant> = None;
|
|
loop {
|
|
if *stop.borrow() {
|
|
break;
|
|
}
|
|
if last_requeue.is_none_or(|t| t.elapsed() > Duration::from_secs(60)) {
|
|
last_requeue = Some(Instant::now());
|
|
match analyzer.requeue_stuck().await {
|
|
Ok(0) => {}
|
|
Ok(n) => tracing::warn!(requeued = n, "interrupted analyses requeued"),
|
|
Err(e) => tracing::error!(error = %e, "requeueing analyses"),
|
|
}
|
|
}
|
|
// Fill the free slots.
|
|
while let Ok(permit) = slots.clone().try_acquire_owned() {
|
|
match analyzer.claim().await {
|
|
Ok(Some(job)) => {
|
|
let analyzer = analyzer.clone();
|
|
tasks.spawn(async move {
|
|
analyzer.handle(job).await;
|
|
drop(permit);
|
|
});
|
|
}
|
|
// No analysis is due: a prompt test run, if one waits.
|
|
Ok(None) => match analyzer.claim_preview().await {
|
|
Ok(Some(job)) => {
|
|
let analyzer = analyzer.clone();
|
|
tasks.spawn(async move {
|
|
analyzer.handle_preview(job).await;
|
|
drop(permit);
|
|
});
|
|
}
|
|
Ok(None) => break,
|
|
Err(e) => {
|
|
tracing::error!(error = %e, "claiming a prompt preview");
|
|
break;
|
|
}
|
|
},
|
|
Err(e) => {
|
|
tracing::error!(error = %e, "claiming an analysis");
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
while tasks.try_join_next().is_some() {}
|
|
|
|
tokio::select! {
|
|
() = crate::notified(listener.as_mut()) => {}
|
|
() = tokio::time::sleep(poll) => {}
|
|
Some(_) = tasks.join_next(), if !tasks.is_empty() => {}
|
|
() = crate::stopped(stop.clone()) => break,
|
|
}
|
|
}
|
|
// Give running analyses a moment, then hand the rest back to the queue.
|
|
let _ = tokio::time::timeout(Duration::from_secs(10), async { while tasks.join_next().await.is_some() {} }).await;
|
|
tasks.abort_all();
|
|
analyzer.release_running().await;
|
|
}
|