//! Writes the requested text analyses (`analyses` rows in `pending` state). //! //! The backend inserts the row and charges the credits; this job claims it //! (`running`), builds the prompt from the stored chart data, asks the model //! through OpenRouter and stores the structured answer (`done`). Temporary //! failures are retried with backoff; after the last attempt, or on a //! permanent error, the analysis is `failed` and the unit is refunded. //! //! Admin test runs of an edited prompt (`prompt_previews`) go through the //! same loop when no analysis is due: one attempt, no credits, no retries. //! //! Two lanes, each with its own slots, so a short text never waits behind //! the long ones: the quick lane writes summaries and answers questions, the //! slow lane writes the detailed analyses and forecasts (and the test runs). //! A free slow slot also takes quick work. A lane of size 0 is off, so the //! lanes can run as separate deployments; several workers may share the //! queue, rows are claimed with `FOR UPDATE SKIP LOCKED`. //! //! Fallback: when the model chosen on the admin page fails twice (or once with //! a permanent error), the default model (`fallback_model`) takes over with a //! fresh set of attempts; only when that fails too is the unit refunded. pub mod compact; pub mod openrouter; pub mod prompt; pub mod questions; use std::collections::HashSet; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use serde_json::Value; use sqlx::PgPool; use tokio::sync::{Semaphore, watch}; use uuid::Uuid; use self::openrouter::{Completion, LlmError, OpenRouter}; use self::prompt::{Kind, PROMPT_VERSION, Prompt}; use self::questions::QuestionJob; /// A `running` analysis older than this was left behind by a stopped worker. const STUCK_AFTER_MINUTES: i32 = 15; #[derive(Debug, sqlx::FromRow)] pub struct Job { pub id: Uuid, pub kind: String, pub chart_id: Option, pub synastry_id: Option, pub forecast_id: Option, pub language: String, pub model: String, pub prompt_version: String, pub attempts: i32, /// The default model, when the chosen one is another (see the module docs). pub fallback_model: Option, pub on_fallback: bool, /// When this attempt started; also its token: an admin restart clears it, /// so a superseded attempt neither writes its result nor keeps running. pub started_at: time::OffsetDateTime, } impl Job { /// The model of this attempt. pub fn active_model(&self) -> &str { active_model(&self.model, self.fallback_model.as_deref(), self.on_fallback) } } pub fn active_model<'a>(model: &'a str, fallback: Option<&'a str>, on_fallback: bool) -> &'a str { match fallback { Some(f) if on_fallback => f, _ => model, } } /// What happens after a failed attempt. #[derive(Debug, PartialEq, Eq)] pub enum Next { Retry(Duration), /// Switch to the fallback model, with a fresh set of attempts, at once. Fallback, /// Give up (and refund). Fail, } /// The chosen model gets two attempts (one after a permanent error), then the /// fallback model takes over with its own `max_attempts`. pub fn next_step(error: &LlmError, attempts: i32, max_attempts: i32, can_fall_back: bool) -> Next { if can_fall_back && (matches!(error, LlmError::Permanent(_)) || attempts >= 2) { return Next::Fallback; } match error { LlmError::Transient(_) if attempts < max_attempts => Next::Retry(backoff(attempts)), _ => Next::Fail, } } /// A test run of an edited system prompt on a chart. #[derive(Debug, sqlx::FromRow)] pub struct PreviewJob { pub id: Uuid, pub kind: String, pub chart_id: Option, pub synastry_id: Option, pub forecast_id: Option, pub language: String, pub model: String, pub system: String, } /// Which analyses a claim may take. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Lane { /// The short summary of a chart (questions are claimed separately). Quick, /// The detailed analyses and the yearly forecast. Slow, } impl Lane { fn kinds(self) -> &'static [&'static str] { match self { Lane::Quick => &["summary"], Lane::Slow => &["natal", "synastry", "forecast"], } } } /// Slots per lane; 0 turns a lane off. #[derive(Debug, Clone, Copy)] pub struct Lanes { pub quick: usize, pub slow: usize, } pub struct Analyzer { pub db: PgPool, pub llm: OpenRouter, pub max_attempts: i32, pub lanes: Lanes, /// Analyses and previews being written right now (released on shutdown). running: Mutex>, } /// Wait before the next attempt: 30 s, 2 min, 8 min, ... at most 1 hour. pub fn backoff(attempts: i32) -> Duration { let exp = attempts.clamp(1, 6).saturating_sub(1) as u32; Duration::from_secs((30 * 4u64.pow(exp)).min(3600)) } impl Analyzer { pub fn new(db: PgPool, llm: OpenRouter, max_attempts: i32, lanes: Lanes) -> Self { Self { db, llm, max_attempts, lanes, running: Mutex::default() } } /// Takes the next due analysis of the lane, if any. pub async fn claim(&self, lane: Lane) -> Result, sqlx::Error> { let job: Option = sqlx::query_as( "UPDATE analyses SET status = 'running', started_at = now(), attempts = attempts + 1 \ WHERE id = (SELECT id FROM analyses WHERE status = 'pending' AND next_attempt_at <= now() \ AND kind = ANY($1) \ ORDER BY next_attempt_at LIMIT 1 FOR UPDATE SKIP LOCKED) \ RETURNING id, kind, chart_id, synastry_id, forecast_id, language, model, prompt_version, attempts, \ fallback_model, on_fallback, started_at", ) .bind(lane.kinds()) .fetch_optional(&self.db) .await?; if let Some(job) = &job { self.running.lock().unwrap().insert(job.id); } Ok(job) } /// Builds the prompt from the stored chart or synastry. pub async fn prompt_for(&self, job: &Job) -> Result { let kind = Kind::parse(&job.kind).ok_or_else(|| LlmError::Permanent(format!("unknown kind '{}'", job.kind)))?; let db_err = |e: sqlx::Error| LlmError::Transient(format!("database: {e}")); let text = match kind { Kind::Natal | Kind::Summary => self.natal_text(job.chart_id).await?, Kind::Synastry => self.synastry_text(job.synastry_id).await?, Kind::Forecast => self.forecast_text(job.forecast_id).await?, Kind::Question => return Err(LlmError::Permanent("a question is not an analysis".into())), }; let (base, revision) = prompt::split_version(&job.prompt_version); if base != PROMPT_VERSION { tracing::warn!(id = %job.id, requested = %job.prompt_version, current = PROMPT_VERSION, "prompt version differs"); } // An admin-edited system prompt; NULL (or a missing row) means the built-in one. let system: Option = match revision { Some(id) => sqlx::query_scalar("SELECT system FROM prompt_revisions WHERE id = $1") .bind(id) .fetch_optional(&self.db) .await .map_err(db_err)? .flatten(), None => None, }; Ok(prompt::build(kind, &job.language, &text, system.as_deref())) } /// The compact text of a stored natal chart. async fn natal_text(&self, chart_id: Option) -> Result { let (chart,): (Value,) = sqlx::query_as("SELECT result FROM charts WHERE id = $1") .bind(chart_id) .fetch_optional(&self.db) .await .map_err(|e| LlmError::Transient(format!("database: {e}")))? .ok_or_else(|| LlmError::Permanent("the chart was deleted".into()))?; Ok(compact::natal(&chart)) } /// The compact text of a stored synastry with its two charts. async fn synastry_text(&self, synastry_id: Option) -> Result { let (result, chart1, chart2): (Value, Value, Value) = sqlx::query_as( "SELECT s.result, c1.result, c2.result FROM synastries s \ JOIN charts c1 ON c1.id = s.chart1_id JOIN charts c2 ON c2.id = s.chart2_id WHERE s.id = $1", ) .bind(synastry_id) .fetch_optional(&self.db) .await .map_err(|e| LlmError::Transient(format!("database: {e}")))? .ok_or_else(|| LlmError::Permanent("the synastry was deleted".into()))?; Ok(compact::synastry(&chart1, &chart2, &result)) } /// The compact text of a stored forecast with its natal chart. async fn forecast_text(&self, forecast_id: Option) -> Result { let (result, chart): (Value, Value) = sqlx::query_as( "SELECT f.result, c.result FROM forecasts f JOIN charts c ON c.id = f.chart_id WHERE f.id = $1", ) .bind(forecast_id) .fetch_optional(&self.db) .await .map_err(|e| LlmError::Transient(format!("database: {e}")))? .ok_or_else(|| LlmError::Permanent("the forecast was deleted".into()))?; Ok(compact::forecast(&chart, &result)) } /// Takes the oldest waiting prompt test run, if any. pub async fn claim_preview(&self) -> Result, sqlx::Error> { let job: Option = sqlx::query_as( "UPDATE prompt_previews SET status = 'running', started_at = now() \ WHERE id = (SELECT id FROM prompt_previews WHERE status = 'pending' \ ORDER BY created_at LIMIT 1 FOR UPDATE SKIP LOCKED) \ RETURNING id, kind, chart_id, synastry_id, forecast_id, language, model, system", ) .fetch_optional(&self.db) .await?; if let Some(job) = &job { self.running.lock().unwrap().insert(job.id); } Ok(job) } /// Writes one test run with the edited system prompt (a single attempt). pub async fn handle_preview(&self, job: PreviewJob) { let started = Instant::now(); let result = async { let kind = Kind::parse(&job.kind).ok_or_else(|| LlmError::Permanent(format!("unknown kind '{}'", job.kind)))?; let text = match kind { Kind::Synastry => self.synastry_text(job.synastry_id).await?, Kind::Forecast => self.forecast_text(job.forecast_id).await?, Kind::Natal | Kind::Summary => self.natal_text(job.chart_id).await?, Kind::Question => return Err(LlmError::Permanent("a question has no test run".into())), }; let built = prompt::build(kind, &job.language, &text, Some(&job.system)); let mut c = self.llm.complete(&job.model, &built).await?; prompt::polish(&job.language, &mut c.content); Ok::<_, LlmError>(c) } .await; let saved = match &result { Ok(c) => { sqlx::query( "UPDATE prompt_previews SET status = 'done', content = $2, model_used = $3, input_tokens = $4, \ output_tokens = $5, cost_usd = $6, completed_at = now() WHERE id = $1 AND status = 'running'", ) .bind(job.id) .bind(&c.content) .bind(&c.model) .bind(c.input_tokens) .bind(c.output_tokens) .bind(c.cost_usd) .execute(&self.db) .await } Err(e) => { self.log_error("preview", job.id, &job.kind, &job.model, 1, "failed", e.message(), started.elapsed()) .await; sqlx::query( "UPDATE prompt_previews SET status = 'failed', error = $2, completed_at = now() \ WHERE id = $1 AND status = 'running'", ) .bind(job.id) .bind(e.message()) .execute(&self.db) .await } }; match (&result, saved) { (_, Err(e)) => tracing::error!(id = %job.id, error = %e, "saving the prompt preview failed"), (Ok(c), Ok(_)) => tracing::info!( id = %job.id, model = %c.model, cost_usd = c.cost_usd, secs = started.elapsed().as_secs(), "prompt preview done" ), (Err(e), Ok(_)) => tracing::warn!(id = %job.id, error = %e.message(), "prompt preview failed"), } self.running.lock().unwrap().remove(&job.id); } /// Writes one claimed analysis and records the outcome. pub async fn handle(&self, job: Job) { let started = Instant::now(); let work = async { match self.prompt_for(&job).await { Ok(prompt) => self.llm.complete(job.active_model(), &prompt).await.map(|mut c| { prompt::polish(&job.language, &mut c.content); c }), Err(e) => Err(e), } }; let result = tokio::select! { result = work => result, () = self.superseded("analyses", job.id, job.started_at) => { tracing::info!(id = %job.id, "analysis restarted by an admin; this attempt is dropped"); return; } }; if let Err(e) = self.finish(&job, result, started.elapsed()).await { tracing::error!(id = %job.id, error = %e, "saving the analysis result failed"); } self.running.lock().unwrap().remove(&job.id); } async fn finish(&self, job: &Job, result: Result, took: Duration) -> Result<(), sqlx::Error> { match result { Ok(c) => { sqlx::query( "UPDATE analyses SET status = 'done', content = $2, model_used = $3, input_tokens = $4, \ output_tokens = $5, cost_usd = $6, error = NULL, completed_at = now() \ WHERE id = $1 AND status = 'running' AND started_at = $7", ) .bind(job.id) .bind(&c.content) .bind(&c.model) .bind(c.input_tokens) .bind(c.output_tokens) .bind(c.cost_usd) .bind(job.started_at) .execute(&self.db) .await?; tracing::info!( id = %job.id, model = %c.model, input_tokens = c.input_tokens, output_tokens = c.output_tokens, cost_usd = c.cost_usd, secs = took.as_secs(), "analysis done" ); } Err(e) => { let can_fall_back = !job.on_fallback && job.fallback_model.is_some(); let next = next_step(&e, job.attempts, self.max_attempts, can_fall_back); let outcome = match next { Next::Retry(_) => "retry", Next::Fallback => "fallback", Next::Fail => "failed", }; self.log_error( "analysis", job.id, &job.kind, job.active_model(), job.attempts, outcome, e.message(), took, ) .await; match next { Next::Retry(wait) => { sqlx::query( "UPDATE analyses SET status = 'pending', error = $2, \ next_attempt_at = now() + make_interval(secs => $3) \ WHERE id = $1 AND status = 'running' AND started_at = $4", ) .bind(job.id) .bind(e.message()) .bind(wait.as_secs_f64()) .bind(job.started_at) .execute(&self.db) .await?; tracing::warn!(id = %job.id, attempts = job.attempts, retry_in_secs = wait.as_secs(), error = %e.message(), "analysis will be retried"); } Next::Fallback => { sqlx::query( "UPDATE analyses SET status = 'pending', error = $2, on_fallback = true, attempts = 0, \ next_attempt_at = now() WHERE id = $1 AND status = 'running' AND started_at = $3", ) .bind(job.id) .bind(e.message()) .bind(job.started_at) .execute(&self.db) .await?; tracing::warn!( id = %job.id, model = %job.model, fallback = ?job.fallback_model, error = %e.message(), "the chosen model failed, the default model takes over" ); } Next::Fail => { self.fail(job.id, job.started_at, e.message()).await?; tracing::error!(id = %job.id, attempts = job.attempts, error = %e.message(), "analysis failed"); } } } } Ok(()) } /// Marks the analysis failed and gives its unit back (once). async fn fail(&self, id: Uuid, started_at: time::OffsetDateTime, message: &str) -> Result<(), sqlx::Error> { let mut tx = self.db.begin().await?; let row: Option<(Option, i32, String)> = sqlx::query_as( "UPDATE analyses SET status = 'failed', error = $2, completed_at = now() \ WHERE id = $1 AND status = 'running' AND started_at = $3 RETURNING requested_by, credits_charged, kind", ) .bind(id) .bind(message) .bind(started_at) .fetch_optional(&mut *tx) .await?; if let Some((Some(user), units, kind)) = row && units > 0 { let unit = match kind.as_str() { "synastry" => "synastry", "forecast" => "forecast", _ => "natal", }; sqlx::query( "INSERT INTO credit_ledger (id, user_id, unit, delta, reason, analysis_id, note) \ VALUES ($1, $2, $3, $4, 'refund', $5, 'analysis failed') ON CONFLICT DO NOTHING", ) .bind(Uuid::now_v7()) .bind(user) .bind(unit) .bind(units) .bind(id) .execute(&mut *tx) .await?; } tx.commit().await } /// Puts analyses left `running` by a stopped worker back in the queue. pub async fn requeue_stuck(&self) -> Result { let result = sqlx::query( "UPDATE analyses SET status = 'pending', error = 'interrupted', next_attempt_at = now() \ WHERE status = 'running' AND started_at < now() - make_interval(mins => $1)", ) .bind(STUCK_AFTER_MINUTES) .execute(&self.db) .await?; sqlx::query( "UPDATE questions SET status = 'pending', error = 'interrupted', next_attempt_at = now() \ WHERE status = 'running' AND started_at < now() - make_interval(mins => $1)", ) .bind(STUCK_AFTER_MINUTES) .execute(&self.db) .await?; // A test run is not retried: the admin can start it again. sqlx::query( "UPDATE prompt_previews SET status = 'failed', error = 'interrupted', completed_at = now() \ WHERE status = 'running' AND started_at < now() - make_interval(mins => $1)", ) .bind(STUCK_AFTER_MINUTES) .execute(&self.db) .await?; Ok(result.rows_affected()) } /// Records a failed attempt in the admin error log (a failure here is only logged). #[allow(clippy::too_many_arguments)] pub(crate) async fn log_error( &self, source: &str, item_id: Uuid, kind: &str, model: &str, attempt: i32, outcome: &str, error: &str, took: Duration, ) { let result = sqlx::query( "INSERT INTO generation_errors (source, item_id, kind, model, attempt, outcome, error, seconds) \ VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", ) .bind(source) .bind(item_id) .bind(kind) .bind(model) .bind(attempt) .bind(outcome) .bind(error.chars().take(2000).collect::()) .bind(i32::try_from(took.as_secs()).unwrap_or(i32::MAX)) .execute(&self.db) .await; if let Err(e) = result { tracing::error!(error = %e, "writing the error log"); } } /// Resolves when this attempt is no longer the current one (an admin /// restarted it); checked every 10 seconds, never on a database error. async fn superseded(&self, table: &str, id: Uuid, started_at: time::OffsetDateTime) { let query = format!("SELECT status = 'running' AND started_at = $2 FROM {table} WHERE id = $1"); loop { tokio::time::sleep(Duration::from_secs(10)).await; match sqlx::query_scalar::<_, Option>(&query).bind(id).bind(started_at).fetch_optional(&self.db).await { Ok(Some(Some(true))) => {} Ok(_) => return, Err(_) => {} } } } /// On shutdown: analyses still being written go back to `pending` at once. async fn release_running(&self) { let ids: Vec = self.running.lock().unwrap().drain().collect(); if ids.is_empty() { return; } let result = sqlx::query( "UPDATE analyses SET status = 'pending', error = 'interrupted', next_attempt_at = now(), \ attempts = greatest(attempts - 1, 0) WHERE id = ANY($1) AND status = 'running'", ) .bind(&ids) .execute(&self.db) .await; match result { Ok(r) => tracing::info!(released = r.rows_affected(), "unfinished analyses put back in the queue"), Err(e) => tracing::error!(error = %e, "releasing unfinished analyses"), } let result = sqlx::query( "UPDATE questions SET status = 'pending', error = 'interrupted', next_attempt_at = now(), \ attempts = greatest(attempts - 1, 0) WHERE id = ANY($1) AND status = 'running'", ) .bind(&ids) .execute(&self.db) .await; if let Err(e) = result { tracing::error!(error = %e, "releasing unfinished questions"); } let result = sqlx::query( "UPDATE prompt_previews SET status = 'failed', error = 'interrupted', completed_at = now() \ WHERE id = ANY($1) AND status = 'running'", ) .bind(&ids) .execute(&self.db) .await; if let Err(e) = result { tracing::error!(error = %e, "releasing unfinished prompt previews"); } } } /// Work a free slot can take, in order of preference. pub(crate) enum Work { Analysis(Job), Question(QuestionJob), Preview(PreviewJob), } impl Analyzer { /// Quick work: a question first (someone is waiting for a short answer), /// then a summary. pub(crate) async fn claim_quick(&self) -> Result, sqlx::Error> { if let Some(job) = self.claim_question().await? { return Ok(Some(Work::Question(job))); } Ok(self.claim(Lane::Quick).await?.map(Work::Analysis)) } /// Slow work: a detailed analysis, else quick work, else a test run. pub(crate) async fn claim_slow(&self) -> Result, sqlx::Error> { if let Some(job) = self.claim(Lane::Slow).await? { return Ok(Some(Work::Analysis(job))); } if let Some(work) = self.claim_quick().await? { return Ok(Some(work)); } Ok(self.claim_preview().await?.map(Work::Preview)) } async fn work(&self, work: Work) { match work { Work::Analysis(job) => self.handle(job).await, Work::Question(job) => self.handle_question(job).await, Work::Preview(job) => self.handle_preview(job).await, } } } pub async fn run(analyzer: Arc, poll: Duration, stop: watch::Receiver) { let quick = Arc::new(Semaphore::new(analyzer.lanes.quick)); let slow = Arc::new(Semaphore::new(analyzer.lanes.slow)); let mut tasks = tokio::task::JoinSet::new(); let mut listener = crate::listen(&analyzer.db, "analyses").await; let mut last_requeue: Option = None; loop { if *stop.borrow() { break; } if last_requeue.is_none_or(|t| t.elapsed() > Duration::from_secs(60)) { last_requeue = Some(Instant::now()); match analyzer.requeue_stuck().await { Ok(0) => {} Ok(n) => tracing::warn!(requeued = n, "interrupted analyses requeued"), Err(e) => tracing::error!(error = %e, "requeueing analyses"), } } // Fill the free slots: the quick lane first, then the slow one. for lane in [Lane::Quick, Lane::Slow] { let slots = if lane == Lane::Quick { &quick } else { &slow }; while let Ok(permit) = slots.clone().try_acquire_owned() { let claimed = match lane { Lane::Quick => analyzer.claim_quick().await, Lane::Slow => analyzer.claim_slow().await, }; match claimed { Ok(Some(work)) => { let analyzer = analyzer.clone(); tasks.spawn(async move { analyzer.work(work).await; drop(permit); }); } Ok(None) => break, Err(e) => { tracing::error!(error = %e, ?lane, "claiming work"); break; } } } } while tasks.try_join_next().is_some() {} tokio::select! { () = crate::notified(listener.as_mut()) => {} () = tokio::time::sleep(poll) => {} Some(_) = tasks.join_next(), if !tasks.is_empty() => {} () = crate::stopped(stop.clone()) => break, } } // Give running analyses a moment, then hand the rest back to the queue. let _ = tokio::time::timeout(Duration::from_secs(10), async { while tasks.join_next().await.is_some() {} }).await; tasks.abort_all(); analyzer.release_running().await; }