2026-11-15 12:18:40 +00:00
|
|
|
//! Writes the requested text analyses (`analyses` rows in `pending` state).
|
|
|
|
|
//!
|
|
|
|
|
//! The backend inserts the row and charges the credits; this job claims it
|
|
|
|
|
//! (`running`), builds the prompt from the stored chart data, asks the model
|
|
|
|
|
//! through OpenRouter and stores the structured answer (`done`). Temporary
|
|
|
|
|
//! failures are retried with backoff; after the last attempt, or on a
|
2026-11-16 17:39:15 +00:00
|
|
|
//! permanent error, the analysis is `failed` and the unit is refunded.
|
2026-11-17 14:41:12 +00:00
|
|
|
//!
|
|
|
|
|
//! Admin test runs of an edited prompt (`prompt_previews`) go through the
|
|
|
|
|
//! same loop when no analysis is due: one attempt, no credits, no retries.
|
2026-11-18 21:11:08 +00:00
|
|
|
//!
|
2026-11-19 06:40:09 +00:00
|
|
|
//! Two lanes, each with its own slots, so a short text never waits behind
|
|
|
|
|
//! the long ones: the quick lane writes summaries and answers questions, the
|
|
|
|
|
//! slow lane writes the detailed analyses and forecasts (and the test runs).
|
|
|
|
|
//! A free slow slot also takes quick work. A lane of size 0 is off, so the
|
|
|
|
|
//! lanes can run as separate deployments; several workers may share the
|
|
|
|
|
//! queue, rows are claimed with `FOR UPDATE SKIP LOCKED`.
|
|
|
|
|
//!
|
2026-11-18 21:11:08 +00:00
|
|
|
//! Fallback: when the model chosen on the admin page fails twice (or once with
|
|
|
|
|
//! a permanent error), the default model (`fallback_model`) takes over with a
|
|
|
|
|
//! fresh set of attempts; only when that fails too is the unit refunded.
|
2026-11-15 12:18:40 +00:00
|
|
|
|
|
|
|
|
pub mod compact;
|
|
|
|
|
pub mod openrouter;
|
|
|
|
|
pub mod prompt;
|
2026-11-18 18:44:20 +00:00
|
|
|
pub mod questions;
|
2026-11-15 12:18:40 +00:00
|
|
|
|
|
|
|
|
use std::collections::HashSet;
|
|
|
|
|
use std::sync::{Arc, Mutex};
|
|
|
|
|
use std::time::{Duration, Instant};
|
|
|
|
|
|
|
|
|
|
use serde_json::Value;
|
|
|
|
|
use sqlx::PgPool;
|
|
|
|
|
use tokio::sync::{Semaphore, watch};
|
|
|
|
|
use uuid::Uuid;
|
|
|
|
|
|
|
|
|
|
use self::openrouter::{Completion, LlmError, OpenRouter};
|
|
|
|
|
use self::prompt::{Kind, PROMPT_VERSION, Prompt};
|
2026-11-19 06:40:09 +00:00
|
|
|
use self::questions::QuestionJob;
|
2026-11-15 12:18:40 +00:00
|
|
|
|
|
|
|
|
/// A `running` analysis older than this was left behind by a stopped worker.
|
|
|
|
|
const STUCK_AFTER_MINUTES: i32 = 15;
|
|
|
|
|
|
|
|
|
|
#[derive(Debug, sqlx::FromRow)]
|
|
|
|
|
pub struct Job {
|
|
|
|
|
pub id: Uuid,
|
|
|
|
|
pub kind: String,
|
|
|
|
|
pub chart_id: Option<Uuid>,
|
|
|
|
|
pub synastry_id: Option<Uuid>,
|
2026-11-18 08:19:13 +00:00
|
|
|
pub forecast_id: Option<Uuid>,
|
2026-11-15 12:18:40 +00:00
|
|
|
pub language: String,
|
|
|
|
|
pub model: String,
|
|
|
|
|
pub prompt_version: String,
|
|
|
|
|
pub attempts: i32,
|
2026-11-18 21:11:08 +00:00
|
|
|
/// The default model, when the chosen one is another (see the module docs).
|
|
|
|
|
pub fallback_model: Option<String>,
|
|
|
|
|
pub on_fallback: bool,
|
2026-11-18 21:31:09 +00:00
|
|
|
/// When this attempt started; also its token: an admin restart clears it,
|
|
|
|
|
/// so a superseded attempt neither writes its result nor keeps running.
|
|
|
|
|
pub started_at: time::OffsetDateTime,
|
2026-11-18 21:11:08 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
impl Job {
|
|
|
|
|
/// The model of this attempt.
|
|
|
|
|
pub fn active_model(&self) -> &str {
|
|
|
|
|
active_model(&self.model, self.fallback_model.as_deref(), self.on_fallback)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub fn active_model<'a>(model: &'a str, fallback: Option<&'a str>, on_fallback: bool) -> &'a str {
|
|
|
|
|
match fallback {
|
|
|
|
|
Some(f) if on_fallback => f,
|
|
|
|
|
_ => model,
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// What happens after a failed attempt.
|
|
|
|
|
#[derive(Debug, PartialEq, Eq)]
|
|
|
|
|
pub enum Next {
|
|
|
|
|
Retry(Duration),
|
|
|
|
|
/// Switch to the fallback model, with a fresh set of attempts, at once.
|
|
|
|
|
Fallback,
|
|
|
|
|
/// Give up (and refund).
|
|
|
|
|
Fail,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// The chosen model gets two attempts (one after a permanent error), then the
|
|
|
|
|
/// fallback model takes over with its own `max_attempts`.
|
|
|
|
|
pub fn next_step(error: &LlmError, attempts: i32, max_attempts: i32, can_fall_back: bool) -> Next {
|
|
|
|
|
if can_fall_back && (matches!(error, LlmError::Permanent(_)) || attempts >= 2) {
|
|
|
|
|
return Next::Fallback;
|
|
|
|
|
}
|
|
|
|
|
match error {
|
|
|
|
|
LlmError::Transient(_) if attempts < max_attempts => Next::Retry(backoff(attempts)),
|
|
|
|
|
_ => Next::Fail,
|
|
|
|
|
}
|
2026-11-15 12:18:40 +00:00
|
|
|
}
|
|
|
|
|
|
2026-11-17 14:41:12 +00:00
|
|
|
/// A test run of an edited system prompt on a chart.
|
|
|
|
|
#[derive(Debug, sqlx::FromRow)]
|
|
|
|
|
pub struct PreviewJob {
|
|
|
|
|
pub id: Uuid,
|
|
|
|
|
pub kind: String,
|
2026-11-17 16:04:44 +00:00
|
|
|
pub chart_id: Option<Uuid>,
|
|
|
|
|
pub synastry_id: Option<Uuid>,
|
2026-11-18 11:10:40 +00:00
|
|
|
pub forecast_id: Option<Uuid>,
|
2026-11-17 14:41:12 +00:00
|
|
|
pub language: String,
|
|
|
|
|
pub model: String,
|
|
|
|
|
pub system: String,
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-19 06:40:09 +00:00
|
|
|
/// Which analyses a claim may take.
|
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
|
|
|
pub enum Lane {
|
|
|
|
|
/// The short summary of a chart (questions are claimed separately).
|
|
|
|
|
Quick,
|
|
|
|
|
/// The detailed analyses and the yearly forecast.
|
|
|
|
|
Slow,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
impl Lane {
|
|
|
|
|
fn kinds(self) -> &'static [&'static str] {
|
|
|
|
|
match self {
|
|
|
|
|
Lane::Quick => &["summary"],
|
|
|
|
|
Lane::Slow => &["natal", "synastry", "forecast"],
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Slots per lane; 0 turns a lane off.
|
|
|
|
|
#[derive(Debug, Clone, Copy)]
|
|
|
|
|
pub struct Lanes {
|
|
|
|
|
pub quick: usize,
|
|
|
|
|
pub slow: usize,
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-15 12:18:40 +00:00
|
|
|
pub struct Analyzer {
|
|
|
|
|
pub db: PgPool,
|
|
|
|
|
pub llm: OpenRouter,
|
|
|
|
|
pub max_attempts: i32,
|
2026-11-19 06:40:09 +00:00
|
|
|
pub lanes: Lanes,
|
2026-11-17 14:41:12 +00:00
|
|
|
/// Analyses and previews being written right now (released on shutdown).
|
2026-11-15 12:18:40 +00:00
|
|
|
running: Mutex<HashSet<Uuid>>,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Wait before the next attempt: 30 s, 2 min, 8 min, ... at most 1 hour.
|
|
|
|
|
pub fn backoff(attempts: i32) -> Duration {
|
|
|
|
|
let exp = attempts.clamp(1, 6).saturating_sub(1) as u32;
|
|
|
|
|
Duration::from_secs((30 * 4u64.pow(exp)).min(3600))
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
impl Analyzer {
|
2026-11-19 06:40:09 +00:00
|
|
|
pub fn new(db: PgPool, llm: OpenRouter, max_attempts: i32, lanes: Lanes) -> Self {
|
|
|
|
|
Self { db, llm, max_attempts, lanes, running: Mutex::default() }
|
2026-11-15 12:18:40 +00:00
|
|
|
}
|
|
|
|
|
|
2026-11-19 06:40:09 +00:00
|
|
|
/// Takes the next due analysis of the lane, if any.
|
|
|
|
|
pub async fn claim(&self, lane: Lane) -> Result<Option<Job>, sqlx::Error> {
|
2026-11-15 12:18:40 +00:00
|
|
|
let job: Option<Job> = sqlx::query_as(
|
|
|
|
|
"UPDATE analyses SET status = 'running', started_at = now(), attempts = attempts + 1 \
|
|
|
|
|
WHERE id = (SELECT id FROM analyses WHERE status = 'pending' AND next_attempt_at <= now() \
|
2026-11-19 06:40:09 +00:00
|
|
|
AND kind = ANY($1) \
|
2026-11-15 12:18:40 +00:00
|
|
|
ORDER BY next_attempt_at LIMIT 1 FOR UPDATE SKIP LOCKED) \
|
2026-11-18 21:11:08 +00:00
|
|
|
RETURNING id, kind, chart_id, synastry_id, forecast_id, language, model, prompt_version, attempts, \
|
2026-11-18 21:31:09 +00:00
|
|
|
fallback_model, on_fallback, started_at",
|
2026-11-15 12:18:40 +00:00
|
|
|
)
|
2026-11-19 06:40:09 +00:00
|
|
|
.bind(lane.kinds())
|
2026-11-15 12:18:40 +00:00
|
|
|
.fetch_optional(&self.db)
|
|
|
|
|
.await?;
|
|
|
|
|
if let Some(job) = &job {
|
|
|
|
|
self.running.lock().unwrap().insert(job.id);
|
|
|
|
|
}
|
|
|
|
|
Ok(job)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Builds the prompt from the stored chart or synastry.
|
|
|
|
|
pub async fn prompt_for(&self, job: &Job) -> Result<Prompt, LlmError> {
|
|
|
|
|
let kind = Kind::parse(&job.kind).ok_or_else(|| LlmError::Permanent(format!("unknown kind '{}'", job.kind)))?;
|
|
|
|
|
let db_err = |e: sqlx::Error| LlmError::Transient(format!("database: {e}"));
|
|
|
|
|
let text = match kind {
|
2026-11-17 14:41:12 +00:00
|
|
|
Kind::Natal | Kind::Summary => self.natal_text(job.chart_id).await?,
|
2026-11-17 16:04:44 +00:00
|
|
|
Kind::Synastry => self.synastry_text(job.synastry_id).await?,
|
2026-11-18 08:19:13 +00:00
|
|
|
Kind::Forecast => self.forecast_text(job.forecast_id).await?,
|
2026-11-18 18:44:20 +00:00
|
|
|
Kind::Question => return Err(LlmError::Permanent("a question is not an analysis".into())),
|
2026-11-15 12:18:40 +00:00
|
|
|
};
|
2026-11-17 14:25:08 +00:00
|
|
|
let (base, revision) = prompt::split_version(&job.prompt_version);
|
|
|
|
|
if base != PROMPT_VERSION {
|
2026-11-15 12:18:40 +00:00
|
|
|
tracing::warn!(id = %job.id, requested = %job.prompt_version, current = PROMPT_VERSION, "prompt version differs");
|
|
|
|
|
}
|
2026-11-17 14:25:08 +00:00
|
|
|
// An admin-edited system prompt; NULL (or a missing row) means the built-in one.
|
|
|
|
|
let system: Option<String> = match revision {
|
|
|
|
|
Some(id) => sqlx::query_scalar("SELECT system FROM prompt_revisions WHERE id = $1")
|
|
|
|
|
.bind(id)
|
|
|
|
|
.fetch_optional(&self.db)
|
|
|
|
|
.await
|
|
|
|
|
.map_err(db_err)?
|
|
|
|
|
.flatten(),
|
|
|
|
|
None => None,
|
|
|
|
|
};
|
|
|
|
|
Ok(prompt::build(kind, &job.language, &text, system.as_deref()))
|
2026-11-15 12:18:40 +00:00
|
|
|
}
|
|
|
|
|
|
2026-11-17 14:41:12 +00:00
|
|
|
/// The compact text of a stored natal chart.
|
|
|
|
|
async fn natal_text(&self, chart_id: Option<Uuid>) -> Result<String, LlmError> {
|
|
|
|
|
let (chart,): (Value,) = sqlx::query_as("SELECT result FROM charts WHERE id = $1")
|
|
|
|
|
.bind(chart_id)
|
|
|
|
|
.fetch_optional(&self.db)
|
|
|
|
|
.await
|
|
|
|
|
.map_err(|e| LlmError::Transient(format!("database: {e}")))?
|
|
|
|
|
.ok_or_else(|| LlmError::Permanent("the chart was deleted".into()))?;
|
|
|
|
|
Ok(compact::natal(&chart))
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-17 16:04:44 +00:00
|
|
|
/// The compact text of a stored synastry with its two charts.
|
|
|
|
|
async fn synastry_text(&self, synastry_id: Option<Uuid>) -> Result<String, LlmError> {
|
|
|
|
|
let (result, chart1, chart2): (Value, Value, Value) = sqlx::query_as(
|
|
|
|
|
"SELECT s.result, c1.result, c2.result FROM synastries s \
|
|
|
|
|
JOIN charts c1 ON c1.id = s.chart1_id JOIN charts c2 ON c2.id = s.chart2_id WHERE s.id = $1",
|
|
|
|
|
)
|
|
|
|
|
.bind(synastry_id)
|
|
|
|
|
.fetch_optional(&self.db)
|
|
|
|
|
.await
|
|
|
|
|
.map_err(|e| LlmError::Transient(format!("database: {e}")))?
|
|
|
|
|
.ok_or_else(|| LlmError::Permanent("the synastry was deleted".into()))?;
|
|
|
|
|
Ok(compact::synastry(&chart1, &chart2, &result))
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-18 08:19:13 +00:00
|
|
|
/// The compact text of a stored forecast with its natal chart.
|
|
|
|
|
async fn forecast_text(&self, forecast_id: Option<Uuid>) -> Result<String, LlmError> {
|
|
|
|
|
let (result, chart): (Value, Value) = sqlx::query_as(
|
|
|
|
|
"SELECT f.result, c.result FROM forecasts f JOIN charts c ON c.id = f.chart_id WHERE f.id = $1",
|
|
|
|
|
)
|
|
|
|
|
.bind(forecast_id)
|
|
|
|
|
.fetch_optional(&self.db)
|
|
|
|
|
.await
|
|
|
|
|
.map_err(|e| LlmError::Transient(format!("database: {e}")))?
|
|
|
|
|
.ok_or_else(|| LlmError::Permanent("the forecast was deleted".into()))?;
|
|
|
|
|
Ok(compact::forecast(&chart, &result))
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-17 14:41:12 +00:00
|
|
|
/// Takes the oldest waiting prompt test run, if any.
|
|
|
|
|
pub async fn claim_preview(&self) -> Result<Option<PreviewJob>, sqlx::Error> {
|
|
|
|
|
let job: Option<PreviewJob> = sqlx::query_as(
|
|
|
|
|
"UPDATE prompt_previews SET status = 'running', started_at = now() \
|
|
|
|
|
WHERE id = (SELECT id FROM prompt_previews WHERE status = 'pending' \
|
|
|
|
|
ORDER BY created_at LIMIT 1 FOR UPDATE SKIP LOCKED) \
|
2026-11-18 11:10:40 +00:00
|
|
|
RETURNING id, kind, chart_id, synastry_id, forecast_id, language, model, system",
|
2026-11-17 14:41:12 +00:00
|
|
|
)
|
|
|
|
|
.fetch_optional(&self.db)
|
|
|
|
|
.await?;
|
|
|
|
|
if let Some(job) = &job {
|
|
|
|
|
self.running.lock().unwrap().insert(job.id);
|
|
|
|
|
}
|
|
|
|
|
Ok(job)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Writes one test run with the edited system prompt (a single attempt).
|
|
|
|
|
pub async fn handle_preview(&self, job: PreviewJob) {
|
|
|
|
|
let started = Instant::now();
|
|
|
|
|
let result = async {
|
2026-11-17 16:04:44 +00:00
|
|
|
let kind =
|
|
|
|
|
Kind::parse(&job.kind).ok_or_else(|| LlmError::Permanent(format!("unknown kind '{}'", job.kind)))?;
|
|
|
|
|
let text = match kind {
|
|
|
|
|
Kind::Synastry => self.synastry_text(job.synastry_id).await?,
|
2026-11-18 11:10:40 +00:00
|
|
|
Kind::Forecast => self.forecast_text(job.forecast_id).await?,
|
|
|
|
|
Kind::Natal | Kind::Summary => self.natal_text(job.chart_id).await?,
|
2026-11-18 18:44:20 +00:00
|
|
|
Kind::Question => return Err(LlmError::Permanent("a question has no test run".into())),
|
2026-11-17 16:04:44 +00:00
|
|
|
};
|
2026-11-17 14:41:12 +00:00
|
|
|
let built = prompt::build(kind, &job.language, &text, Some(&job.system));
|
|
|
|
|
let mut c = self.llm.complete(&job.model, &built).await?;
|
|
|
|
|
prompt::polish(&job.language, &mut c.content);
|
|
|
|
|
Ok::<_, LlmError>(c)
|
|
|
|
|
}
|
|
|
|
|
.await;
|
|
|
|
|
let saved = match &result {
|
|
|
|
|
Ok(c) => {
|
|
|
|
|
sqlx::query(
|
|
|
|
|
"UPDATE prompt_previews SET status = 'done', content = $2, model_used = $3, input_tokens = $4, \
|
|
|
|
|
output_tokens = $5, cost_usd = $6, completed_at = now() WHERE id = $1 AND status = 'running'",
|
|
|
|
|
)
|
|
|
|
|
.bind(job.id)
|
|
|
|
|
.bind(&c.content)
|
|
|
|
|
.bind(&c.model)
|
|
|
|
|
.bind(c.input_tokens)
|
|
|
|
|
.bind(c.output_tokens)
|
|
|
|
|
.bind(c.cost_usd)
|
|
|
|
|
.execute(&self.db)
|
|
|
|
|
.await
|
|
|
|
|
}
|
|
|
|
|
Err(e) => {
|
2026-11-18 21:47:37 +00:00
|
|
|
self.log_error("preview", job.id, &job.kind, &job.model, 1, "failed", e.message(), started.elapsed())
|
|
|
|
|
.await;
|
2026-11-17 14:41:12 +00:00
|
|
|
sqlx::query(
|
|
|
|
|
"UPDATE prompt_previews SET status = 'failed', error = $2, completed_at = now() \
|
|
|
|
|
WHERE id = $1 AND status = 'running'",
|
|
|
|
|
)
|
|
|
|
|
.bind(job.id)
|
|
|
|
|
.bind(e.message())
|
|
|
|
|
.execute(&self.db)
|
|
|
|
|
.await
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
match (&result, saved) {
|
|
|
|
|
(_, Err(e)) => tracing::error!(id = %job.id, error = %e, "saving the prompt preview failed"),
|
|
|
|
|
(Ok(c), Ok(_)) => tracing::info!(
|
|
|
|
|
id = %job.id, model = %c.model, cost_usd = c.cost_usd, secs = started.elapsed().as_secs(), "prompt preview done"
|
|
|
|
|
),
|
|
|
|
|
(Err(e), Ok(_)) => tracing::warn!(id = %job.id, error = %e.message(), "prompt preview failed"),
|
|
|
|
|
}
|
|
|
|
|
self.running.lock().unwrap().remove(&job.id);
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-15 12:18:40 +00:00
|
|
|
/// Writes one claimed analysis and records the outcome.
|
|
|
|
|
pub async fn handle(&self, job: Job) {
|
|
|
|
|
let started = Instant::now();
|
2026-11-18 21:31:09 +00:00
|
|
|
let work = async {
|
|
|
|
|
match self.prompt_for(&job).await {
|
|
|
|
|
Ok(prompt) => self.llm.complete(job.active_model(), &prompt).await.map(|mut c| {
|
|
|
|
|
prompt::polish(&job.language, &mut c.content);
|
|
|
|
|
c
|
|
|
|
|
}),
|
|
|
|
|
Err(e) => Err(e),
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
let result = tokio::select! {
|
|
|
|
|
result = work => result,
|
|
|
|
|
() = self.superseded("analyses", job.id, job.started_at) => {
|
|
|
|
|
tracing::info!(id = %job.id, "analysis restarted by an admin; this attempt is dropped");
|
|
|
|
|
return;
|
|
|
|
|
}
|
2026-11-15 12:18:40 +00:00
|
|
|
};
|
|
|
|
|
if let Err(e) = self.finish(&job, result, started.elapsed()).await {
|
|
|
|
|
tracing::error!(id = %job.id, error = %e, "saving the analysis result failed");
|
|
|
|
|
}
|
|
|
|
|
self.running.lock().unwrap().remove(&job.id);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async fn finish(&self, job: &Job, result: Result<Completion, LlmError>, took: Duration) -> Result<(), sqlx::Error> {
|
|
|
|
|
match result {
|
|
|
|
|
Ok(c) => {
|
|
|
|
|
sqlx::query(
|
|
|
|
|
"UPDATE analyses SET status = 'done', content = $2, model_used = $3, input_tokens = $4, \
|
|
|
|
|
output_tokens = $5, cost_usd = $6, error = NULL, completed_at = now() \
|
2026-11-18 21:31:09 +00:00
|
|
|
WHERE id = $1 AND status = 'running' AND started_at = $7",
|
2026-11-15 12:18:40 +00:00
|
|
|
)
|
|
|
|
|
.bind(job.id)
|
|
|
|
|
.bind(&c.content)
|
|
|
|
|
.bind(&c.model)
|
|
|
|
|
.bind(c.input_tokens)
|
|
|
|
|
.bind(c.output_tokens)
|
|
|
|
|
.bind(c.cost_usd)
|
2026-11-18 21:31:09 +00:00
|
|
|
.bind(job.started_at)
|
2026-11-15 12:18:40 +00:00
|
|
|
.execute(&self.db)
|
|
|
|
|
.await?;
|
|
|
|
|
tracing::info!(
|
|
|
|
|
id = %job.id, model = %c.model, input_tokens = c.input_tokens, output_tokens = c.output_tokens,
|
|
|
|
|
cost_usd = c.cost_usd, secs = took.as_secs(), "analysis done"
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
Err(e) => {
|
2026-11-18 21:11:08 +00:00
|
|
|
let can_fall_back = !job.on_fallback && job.fallback_model.is_some();
|
2026-11-18 21:47:37 +00:00
|
|
|
let next = next_step(&e, job.attempts, self.max_attempts, can_fall_back);
|
|
|
|
|
let outcome = match next {
|
|
|
|
|
Next::Retry(_) => "retry",
|
|
|
|
|
Next::Fallback => "fallback",
|
|
|
|
|
Next::Fail => "failed",
|
|
|
|
|
};
|
|
|
|
|
self.log_error(
|
|
|
|
|
"analysis",
|
|
|
|
|
job.id,
|
|
|
|
|
&job.kind,
|
|
|
|
|
job.active_model(),
|
|
|
|
|
job.attempts,
|
|
|
|
|
outcome,
|
|
|
|
|
e.message(),
|
|
|
|
|
took,
|
|
|
|
|
)
|
|
|
|
|
.await;
|
|
|
|
|
match next {
|
2026-11-18 21:11:08 +00:00
|
|
|
Next::Retry(wait) => {
|
|
|
|
|
sqlx::query(
|
|
|
|
|
"UPDATE analyses SET status = 'pending', error = $2, \
|
2026-11-18 21:31:09 +00:00
|
|
|
next_attempt_at = now() + make_interval(secs => $3) \
|
|
|
|
|
WHERE id = $1 AND status = 'running' AND started_at = $4",
|
2026-11-18 21:11:08 +00:00
|
|
|
)
|
|
|
|
|
.bind(job.id)
|
|
|
|
|
.bind(e.message())
|
|
|
|
|
.bind(wait.as_secs_f64())
|
2026-11-18 21:31:09 +00:00
|
|
|
.bind(job.started_at)
|
2026-11-18 21:11:08 +00:00
|
|
|
.execute(&self.db)
|
|
|
|
|
.await?;
|
|
|
|
|
tracing::warn!(id = %job.id, attempts = job.attempts, retry_in_secs = wait.as_secs(), error = %e.message(), "analysis will be retried");
|
|
|
|
|
}
|
|
|
|
|
Next::Fallback => {
|
|
|
|
|
sqlx::query(
|
|
|
|
|
"UPDATE analyses SET status = 'pending', error = $2, on_fallback = true, attempts = 0, \
|
2026-11-18 21:31:09 +00:00
|
|
|
next_attempt_at = now() WHERE id = $1 AND status = 'running' AND started_at = $3",
|
2026-11-18 21:11:08 +00:00
|
|
|
)
|
|
|
|
|
.bind(job.id)
|
|
|
|
|
.bind(e.message())
|
2026-11-18 21:31:09 +00:00
|
|
|
.bind(job.started_at)
|
2026-11-18 21:11:08 +00:00
|
|
|
.execute(&self.db)
|
|
|
|
|
.await?;
|
|
|
|
|
tracing::warn!(
|
|
|
|
|
id = %job.id, model = %job.model, fallback = ?job.fallback_model, error = %e.message(),
|
|
|
|
|
"the chosen model failed, the default model takes over"
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
Next::Fail => {
|
2026-11-18 21:31:09 +00:00
|
|
|
self.fail(job.id, job.started_at, e.message()).await?;
|
2026-11-18 21:11:08 +00:00
|
|
|
tracing::error!(id = %job.id, attempts = job.attempts, error = %e.message(), "analysis failed");
|
|
|
|
|
}
|
|
|
|
|
}
|
2026-11-15 12:18:40 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
Ok(())
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-16 17:39:15 +00:00
|
|
|
/// Marks the analysis failed and gives its unit back (once).
|
2026-11-18 21:31:09 +00:00
|
|
|
async fn fail(&self, id: Uuid, started_at: time::OffsetDateTime, message: &str) -> Result<(), sqlx::Error> {
|
2026-11-15 12:18:40 +00:00
|
|
|
let mut tx = self.db.begin().await?;
|
2026-11-16 17:39:15 +00:00
|
|
|
let row: Option<(Option<Uuid>, i32, String)> = sqlx::query_as(
|
2026-11-15 12:18:40 +00:00
|
|
|
"UPDATE analyses SET status = 'failed', error = $2, completed_at = now() \
|
2026-11-18 21:31:09 +00:00
|
|
|
WHERE id = $1 AND status = 'running' AND started_at = $3 RETURNING requested_by, credits_charged, kind",
|
2026-11-15 12:18:40 +00:00
|
|
|
)
|
|
|
|
|
.bind(id)
|
|
|
|
|
.bind(message)
|
2026-11-18 21:31:09 +00:00
|
|
|
.bind(started_at)
|
2026-11-15 12:18:40 +00:00
|
|
|
.fetch_optional(&mut *tx)
|
|
|
|
|
.await?;
|
2026-11-16 17:39:15 +00:00
|
|
|
if let Some((Some(user), units, kind)) = row
|
|
|
|
|
&& units > 0
|
2026-11-15 12:18:40 +00:00
|
|
|
{
|
2026-11-18 13:15:57 +00:00
|
|
|
let unit = match kind.as_str() {
|
|
|
|
|
"synastry" => "synastry",
|
|
|
|
|
"forecast" => "forecast",
|
|
|
|
|
_ => "natal",
|
|
|
|
|
};
|
2026-11-15 12:18:40 +00:00
|
|
|
sqlx::query(
|
2026-11-16 17:39:15 +00:00
|
|
|
"INSERT INTO credit_ledger (id, user_id, unit, delta, reason, analysis_id, note) \
|
|
|
|
|
VALUES ($1, $2, $3, $4, 'refund', $5, 'analysis failed') ON CONFLICT DO NOTHING",
|
2026-11-15 12:18:40 +00:00
|
|
|
)
|
|
|
|
|
.bind(Uuid::now_v7())
|
|
|
|
|
.bind(user)
|
2026-11-16 17:39:15 +00:00
|
|
|
.bind(unit)
|
|
|
|
|
.bind(units)
|
2026-11-15 12:18:40 +00:00
|
|
|
.bind(id)
|
|
|
|
|
.execute(&mut *tx)
|
|
|
|
|
.await?;
|
|
|
|
|
}
|
|
|
|
|
tx.commit().await
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Puts analyses left `running` by a stopped worker back in the queue.
|
|
|
|
|
pub async fn requeue_stuck(&self) -> Result<u64, sqlx::Error> {
|
|
|
|
|
let result = sqlx::query(
|
|
|
|
|
"UPDATE analyses SET status = 'pending', error = 'interrupted', next_attempt_at = now() \
|
|
|
|
|
WHERE status = 'running' AND started_at < now() - make_interval(mins => $1)",
|
|
|
|
|
)
|
|
|
|
|
.bind(STUCK_AFTER_MINUTES)
|
|
|
|
|
.execute(&self.db)
|
|
|
|
|
.await?;
|
2026-11-18 18:44:20 +00:00
|
|
|
sqlx::query(
|
|
|
|
|
"UPDATE questions SET status = 'pending', error = 'interrupted', next_attempt_at = now() \
|
|
|
|
|
WHERE status = 'running' AND started_at < now() - make_interval(mins => $1)",
|
|
|
|
|
)
|
|
|
|
|
.bind(STUCK_AFTER_MINUTES)
|
|
|
|
|
.execute(&self.db)
|
|
|
|
|
.await?;
|
2026-11-17 14:41:12 +00:00
|
|
|
// A test run is not retried: the admin can start it again.
|
|
|
|
|
sqlx::query(
|
|
|
|
|
"UPDATE prompt_previews SET status = 'failed', error = 'interrupted', completed_at = now() \
|
|
|
|
|
WHERE status = 'running' AND started_at < now() - make_interval(mins => $1)",
|
|
|
|
|
)
|
|
|
|
|
.bind(STUCK_AFTER_MINUTES)
|
|
|
|
|
.execute(&self.db)
|
|
|
|
|
.await?;
|
2026-11-15 12:18:40 +00:00
|
|
|
Ok(result.rows_affected())
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-18 21:47:37 +00:00
|
|
|
/// Records a failed attempt in the admin error log (a failure here is only logged).
|
|
|
|
|
#[allow(clippy::too_many_arguments)]
|
|
|
|
|
pub(crate) async fn log_error(
|
|
|
|
|
&self,
|
|
|
|
|
source: &str,
|
|
|
|
|
item_id: Uuid,
|
|
|
|
|
kind: &str,
|
|
|
|
|
model: &str,
|
|
|
|
|
attempt: i32,
|
|
|
|
|
outcome: &str,
|
|
|
|
|
error: &str,
|
|
|
|
|
took: Duration,
|
|
|
|
|
) {
|
|
|
|
|
let result = sqlx::query(
|
|
|
|
|
"INSERT INTO generation_errors (source, item_id, kind, model, attempt, outcome, error, seconds) \
|
|
|
|
|
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)",
|
|
|
|
|
)
|
|
|
|
|
.bind(source)
|
|
|
|
|
.bind(item_id)
|
|
|
|
|
.bind(kind)
|
|
|
|
|
.bind(model)
|
|
|
|
|
.bind(attempt)
|
|
|
|
|
.bind(outcome)
|
|
|
|
|
.bind(error.chars().take(2000).collect::<String>())
|
|
|
|
|
.bind(i32::try_from(took.as_secs()).unwrap_or(i32::MAX))
|
|
|
|
|
.execute(&self.db)
|
|
|
|
|
.await;
|
|
|
|
|
if let Err(e) = result {
|
|
|
|
|
tracing::error!(error = %e, "writing the error log");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-18 21:31:09 +00:00
|
|
|
/// Resolves when this attempt is no longer the current one (an admin
|
|
|
|
|
/// restarted it); checked every 10 seconds, never on a database error.
|
|
|
|
|
async fn superseded(&self, table: &str, id: Uuid, started_at: time::OffsetDateTime) {
|
|
|
|
|
let query = format!("SELECT status = 'running' AND started_at = $2 FROM {table} WHERE id = $1");
|
|
|
|
|
loop {
|
|
|
|
|
tokio::time::sleep(Duration::from_secs(10)).await;
|
|
|
|
|
match sqlx::query_scalar::<_, Option<bool>>(&query).bind(id).bind(started_at).fetch_optional(&self.db).await
|
|
|
|
|
{
|
|
|
|
|
Ok(Some(Some(true))) => {}
|
|
|
|
|
Ok(_) => return,
|
|
|
|
|
Err(_) => {}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-15 12:18:40 +00:00
|
|
|
/// On shutdown: analyses still being written go back to `pending` at once.
|
|
|
|
|
async fn release_running(&self) {
|
|
|
|
|
let ids: Vec<Uuid> = self.running.lock().unwrap().drain().collect();
|
|
|
|
|
if ids.is_empty() {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
let result = sqlx::query(
|
|
|
|
|
"UPDATE analyses SET status = 'pending', error = 'interrupted', next_attempt_at = now(), \
|
|
|
|
|
attempts = greatest(attempts - 1, 0) WHERE id = ANY($1) AND status = 'running'",
|
|
|
|
|
)
|
|
|
|
|
.bind(&ids)
|
|
|
|
|
.execute(&self.db)
|
|
|
|
|
.await;
|
|
|
|
|
match result {
|
|
|
|
|
Ok(r) => tracing::info!(released = r.rows_affected(), "unfinished analyses put back in the queue"),
|
|
|
|
|
Err(e) => tracing::error!(error = %e, "releasing unfinished analyses"),
|
|
|
|
|
}
|
2026-11-18 18:44:20 +00:00
|
|
|
let result = sqlx::query(
|
|
|
|
|
"UPDATE questions SET status = 'pending', error = 'interrupted', next_attempt_at = now(), \
|
|
|
|
|
attempts = greatest(attempts - 1, 0) WHERE id = ANY($1) AND status = 'running'",
|
|
|
|
|
)
|
|
|
|
|
.bind(&ids)
|
|
|
|
|
.execute(&self.db)
|
|
|
|
|
.await;
|
|
|
|
|
if let Err(e) = result {
|
|
|
|
|
tracing::error!(error = %e, "releasing unfinished questions");
|
|
|
|
|
}
|
2026-11-17 14:41:12 +00:00
|
|
|
let result = sqlx::query(
|
|
|
|
|
"UPDATE prompt_previews SET status = 'failed', error = 'interrupted', completed_at = now() \
|
|
|
|
|
WHERE id = ANY($1) AND status = 'running'",
|
|
|
|
|
)
|
|
|
|
|
.bind(&ids)
|
|
|
|
|
.execute(&self.db)
|
|
|
|
|
.await;
|
|
|
|
|
if let Err(e) = result {
|
|
|
|
|
tracing::error!(error = %e, "releasing unfinished prompt previews");
|
|
|
|
|
}
|
2026-11-15 12:18:40 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-19 06:40:09 +00:00
|
|
|
/// Work a free slot can take, in order of preference.
|
|
|
|
|
pub(crate) enum Work {
|
|
|
|
|
Analysis(Job),
|
|
|
|
|
Question(QuestionJob),
|
|
|
|
|
Preview(PreviewJob),
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
impl Analyzer {
|
|
|
|
|
/// Quick work: a question first (someone is waiting for a short answer),
|
|
|
|
|
/// then a summary.
|
|
|
|
|
pub(crate) async fn claim_quick(&self) -> Result<Option<Work>, sqlx::Error> {
|
|
|
|
|
if let Some(job) = self.claim_question().await? {
|
|
|
|
|
return Ok(Some(Work::Question(job)));
|
|
|
|
|
}
|
|
|
|
|
Ok(self.claim(Lane::Quick).await?.map(Work::Analysis))
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Slow work: a detailed analysis, else quick work, else a test run.
|
|
|
|
|
pub(crate) async fn claim_slow(&self) -> Result<Option<Work>, sqlx::Error> {
|
|
|
|
|
if let Some(job) = self.claim(Lane::Slow).await? {
|
|
|
|
|
return Ok(Some(Work::Analysis(job)));
|
|
|
|
|
}
|
|
|
|
|
if let Some(work) = self.claim_quick().await? {
|
|
|
|
|
return Ok(Some(work));
|
|
|
|
|
}
|
|
|
|
|
Ok(self.claim_preview().await?.map(Work::Preview))
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async fn work(&self, work: Work) {
|
|
|
|
|
match work {
|
|
|
|
|
Work::Analysis(job) => self.handle(job).await,
|
|
|
|
|
Work::Question(job) => self.handle_question(job).await,
|
|
|
|
|
Work::Preview(job) => self.handle_preview(job).await,
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-15 12:18:40 +00:00
|
|
|
pub async fn run(analyzer: Arc<Analyzer>, poll: Duration, stop: watch::Receiver<bool>) {
|
2026-11-19 06:40:09 +00:00
|
|
|
let quick = Arc::new(Semaphore::new(analyzer.lanes.quick));
|
|
|
|
|
let slow = Arc::new(Semaphore::new(analyzer.lanes.slow));
|
2026-11-15 12:18:40 +00:00
|
|
|
let mut tasks = tokio::task::JoinSet::new();
|
|
|
|
|
let mut listener = crate::listen(&analyzer.db, "analyses").await;
|
|
|
|
|
let mut last_requeue: Option<Instant> = None;
|
|
|
|
|
loop {
|
|
|
|
|
if *stop.borrow() {
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
if last_requeue.is_none_or(|t| t.elapsed() > Duration::from_secs(60)) {
|
|
|
|
|
last_requeue = Some(Instant::now());
|
|
|
|
|
match analyzer.requeue_stuck().await {
|
|
|
|
|
Ok(0) => {}
|
|
|
|
|
Ok(n) => tracing::warn!(requeued = n, "interrupted analyses requeued"),
|
|
|
|
|
Err(e) => tracing::error!(error = %e, "requeueing analyses"),
|
|
|
|
|
}
|
|
|
|
|
}
|
2026-11-19 06:40:09 +00:00
|
|
|
// Fill the free slots: the quick lane first, then the slow one.
|
|
|
|
|
for lane in [Lane::Quick, Lane::Slow] {
|
|
|
|
|
let slots = if lane == Lane::Quick { &quick } else { &slow };
|
|
|
|
|
while let Ok(permit) = slots.clone().try_acquire_owned() {
|
|
|
|
|
let claimed = match lane {
|
|
|
|
|
Lane::Quick => analyzer.claim_quick().await,
|
|
|
|
|
Lane::Slow => analyzer.claim_slow().await,
|
|
|
|
|
};
|
|
|
|
|
match claimed {
|
|
|
|
|
Ok(Some(work)) => {
|
2026-11-17 14:41:12 +00:00
|
|
|
let analyzer = analyzer.clone();
|
|
|
|
|
tasks.spawn(async move {
|
2026-11-19 06:40:09 +00:00
|
|
|
analyzer.work(work).await;
|
2026-11-17 14:41:12 +00:00
|
|
|
drop(permit);
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
Ok(None) => break,
|
|
|
|
|
Err(e) => {
|
2026-11-19 06:40:09 +00:00
|
|
|
tracing::error!(error = %e, ?lane, "claiming work");
|
2026-11-17 14:41:12 +00:00
|
|
|
break;
|
|
|
|
|
}
|
2026-11-15 12:18:40 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
while tasks.try_join_next().is_some() {}
|
|
|
|
|
|
|
|
|
|
tokio::select! {
|
|
|
|
|
() = crate::notified(listener.as_mut()) => {}
|
|
|
|
|
() = tokio::time::sleep(poll) => {}
|
|
|
|
|
Some(_) = tasks.join_next(), if !tasks.is_empty() => {}
|
|
|
|
|
() = crate::stopped(stop.clone()) => break,
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
// Give running analyses a moment, then hand the rest back to the queue.
|
|
|
|
|
let _ = tokio::time::timeout(Duration::from_secs(10), async { while tasks.join_next().await.is_some() {} }).await;
|
|
|
|
|
tasks.abort_all();
|
|
|
|
|
analyzer.release_running().await;
|
|
|
|
|
}
|