Write text analyses with OpenRouter in the worker
- analysis job: claims pending analyses (SKIP LOCKED, NOTIFY wake-up), builds a compact chart text without birth data, asks OpenRouter with a strict JSON schema, validates and stores the answer, model used, tokens and cost; retries transient errors, refunds credits on final failure, requeues interrupted jobs - prompt v1 for natal and synastry analyses, Hungarian style guide - compare-models subcommand for blind model comparison - migration 0003: attempts, next_attempt_at, model_used, cost_usd, notify trigger, one refund per analysis - db: rebuild when migrations change - compose: worker reads OPENROUTER_API_KEY from .env.secrets Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012vLHzh5qUhFtmJhzkcTbXL
This commit is contained in:
@@ -0,0 +1,272 @@
|
||||
//! Writes the requested text analyses (`analyses` rows in `pending` state).
|
||||
//!
|
||||
//! The backend inserts the row and charges the credits; this job claims it
|
||||
//! (`running`), builds the prompt from the stored chart data, asks the model
|
||||
//! through OpenRouter and stores the structured answer (`done`). Temporary
|
||||
//! failures are retried with backoff; after the last attempt, or on a
|
||||
//! permanent error, the analysis is `failed` and the credits are refunded.
|
||||
|
||||
pub mod compact;
|
||||
pub mod openrouter;
|
||||
pub mod prompt;
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use serde_json::Value;
|
||||
use sqlx::PgPool;
|
||||
use tokio::sync::{Semaphore, watch};
|
||||
use uuid::Uuid;
|
||||
|
||||
use self::openrouter::{Completion, LlmError, OpenRouter};
|
||||
use self::prompt::{Kind, PROMPT_VERSION, Prompt};
|
||||
|
||||
/// A `running` analysis older than this was left behind by a stopped worker.
|
||||
const STUCK_AFTER_MINUTES: i32 = 15;
|
||||
|
||||
#[derive(Debug, sqlx::FromRow)]
|
||||
pub struct Job {
|
||||
pub id: Uuid,
|
||||
pub kind: String,
|
||||
pub chart_id: Option<Uuid>,
|
||||
pub synastry_id: Option<Uuid>,
|
||||
pub language: String,
|
||||
pub model: String,
|
||||
pub prompt_version: String,
|
||||
pub attempts: i32,
|
||||
}
|
||||
|
||||
pub struct Analyzer {
|
||||
pub db: PgPool,
|
||||
pub llm: OpenRouter,
|
||||
pub max_attempts: i32,
|
||||
pub concurrency: usize,
|
||||
/// Analyses being written right now (put back to `pending` on shutdown).
|
||||
running: Mutex<HashSet<Uuid>>,
|
||||
}
|
||||
|
||||
/// Wait before the next attempt: 30 s, 2 min, 8 min, ... at most 1 hour.
|
||||
pub fn backoff(attempts: i32) -> Duration {
|
||||
let exp = attempts.clamp(1, 6).saturating_sub(1) as u32;
|
||||
Duration::from_secs((30 * 4u64.pow(exp)).min(3600))
|
||||
}
|
||||
|
||||
impl Analyzer {
|
||||
pub fn new(db: PgPool, llm: OpenRouter, max_attempts: i32, concurrency: usize) -> Self {
|
||||
Self { db, llm, max_attempts, concurrency: concurrency.max(1), running: Mutex::default() }
|
||||
}
|
||||
|
||||
/// Takes the next due analysis, if any.
|
||||
pub async fn claim(&self) -> Result<Option<Job>, sqlx::Error> {
|
||||
let job: Option<Job> = sqlx::query_as(
|
||||
"UPDATE analyses SET status = 'running', started_at = now(), attempts = attempts + 1 \
|
||||
WHERE id = (SELECT id FROM analyses WHERE status = 'pending' AND next_attempt_at <= now() \
|
||||
ORDER BY next_attempt_at LIMIT 1 FOR UPDATE SKIP LOCKED) \
|
||||
RETURNING id, kind, chart_id, synastry_id, language, model, prompt_version, attempts",
|
||||
)
|
||||
.fetch_optional(&self.db)
|
||||
.await?;
|
||||
if let Some(job) = &job {
|
||||
self.running.lock().unwrap().insert(job.id);
|
||||
}
|
||||
Ok(job)
|
||||
}
|
||||
|
||||
/// Builds the prompt from the stored chart or synastry.
|
||||
pub async fn prompt_for(&self, job: &Job) -> Result<Prompt, LlmError> {
|
||||
let kind = Kind::parse(&job.kind).ok_or_else(|| LlmError::Permanent(format!("unknown kind '{}'", job.kind)))?;
|
||||
let db_err = |e: sqlx::Error| LlmError::Transient(format!("database: {e}"));
|
||||
let gone = || LlmError::Permanent("the chart was deleted".into());
|
||||
let text = match kind {
|
||||
Kind::Natal => {
|
||||
let (chart,): (Value,) = sqlx::query_as("SELECT result FROM charts WHERE id = $1")
|
||||
.bind(job.chart_id)
|
||||
.fetch_optional(&self.db)
|
||||
.await
|
||||
.map_err(db_err)?
|
||||
.ok_or_else(gone)?;
|
||||
compact::natal(&chart)
|
||||
}
|
||||
Kind::Synastry => {
|
||||
let (result, chart1, chart2): (Value, Value, Value) = sqlx::query_as(
|
||||
"SELECT s.result, c1.result, c2.result FROM synastries s \
|
||||
JOIN charts c1 ON c1.id = s.chart1_id JOIN charts c2 ON c2.id = s.chart2_id WHERE s.id = $1",
|
||||
)
|
||||
.bind(job.synastry_id)
|
||||
.fetch_optional(&self.db)
|
||||
.await
|
||||
.map_err(db_err)?
|
||||
.ok_or_else(gone)?;
|
||||
compact::synastry(&chart1, &chart2, &result)
|
||||
}
|
||||
};
|
||||
if job.prompt_version != PROMPT_VERSION {
|
||||
tracing::warn!(id = %job.id, requested = %job.prompt_version, current = PROMPT_VERSION, "prompt version differs");
|
||||
}
|
||||
Ok(prompt::build(kind, &job.language, &text))
|
||||
}
|
||||
|
||||
/// Writes one claimed analysis and records the outcome.
|
||||
pub async fn handle(&self, job: Job) {
|
||||
let started = Instant::now();
|
||||
let result = match self.prompt_for(&job).await {
|
||||
Ok(prompt) => self.llm.complete(&job.model, &prompt).await,
|
||||
Err(e) => Err(e),
|
||||
};
|
||||
if let Err(e) = self.finish(&job, result, started.elapsed()).await {
|
||||
tracing::error!(id = %job.id, error = %e, "saving the analysis result failed");
|
||||
}
|
||||
self.running.lock().unwrap().remove(&job.id);
|
||||
}
|
||||
|
||||
async fn finish(&self, job: &Job, result: Result<Completion, LlmError>, took: Duration) -> Result<(), sqlx::Error> {
|
||||
match result {
|
||||
Ok(c) => {
|
||||
sqlx::query(
|
||||
"UPDATE analyses SET status = 'done', content = $2, model_used = $3, input_tokens = $4, \
|
||||
output_tokens = $5, cost_usd = $6, error = NULL, completed_at = now() \
|
||||
WHERE id = $1 AND status = 'running'",
|
||||
)
|
||||
.bind(job.id)
|
||||
.bind(&c.content)
|
||||
.bind(&c.model)
|
||||
.bind(c.input_tokens)
|
||||
.bind(c.output_tokens)
|
||||
.bind(c.cost_usd)
|
||||
.execute(&self.db)
|
||||
.await?;
|
||||
tracing::info!(
|
||||
id = %job.id, model = %c.model, input_tokens = c.input_tokens, output_tokens = c.output_tokens,
|
||||
cost_usd = c.cost_usd, secs = took.as_secs(), "analysis done"
|
||||
);
|
||||
}
|
||||
Err(LlmError::Transient(message)) if job.attempts < self.max_attempts => {
|
||||
let wait = backoff(job.attempts);
|
||||
sqlx::query(
|
||||
"UPDATE analyses SET status = 'pending', error = $2, \
|
||||
next_attempt_at = now() + make_interval(secs => $3) WHERE id = $1 AND status = 'running'",
|
||||
)
|
||||
.bind(job.id)
|
||||
.bind(&message)
|
||||
.bind(wait.as_secs_f64())
|
||||
.execute(&self.db)
|
||||
.await?;
|
||||
tracing::warn!(id = %job.id, attempts = job.attempts, retry_in_secs = wait.as_secs(), error = %message, "analysis will be retried");
|
||||
}
|
||||
Err(e) => {
|
||||
self.fail(job.id, e.message()).await?;
|
||||
tracing::error!(id = %job.id, attempts = job.attempts, error = %e.message(), "analysis failed");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Marks the analysis failed and gives the credits back (once).
|
||||
async fn fail(&self, id: Uuid, message: &str) -> Result<(), sqlx::Error> {
|
||||
let mut tx = self.db.begin().await?;
|
||||
let row: Option<(Option<Uuid>, i32)> = sqlx::query_as(
|
||||
"UPDATE analyses SET status = 'failed', error = $2, completed_at = now() \
|
||||
WHERE id = $1 AND status = 'running' RETURNING requested_by, credits_charged",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(message)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await?;
|
||||
if let Some((Some(user), credits)) = row
|
||||
&& credits > 0
|
||||
{
|
||||
sqlx::query(
|
||||
"INSERT INTO credit_ledger (id, user_id, delta, reason, analysis_id, note) \
|
||||
VALUES ($1, $2, $3, 'refund', $4, 'analysis failed') ON CONFLICT DO NOTHING",
|
||||
)
|
||||
.bind(Uuid::now_v7())
|
||||
.bind(user)
|
||||
.bind(credits)
|
||||
.bind(id)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
}
|
||||
tx.commit().await
|
||||
}
|
||||
|
||||
/// Puts analyses left `running` by a stopped worker back in the queue.
|
||||
pub async fn requeue_stuck(&self) -> Result<u64, sqlx::Error> {
|
||||
let result = sqlx::query(
|
||||
"UPDATE analyses SET status = 'pending', error = 'interrupted', next_attempt_at = now() \
|
||||
WHERE status = 'running' AND started_at < now() - make_interval(mins => $1)",
|
||||
)
|
||||
.bind(STUCK_AFTER_MINUTES)
|
||||
.execute(&self.db)
|
||||
.await?;
|
||||
Ok(result.rows_affected())
|
||||
}
|
||||
|
||||
/// On shutdown: analyses still being written go back to `pending` at once.
|
||||
async fn release_running(&self) {
|
||||
let ids: Vec<Uuid> = self.running.lock().unwrap().drain().collect();
|
||||
if ids.is_empty() {
|
||||
return;
|
||||
}
|
||||
let result = sqlx::query(
|
||||
"UPDATE analyses SET status = 'pending', error = 'interrupted', next_attempt_at = now(), \
|
||||
attempts = greatest(attempts - 1, 0) WHERE id = ANY($1) AND status = 'running'",
|
||||
)
|
||||
.bind(&ids)
|
||||
.execute(&self.db)
|
||||
.await;
|
||||
match result {
|
||||
Ok(r) => tracing::info!(released = r.rows_affected(), "unfinished analyses put back in the queue"),
|
||||
Err(e) => tracing::error!(error = %e, "releasing unfinished analyses"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn run(analyzer: Arc<Analyzer>, poll: Duration, stop: watch::Receiver<bool>) {
|
||||
let slots = Arc::new(Semaphore::new(analyzer.concurrency));
|
||||
let mut tasks = tokio::task::JoinSet::new();
|
||||
let mut listener = crate::listen(&analyzer.db, "analyses").await;
|
||||
let mut last_requeue: Option<Instant> = None;
|
||||
loop {
|
||||
if *stop.borrow() {
|
||||
break;
|
||||
}
|
||||
if last_requeue.is_none_or(|t| t.elapsed() > Duration::from_secs(60)) {
|
||||
last_requeue = Some(Instant::now());
|
||||
match analyzer.requeue_stuck().await {
|
||||
Ok(0) => {}
|
||||
Ok(n) => tracing::warn!(requeued = n, "interrupted analyses requeued"),
|
||||
Err(e) => tracing::error!(error = %e, "requeueing analyses"),
|
||||
}
|
||||
}
|
||||
// Fill the free slots.
|
||||
while let Ok(permit) = slots.clone().try_acquire_owned() {
|
||||
match analyzer.claim().await {
|
||||
Ok(Some(job)) => {
|
||||
let analyzer = analyzer.clone();
|
||||
tasks.spawn(async move {
|
||||
analyzer.handle(job).await;
|
||||
drop(permit);
|
||||
});
|
||||
}
|
||||
Ok(None) => break,
|
||||
Err(e) => {
|
||||
tracing::error!(error = %e, "claiming an analysis");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
while tasks.try_join_next().is_some() {}
|
||||
|
||||
tokio::select! {
|
||||
() = crate::notified(listener.as_mut()) => {}
|
||||
() = tokio::time::sleep(poll) => {}
|
||||
Some(_) = tasks.join_next(), if !tasks.is_empty() => {}
|
||||
() = crate::stopped(stop.clone()) => break,
|
||||
}
|
||||
}
|
||||
// Give running analyses a moment, then hand the rest back to the queue.
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), async { while tasks.join_next().await.is_some() {} }).await;
|
||||
tasks.abort_all();
|
||||
analyzer.release_running().await;
|
||||
}
|
||||
Reference in New Issue
Block a user