Worker: separate quick lane for summaries and answers
Summaries and questions get their own slots (QUICK_CONCURRENCY, default 2), so they never wait behind detailed analyses (ANALYSIS_CONCURRENCY, default 4). A free slow slot also takes quick work; a lane of size 0 is off, so the lanes can run as separate deployments. Documented that several workers can share the queue. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012vLHzh5qUhFtmJhzkcTbXL
This commit is contained in:
@@ -9,6 +9,13 @@
|
||||
//! Admin test runs of an edited prompt (`prompt_previews`) go through the
|
||||
//! same loop when no analysis is due: one attempt, no credits, no retries.
|
||||
//!
|
||||
//! Two lanes, each with its own slots, so a short text never waits behind
|
||||
//! the long ones: the quick lane writes summaries and answers questions, the
|
||||
//! slow lane writes the detailed analyses and forecasts (and the test runs).
|
||||
//! A free slow slot also takes quick work. A lane of size 0 is off, so the
|
||||
//! lanes can run as separate deployments; several workers may share the
|
||||
//! queue, rows are claimed with `FOR UPDATE SKIP LOCKED`.
|
||||
//!
|
||||
//! Fallback: when the model chosen on the admin page fails twice (or once with
|
||||
//! a permanent error), the default model (`fallback_model`) takes over with a
|
||||
//! fresh set of attempts; only when that fails too is the unit refunded.
|
||||
@@ -29,6 +36,7 @@ use uuid::Uuid;
|
||||
|
||||
use self::openrouter::{Completion, LlmError, OpenRouter};
|
||||
use self::prompt::{Kind, PROMPT_VERSION, Prompt};
|
||||
use self::questions::QuestionJob;
|
||||
|
||||
/// A `running` analysis older than this was left behind by a stopped worker.
|
||||
const STUCK_AFTER_MINUTES: i32 = 15;
|
||||
@@ -101,11 +109,36 @@ pub struct PreviewJob {
|
||||
pub system: String,
|
||||
}
|
||||
|
||||
/// Which analyses a claim may take.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Lane {
|
||||
/// The short summary of a chart (questions are claimed separately).
|
||||
Quick,
|
||||
/// The detailed analyses and the yearly forecast.
|
||||
Slow,
|
||||
}
|
||||
|
||||
impl Lane {
|
||||
fn kinds(self) -> &'static [&'static str] {
|
||||
match self {
|
||||
Lane::Quick => &["summary"],
|
||||
Lane::Slow => &["natal", "synastry", "forecast"],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Slots per lane; 0 turns a lane off.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct Lanes {
|
||||
pub quick: usize,
|
||||
pub slow: usize,
|
||||
}
|
||||
|
||||
pub struct Analyzer {
|
||||
pub db: PgPool,
|
||||
pub llm: OpenRouter,
|
||||
pub max_attempts: i32,
|
||||
pub concurrency: usize,
|
||||
pub lanes: Lanes,
|
||||
/// Analyses and previews being written right now (released on shutdown).
|
||||
running: Mutex<HashSet<Uuid>>,
|
||||
}
|
||||
@@ -117,19 +150,21 @@ pub fn backoff(attempts: i32) -> Duration {
|
||||
}
|
||||
|
||||
impl Analyzer {
|
||||
pub fn new(db: PgPool, llm: OpenRouter, max_attempts: i32, concurrency: usize) -> Self {
|
||||
Self { db, llm, max_attempts, concurrency: concurrency.max(1), running: Mutex::default() }
|
||||
pub fn new(db: PgPool, llm: OpenRouter, max_attempts: i32, lanes: Lanes) -> Self {
|
||||
Self { db, llm, max_attempts, lanes, running: Mutex::default() }
|
||||
}
|
||||
|
||||
/// Takes the next due analysis, if any.
|
||||
pub async fn claim(&self) -> Result<Option<Job>, sqlx::Error> {
|
||||
/// Takes the next due analysis of the lane, if any.
|
||||
pub async fn claim(&self, lane: Lane) -> Result<Option<Job>, sqlx::Error> {
|
||||
let job: Option<Job> = sqlx::query_as(
|
||||
"UPDATE analyses SET status = 'running', started_at = now(), attempts = attempts + 1 \
|
||||
WHERE id = (SELECT id FROM analyses WHERE status = 'pending' AND next_attempt_at <= now() \
|
||||
AND kind = ANY($1) \
|
||||
ORDER BY next_attempt_at LIMIT 1 FOR UPDATE SKIP LOCKED) \
|
||||
RETURNING id, kind, chart_id, synastry_id, forecast_id, language, model, prompt_version, attempts, \
|
||||
fallback_model, on_fallback, started_at",
|
||||
)
|
||||
.bind(lane.kinds())
|
||||
.fetch_optional(&self.db)
|
||||
.await?;
|
||||
if let Some(job) = &job {
|
||||
@@ -530,8 +565,46 @@ impl Analyzer {
|
||||
}
|
||||
}
|
||||
|
||||
/// Work a free slot can take, in order of preference.
|
||||
pub(crate) enum Work {
|
||||
Analysis(Job),
|
||||
Question(QuestionJob),
|
||||
Preview(PreviewJob),
|
||||
}
|
||||
|
||||
impl Analyzer {
|
||||
/// Quick work: a question first (someone is waiting for a short answer),
|
||||
/// then a summary.
|
||||
pub(crate) async fn claim_quick(&self) -> Result<Option<Work>, sqlx::Error> {
|
||||
if let Some(job) = self.claim_question().await? {
|
||||
return Ok(Some(Work::Question(job)));
|
||||
}
|
||||
Ok(self.claim(Lane::Quick).await?.map(Work::Analysis))
|
||||
}
|
||||
|
||||
/// Slow work: a detailed analysis, else quick work, else a test run.
|
||||
pub(crate) async fn claim_slow(&self) -> Result<Option<Work>, sqlx::Error> {
|
||||
if let Some(job) = self.claim(Lane::Slow).await? {
|
||||
return Ok(Some(Work::Analysis(job)));
|
||||
}
|
||||
if let Some(work) = self.claim_quick().await? {
|
||||
return Ok(Some(work));
|
||||
}
|
||||
Ok(self.claim_preview().await?.map(Work::Preview))
|
||||
}
|
||||
|
||||
async fn work(&self, work: Work) {
|
||||
match work {
|
||||
Work::Analysis(job) => self.handle(job).await,
|
||||
Work::Question(job) => self.handle_question(job).await,
|
||||
Work::Preview(job) => self.handle_preview(job).await,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn run(analyzer: Arc<Analyzer>, poll: Duration, stop: watch::Receiver<bool>) {
|
||||
let slots = Arc::new(Semaphore::new(analyzer.concurrency));
|
||||
let quick = Arc::new(Semaphore::new(analyzer.lanes.quick));
|
||||
let slow = Arc::new(Semaphore::new(analyzer.lanes.slow));
|
||||
let mut tasks = tokio::task::JoinSet::new();
|
||||
let mut listener = crate::listen(&analyzer.db, "analyses").await;
|
||||
let mut last_requeue: Option<Instant> = None;
|
||||
@@ -547,50 +620,27 @@ pub async fn run(analyzer: Arc<Analyzer>, poll: Duration, stop: watch::Receiver<
|
||||
Err(e) => tracing::error!(error = %e, "requeueing analyses"),
|
||||
}
|
||||
}
|
||||
// Fill the free slots.
|
||||
while let Ok(permit) = slots.clone().try_acquire_owned() {
|
||||
// Questions first: someone is waiting for a short answer.
|
||||
match analyzer.claim_question().await {
|
||||
Ok(Some(job)) => {
|
||||
let analyzer = analyzer.clone();
|
||||
tasks.spawn(async move {
|
||||
analyzer.handle_question(job).await;
|
||||
drop(permit);
|
||||
});
|
||||
continue;
|
||||
}
|
||||
Ok(None) => {}
|
||||
Err(e) => {
|
||||
tracing::error!(error = %e, "claiming a question");
|
||||
break;
|
||||
}
|
||||
}
|
||||
match analyzer.claim().await {
|
||||
Ok(Some(job)) => {
|
||||
let analyzer = analyzer.clone();
|
||||
tasks.spawn(async move {
|
||||
analyzer.handle(job).await;
|
||||
drop(permit);
|
||||
});
|
||||
}
|
||||
// No analysis is due: a prompt test run, if one waits.
|
||||
Ok(None) => match analyzer.claim_preview().await {
|
||||
Ok(Some(job)) => {
|
||||
// Fill the free slots: the quick lane first, then the slow one.
|
||||
for lane in [Lane::Quick, Lane::Slow] {
|
||||
let slots = if lane == Lane::Quick { &quick } else { &slow };
|
||||
while let Ok(permit) = slots.clone().try_acquire_owned() {
|
||||
let claimed = match lane {
|
||||
Lane::Quick => analyzer.claim_quick().await,
|
||||
Lane::Slow => analyzer.claim_slow().await,
|
||||
};
|
||||
match claimed {
|
||||
Ok(Some(work)) => {
|
||||
let analyzer = analyzer.clone();
|
||||
tasks.spawn(async move {
|
||||
analyzer.handle_preview(job).await;
|
||||
analyzer.work(work).await;
|
||||
drop(permit);
|
||||
});
|
||||
}
|
||||
Ok(None) => break,
|
||||
Err(e) => {
|
||||
tracing::error!(error = %e, "claiming a prompt preview");
|
||||
tracing::error!(error = %e, ?lane, "claiming work");
|
||||
break;
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
tracing::error!(error = %e, "claiming an analysis");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user