Files
sorsfonalak/services/backend/src/models.rs
T

118 lines
4.1 KiB
Rust
Raw Normal View History

//! Which model writes which kind of analysis, and the models OpenRouter
//! offers (with prices) for the admin page.
//!
//! The choice is in `model_settings`; without a row the configured default
//! is used (`SUMMARY_MODEL` for the short summary, `ANALYSIS_MODEL` else).
//! The model is never shown to users (a business secret).
use std::sync::Arc;
use std::time::{Duration, Instant};
use analysis_prompt::Kind;
use serde::Serialize;
use serde_json::Value;
use tokio::sync::Mutex;
use crate::AppState;
/// The OpenRouter list is fetched at most this often.
const CATALOG_TTL: Duration = Duration::from_secs(3600);
/// The configured default model of a kind.
pub fn default_model(state: &AppState, kind: Kind) -> &str {
match kind {
Kind::Summary => &state.config.summary_model,
_ => &state.config.analysis_model,
}
}
/// The model new analyses of this kind are written with.
pub async fn model_for(state: &AppState, kind: Kind) -> Result<String, sqlx::Error> {
let chosen: Option<String> = sqlx::query_scalar("SELECT model FROM model_settings WHERE kind = $1")
.bind(kind.as_str())
.fetch_optional(&state.db)
.await?;
Ok(chosen.unwrap_or_else(|| default_model(state, kind).to_owned()))
}
/// One model OpenRouter offers, with its price in USD per million tokens.
#[derive(Debug, Clone, Serialize)]
pub struct ModelInfo {
pub id: String,
pub name: String,
pub input_per_million: f64,
pub output_per_million: f64,
pub context_length: Option<u64>,
}
pub struct Catalog {
url: String,
http: reqwest::Client,
cache: Mutex<Option<(Instant, Arc<Vec<ModelInfo>>)>>,
}
impl Catalog {
pub fn new(url: &str) -> Self {
let http = reqwest::Client::builder().timeout(Duration::from_secs(15)).build().expect("static configuration");
Self { url: url.to_owned(), http, cache: Mutex::default() }
}
/// The models that can write an analysis (they support the structured
/// output the worker asks for), by name; a stale list when OpenRouter is down.
pub async fn models(&self) -> Result<Arc<Vec<ModelInfo>>, String> {
let mut cache = self.cache.lock().await;
if let Some((at, list)) = cache.as_ref()
&& at.elapsed() < CATALOG_TTL
{
return Ok(list.clone());
}
match self.fetch().await {
Ok(list) => {
let list = Arc::new(list);
*cache = Some((Instant::now(), list.clone()));
Ok(list)
}
Err(e) => match cache.as_ref() {
Some((_, list)) => {
tracing::warn!(error = %e, "OpenRouter model list unavailable, using the cached one");
Ok(list.clone())
}
None => Err(e),
},
}
}
async fn fetch(&self) -> Result<Vec<ModelInfo>, String> {
let body: Value = self
.http
.get(&self.url)
.send()
.await
.and_then(reqwest::Response::error_for_status)
.map_err(|e| e.to_string())?
.json()
.await
.map_err(|e| e.to_string())?;
let price = |v: &Value| v.as_str().and_then(|s| s.parse::<f64>().ok()).map(|p| p * 1e6);
let mut list: Vec<ModelInfo> = body["data"]
.as_array()
.ok_or("unexpected OpenRouter answer")?
.iter()
.filter(|m| {
m["supported_parameters"].as_array().is_some_and(|p| p.iter().any(|x| x == "structured_outputs"))
})
.filter_map(|m| {
Some(ModelInfo {
id: m["id"].as_str()?.to_owned(),
name: m["name"].as_str().unwrap_or_default().to_owned(),
input_per_million: price(&m["pricing"]["prompt"])?,
output_per_million: price(&m["pricing"]["completion"])?,
context_length: m["context_length"].as_u64(),
})
})
.collect();
list.sort_by(|a, b| a.name.cmp(&b.name));
Ok(list)
}
}