118 lines
4.1 KiB
Rust
118 lines
4.1 KiB
Rust
//! Which model writes which kind of analysis, and the models OpenRouter
|
|||
|
|
//! offers (with prices) for the admin page.
|
||
|
|
//!
|
||
|
|
//! The choice is in `model_settings`; without a row the configured default
|
||
|
|
//! is used (`SUMMARY_MODEL` for the short summary, `ANALYSIS_MODEL` else).
|
||
|
|
//! The model is never shown to users (a business secret).
|
||
|
|
|
||
|
|
use std::sync::Arc;
|
||
|
|
use std::time::{Duration, Instant};
|
||
|
|
|
||
|
|
use analysis_prompt::Kind;
|
||
|
|
use serde::Serialize;
|
||
|
|
use serde_json::Value;
|
||
|
|
use tokio::sync::Mutex;
|
||
|
|
|
||
|
|
use crate::AppState;
|
||
|
|
|
||
|
|
/// The OpenRouter list is fetched at most this often.
|
||
|
|
const CATALOG_TTL: Duration = Duration::from_secs(3600);
|
||
|
|
|
||
|
|
/// The configured default model of a kind.
|
||
|
|
pub fn default_model(state: &AppState, kind: Kind) -> &str {
|
||
|
|
match kind {
|
||
|
|
Kind::Summary => &state.config.summary_model,
|
||
|
|
_ => &state.config.analysis_model,
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
/// The model new analyses of this kind are written with.
|
||
|
|
pub async fn model_for(state: &AppState, kind: Kind) -> Result<String, sqlx::Error> {
|
||
|
|
let chosen: Option<String> = sqlx::query_scalar("SELECT model FROM model_settings WHERE kind = $1")
|
||
|
|
.bind(kind.as_str())
|
||
|
|
.fetch_optional(&state.db)
|
||
|
|
.await?;
|
||
|
|
Ok(chosen.unwrap_or_else(|| default_model(state, kind).to_owned()))
|
||
|
|
}
|
||
|
|
|
||
|
|
/// One model OpenRouter offers, with its price in USD per million tokens.
|
||
|
|
#[derive(Debug, Clone, Serialize)]
|
||
|
|
pub struct ModelInfo {
|
||
|
|
pub id: String,
|
||
|
|
pub name: String,
|
||
|
|
pub input_per_million: f64,
|
||
|
|
pub output_per_million: f64,
|
||
|
|
pub context_length: Option<u64>,
|
||
|
|
}
|
||
|
|
|
||
|
|
pub struct Catalog {
|
||
|
|
url: String,
|
||
|
|
http: reqwest::Client,
|
||
|
|
cache: Mutex<Option<(Instant, Arc<Vec<ModelInfo>>)>>,
|
||
|
|
}
|
||
|
|
|
||
|
|
impl Catalog {
|
||
|
|
pub fn new(url: &str) -> Self {
|
||
|
|
let http = reqwest::Client::builder().timeout(Duration::from_secs(15)).build().expect("static configuration");
|
||
|
|
Self { url: url.to_owned(), http, cache: Mutex::default() }
|
||
|
|
}
|
||
|
|
|
||
|
|
/// The models that can write an analysis (they support the structured
|
||
|
|
/// output the worker asks for), by name; a stale list when OpenRouter is down.
|
||
|
|
pub async fn models(&self) -> Result<Arc<Vec<ModelInfo>>, String> {
|
||
|
|
let mut cache = self.cache.lock().await;
|
||
|
|
if let Some((at, list)) = cache.as_ref()
|
||
|
|
&& at.elapsed() < CATALOG_TTL
|
||
|
|
{
|
||
|
|
return Ok(list.clone());
|
||
|
|
}
|
||
|
|
match self.fetch().await {
|
||
|
|
Ok(list) => {
|
||
|
|
let list = Arc::new(list);
|
||
|
|
*cache = Some((Instant::now(), list.clone()));
|
||
|
|
Ok(list)
|
||
|
|
}
|
||
|
|
Err(e) => match cache.as_ref() {
|
||
|
|
Some((_, list)) => {
|
||
|
|
tracing::warn!(error = %e, "OpenRouter model list unavailable, using the cached one");
|
||
|
|
Ok(list.clone())
|
||
|
|
}
|
||
|
|
None => Err(e),
|
||
|
|
},
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
async fn fetch(&self) -> Result<Vec<ModelInfo>, String> {
|
||
|
|
let body: Value = self
|
||
|
|
.http
|
||
|
|
.get(&self.url)
|
||
|
|
.send()
|
||
|
|
.await
|
||
|
|
.and_then(reqwest::Response::error_for_status)
|
||
|
|
.map_err(|e| e.to_string())?
|
||
|
|
.json()
|
||
|
|
.await
|
||
|
|
.map_err(|e| e.to_string())?;
|
||
|
|
let price = |v: &Value| v.as_str().and_then(|s| s.parse::<f64>().ok()).map(|p| p * 1e6);
|
||
|
|
let mut list: Vec<ModelInfo> = body["data"]
|
||
|
|
.as_array()
|
||
|
|
.ok_or("unexpected OpenRouter answer")?
|
||
|
|
.iter()
|
||
|
|
.filter(|m| {
|
||
|
|
m["supported_parameters"].as_array().is_some_and(|p| p.iter().any(|x| x == "structured_outputs"))
|
||
|
|
})
|
||
|
|
.filter_map(|m| {
|
||
|
|
Some(ModelInfo {
|
||
|
|
id: m["id"].as_str()?.to_owned(),
|
||
|
|
name: m["name"].as_str().unwrap_or_default().to_owned(),
|
||
|
|
input_per_million: price(&m["pricing"]["prompt"])?,
|
||
|
|
output_per_million: price(&m["pricing"]["completion"])?,
|
||
|
|
context_length: m["context_length"].as_u64(),
|
||
|
|
})
|
||
|
|
})
|
||
|
|
.collect();
|
||
|
|
list.sort_by(|a, b| a.name.cmp(&b.name));
|
||
|
|
Ok(list)
|
||
|
|
}
|
||
|
|
}
|