Files
sorsfonalak/services/worker/src/analysis/openrouter.rs
T

156 lines
5.7 KiB
Rust
Raw Normal View History

//! OpenRouter chat completions with structured (JSON schema) output.
use std::time::Duration;
use serde_json::{Value, json};
use super::prompt::{Prompt, validate};
pub const DEFAULT_BASE_URL: &str = "https://openrouter.ai/api/v1";
/// Room for reasoning tokens as well as the answer.
const MAX_TOKENS: u32 = 16_000;
pub struct OpenRouter {
http: reqwest::Client,
base_url: String,
api_key: String,
/// Sent as `HTTP-Referer` (app attribution on openrouter.ai).
referer: Option<String>,
}
#[derive(Debug, Clone)]
pub struct Completion {
pub content: Value,
/// The model that answered (differs from the requested one for routers like `openrouter/auto`).
pub model: String,
pub input_tokens: Option<i32>,
pub output_tokens: Option<i32>,
pub cost_usd: Option<f64>,
}
#[derive(Debug, PartialEq)]
pub enum LlmError {
/// Worth retrying later (network, rate limit, provider outage, malformed answer).
Transient(String),
/// Retrying will not help (bad request, unknown model, rejected key).
Permanent(String),
}
impl LlmError {
pub fn message(&self) -> &str {
match self {
LlmError::Transient(m) | LlmError::Permanent(m) => m,
}
}
}
/// The JSON object in a reply, also when wrapped in a ``` fence or prose.
fn extract_json(text: &str) -> Option<Value> {
let trimmed = text.trim();
if let Ok(v) = serde_json::from_str(trimmed) {
return Some(v);
}
let (start, end) = (trimmed.find('{')?, trimmed.rfind('}')?);
serde_json::from_str(trimmed.get(start..=end)?).ok()
}
fn error_message(body: &Value, fallback: &str) -> String {
let e = &body["error"];
let message = e["message"].as_str().unwrap_or(fallback);
match e["metadata"]["raw"].as_str() {
Some(raw) => format!("{message}: {}", raw.chars().take(300).collect::<String>()),
None => message.to_owned(),
}
}
impl OpenRouter {
pub fn new(base_url: &str, api_key: &str, referer: Option<String>) -> Self {
let http = reqwest::Client::builder()
.timeout(Duration::from_secs(300))
.connect_timeout(Duration::from_secs(10))
.build()
.expect("HTTP client");
Self { http, base_url: base_url.trim_end_matches('/').to_owned(), api_key: api_key.to_owned(), referer }
}
pub async fn complete(&self, model: &str, prompt: &Prompt) -> Result<Completion, LlmError> {
let body = json!({
"model": model,
"messages": [
{"role": "system", "content": prompt.system},
{"role": "user", "content": prompt.user},
],
"response_format": {
"type": "json_schema",
"json_schema": {"name": prompt.schema_name, "strict": true, "schema": prompt.schema},
},
// Only route to providers that honour the schema.
"provider": {"require_parameters": true},
"max_tokens": MAX_TOKENS,
"usage": {"include": true},
});
let mut request = self
.http
.post(format!("{}/chat/completions", self.base_url))
.bearer_auth(&self.api_key)
.header("X-Title", "Sorsfonalak")
.json(&body);
if let Some(referer) = &self.referer {
request = request.header("HTTP-Referer", referer);
}
let response = request.send().await.map_err(|e| LlmError::Transient(format!("request failed: {e}")))?;
let status = response.status();
let text = response.text().await.map_err(|e| LlmError::Transient(format!("reading response: {e}")))?;
let reply: Value = serde_json::from_str(&text).unwrap_or(Value::Null);
if !status.is_success() {
let message = format!(
"HTTP {}: {}",
status.as_u16(),
error_message(&reply, &text.chars().take(300).collect::<String>())
);
// 402: our OpenRouter balance ran out; 408/429/5xx: try again later.
return Err(match status.as_u16() {
402 | 408 | 429 | 500..=599 => LlmError::Transient(message),
_ => LlmError::Permanent(message),
});
}
if reply["error"].is_object() {
return Err(LlmError::Transient(format!("provider error: {}", error_message(&reply, "unknown"))));
}
let choice = &reply["choices"][0];
if choice["finish_reason"] == "length" {
return Err(LlmError::Transient("answer cut off at the token limit".into()));
}
let raw = choice["message"]["content"].as_str().unwrap_or_default();
let content = extract_json(raw).ok_or_else(|| {
LlmError::Transient(format!("answer is not JSON: {}", raw.chars().take(200).collect::<String>()))
})?;
validate(&prompt.schema, &content)
.map_err(|e| LlmError::Transient(format!("answer does not match the schema: {e}")))?;
let usage = &reply["usage"];
let tokens = |key: &str| usage[key].as_i64().and_then(|n| i32::try_from(n).ok());
Ok(Completion {
content,
model: reply["model"].as_str().unwrap_or(model).to_owned(),
input_tokens: tokens("prompt_tokens"),
output_tokens: tokens("completion_tokens"),
cost_usd: usage["cost"].as_f64(),
})
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn extracts_json_from_fences_and_prose() {
assert_eq!(extract_json(r#"{"a":1}"#), Some(json!({"a": 1})));
assert_eq!(extract_json("```json\n{\"a\": 1}\n```"), Some(json!({"a": 1})));
assert_eq!(extract_json("Here you go: {\"a\": {\"b\": 2}} Enjoy!"), Some(json!({"a": {"b": 2}})));
assert_eq!(extract_json("no json here"), None);
}
}