//! OpenRouter chat completions with structured (JSON schema) output. use std::time::Duration; use serde::ser::{Serialize, SerializeMap, Serializer}; use serde_json::{Value, json}; use super::prompt::{Prompt, validate}; pub const DEFAULT_BASE_URL: &str = "https://openrouter.ai/api/v1"; /// Room for reasoning tokens as well as the answer. const MAX_TOKENS: u32 = 16_000; pub struct OpenRouter { http: reqwest::Client, base_url: String, api_key: String, /// Sent as `HTTP-Referer` (app attribution on openrouter.ai). referer: Option, } #[derive(Debug, Clone)] pub struct Completion { pub content: Value, /// The model that answered (differs from the requested one for routers like `openrouter/auto`). pub model: String, pub input_tokens: Option, pub output_tokens: Option, pub cost_usd: Option, } #[derive(Debug, PartialEq)] pub enum LlmError { /// Worth retrying later (network, rate limit, provider outage, malformed answer). Transient(String), /// Retrying will not help (bad request, unknown model, rejected key). Permanent(String), } impl LlmError { pub fn message(&self) -> &str { match self { LlmError::Transient(m) | LlmError::Permanent(m) => m, } } } /// A JSON schema serialized with every `properties` object in the order of /// its sibling `required` list. `serde_json` sorts object keys, and models /// write strict structured output in schema order: sorted, the title came /// after 9000 tokens of text and was sometimes left empty. pub struct Ordered<'a>(pub &'a Value); impl Serialize for Ordered<'_> { fn serialize(&self, serializer: S) -> Result { match self.0 { Value::Object(map) => { let order: Vec<&str> = map.get("required").and_then(Value::as_array).into_iter().flatten().filter_map(Value::as_str).collect(); let mut out = serializer.serialize_map(Some(map.len()))?; for (key, value) in map { match (key.as_str(), value) { ("properties", Value::Object(props)) => out.serialize_entry(key, &Properties(props, &order))?, _ => out.serialize_entry(key, &Ordered(value))?, } } out.end() } Value::Array(items) => serializer.collect_seq(items.iter().map(Ordered)), other => other.serialize(serializer), } } } /// The request body; a struct (not `json!`) so the schema keeps its order. #[derive(serde::Serialize)] struct Request<'a> { model: &'a str, messages: Value, response_format: ResponseFormat<'a>, provider: Value, max_tokens: u32, usage: Value, } #[derive(serde::Serialize)] struct ResponseFormat<'a> { #[serde(rename = "type")] kind: &'static str, json_schema: JsonSchema<'a>, } #[derive(serde::Serialize)] struct JsonSchema<'a> { name: &'static str, strict: bool, schema: Ordered<'a>, } /// `properties` in `required` order (any others after them). struct Properties<'a>(&'a serde_json::Map, &'a [&'a str]); impl Serialize for Properties<'_> { fn serialize(&self, serializer: S) -> Result { let (props, order) = (self.0, self.1); let mut out = serializer.serialize_map(Some(props.len()))?; for key in order { if let Some(value) = props.get(*key) { out.serialize_entry(key, &Ordered(value))?; } } for (key, value) in props.iter().filter(|(k, _)| !order.contains(&k.as_str())) { out.serialize_entry(key, &Ordered(value))?; } out.end() } } /// The JSON object in a reply, also when wrapped in a ``` fence or prose. fn extract_json(text: &str) -> Option { let trimmed = text.trim(); if let Ok(v) = serde_json::from_str(trimmed) { return Some(v); } let (start, end) = (trimmed.find('{')?, trimmed.rfind('}')?); serde_json::from_str(trimmed.get(start..=end)?).ok() } fn error_message(body: &Value, fallback: &str) -> String { let e = &body["error"]; let message = e["message"].as_str().unwrap_or(fallback); match e["metadata"]["raw"].as_str() { Some(raw) => format!("{message}: {}", raw.chars().take(300).collect::()), None => message.to_owned(), } } impl OpenRouter { pub fn new(base_url: &str, api_key: &str, referer: Option) -> Self { let http = reqwest::Client::builder() .timeout(Duration::from_secs(300)) .connect_timeout(Duration::from_secs(10)) .build() .expect("HTTP client"); Self { http, base_url: base_url.trim_end_matches('/').to_owned(), api_key: api_key.to_owned(), referer } } pub async fn complete(&self, model: &str, prompt: &Prompt) -> Result { let body = Request { model, messages: json!([ {"role": "system", "content": prompt.system}, {"role": "user", "content": prompt.user}, ]), response_format: ResponseFormat { kind: "json_schema", json_schema: JsonSchema { name: prompt.schema_name, strict: true, schema: Ordered(&prompt.schema) }, }, // Only route to providers that honour the schema. provider: json!({"require_parameters": true}), max_tokens: MAX_TOKENS, usage: json!({"include": true}), }; let mut request = self .http .post(format!("{}/chat/completions", self.base_url)) .bearer_auth(&self.api_key) .header("X-Title", "Sorsfonalak") .json(&body); if let Some(referer) = &self.referer { request = request.header("HTTP-Referer", referer); } let response = request.send().await.map_err(|e| LlmError::Transient(format!("request failed: {e}")))?; let status = response.status(); let text = response.text().await.map_err(|e| LlmError::Transient(format!("reading response: {e}")))?; let reply: Value = serde_json::from_str(&text).unwrap_or(Value::Null); if !status.is_success() { let message = format!( "HTTP {}: {}", status.as_u16(), error_message(&reply, &text.chars().take(300).collect::()) ); // 402: our OpenRouter balance ran out; 408/429/5xx: try again later. return Err(match status.as_u16() { 402 | 408 | 429 | 500..=599 => LlmError::Transient(message), _ => LlmError::Permanent(message), }); } if reply["error"].is_object() { return Err(LlmError::Transient(format!("provider error: {}", error_message(&reply, "unknown")))); } let choice = &reply["choices"][0]; if choice["finish_reason"] == "length" { return Err(LlmError::Transient("answer cut off at the token limit".into())); } let raw = choice["message"]["content"].as_str().unwrap_or_default(); let content = extract_json(raw).ok_or_else(|| { LlmError::Transient(format!("answer is not JSON: {}", raw.chars().take(200).collect::())) })?; validate(&prompt.schema, &content) .map_err(|e| LlmError::Transient(format!("answer does not match the schema: {e}")))?; let usage = &reply["usage"]; let tokens = |key: &str| usage[key].as_i64().and_then(|n| i32::try_from(n).ok()); Ok(Completion { content, model: reply["model"].as_str().unwrap_or(model).to_owned(), input_tokens: tokens("prompt_tokens"), output_tokens: tokens("completion_tokens"), cost_usd: usage["cost"].as_f64(), }) } } #[cfg(test)] mod tests { use super::*; #[test] fn schema_properties_follow_required_order() { let schema = super::super::prompt::schema(super::super::prompt::Kind::Natal); let text = serde_json::to_string(&Ordered(&schema)).unwrap(); let pos = |needle: &str| text.find(needle).unwrap(); assert!(pos("\"title\":{") < pos("\"summary\":{")); assert!(pos("\"summary\":{") < pos("\"sections\":{")); assert!(pos("\"personality\":{") < pos("\"growth\":{")); assert!(pos("\"strengths\":{") < pos("\"advice\":{")); // inside a section the title comes before the body let section = &text[pos("\"personality\":{")..]; assert!(section.find("\"title\":{").unwrap() < section.find("\"body\":{").unwrap()); // the content is unchanged assert_eq!(serde_json::from_str::(&text).unwrap(), schema); } #[test] fn extracts_json_from_fences_and_prose() { assert_eq!(extract_json(r#"{"a":1}"#), Some(json!({"a": 1}))); assert_eq!(extract_json("```json\n{\"a\": 1}\n```"), Some(json!({"a": 1}))); assert_eq!(extract_json("Here you go: {\"a\": {\"b\": 2}} Enjoy!"), Some(json!({"a": {"b": 2}}))); assert_eq!(extract_json("no json here"), None); } }