//! `worker compare-models`: writes the same natal (or synastry) analysis with //! several models for a blind comparison. //! //! ```text //! OPENROUTER_API_KEY=... worker compare-models --chart natal.json \ //! --models minimax/minimax-m3,anthropic/claude-sonnet-5.5 [--language hu] [--out DIR] //! ``` //! //! `--chart` is a horoscope-api `/v1/natal` response; `--synastry FILE` (instead //! of `--chart`) takes a `/v1/synastry` response for a relationship analysis; //! `--summary` with `--chart` writes the short free summary instead. The models get shuffled //! letters: `A.md`, `B.md`, ... hold the analyses, `key.json` tells which //! letter is which model (with tokens, cost and time). The output on the //! console does not reveal the mapping. use std::path::PathBuf; use std::time::Instant; use anyhow::{Context, bail}; use rand::seq::SliceRandom; use serde_json::{Value, json}; use crate::analysis::openrouter::{DEFAULT_BASE_URL, OpenRouter}; use crate::analysis::{compact, prompt}; struct Args { input: PathBuf, kind: prompt::Kind, models: Vec, language: String, out: PathBuf, } fn parse(args: Vec) -> anyhow::Result { let (mut chart, mut synastry, mut models, mut language, mut out) = (None, None, Vec::new(), "hu".to_owned(), None); let mut summary = false; let mut it = args.into_iter(); while let Some(flag) = it.next() { let mut value = || it.next().with_context(|| format!("{flag} needs a value")); match flag.as_str() { "--chart" => chart = Some(PathBuf::from(value()?)), "--synastry" => synastry = Some(PathBuf::from(value()?)), "--summary" => summary = true, "--models" => models = value()?.split(',').map(|m| m.trim().to_owned()).filter(|m| !m.is_empty()).collect(), "--language" => language = value()?, "--out" => out = Some(PathBuf::from(value()?)), other => bail!("unknown argument '{other}'"), } } let (input, kind) = match (chart, synastry) { (Some(c), None) if summary => (c, prompt::Kind::Summary), (Some(c), None) => (c, prompt::Kind::Natal), (None, Some(s)) => (s, prompt::Kind::Synastry), _ => bail!("give either --chart or --synastry"), }; if models.is_empty() { bail!("--models is required (comma separated OpenRouter model ids)"); } if models.len() > 26 { bail!("at most 26 models"); } let out = out.unwrap_or_else(|| { let stamp = time::OffsetDateTime::now_utc().unix_timestamp(); PathBuf::from(format!("model-compare-{stamp}")) }); Ok(Args { input, kind, models, language, out }) } pub async fn main(args: Vec) -> anyhow::Result<()> { let args = parse(args)?; let key = std::env::var("OPENROUTER_API_KEY") .ok() .filter(|k| !k.trim().is_empty()) .context("OPENROUTER_API_KEY is not set")?; let base_url = std::env::var("OPENROUTER_BASE_URL").unwrap_or_else(|_| DEFAULT_BASE_URL.into()); let raw = std::fs::read_to_string(&args.input).with_context(|| format!("reading {}", args.input.display()))?; let mut input: Value = serde_json::from_str(&raw).context("the input file is not JSON")?; if input["result"].is_object() { input = input["result"].take(); } let data = match args.kind { prompt::Kind::Synastry => compact::synastry(&input["person1"], &input["person2"], &input), _ => compact::natal(&input), }; let built = prompt::build(args.kind, &args.language, &data); std::fs::create_dir_all(&args.out)?; std::fs::write( args.out.join("prompt.md"), format!( "# System\n\n{}\n\n# User\n\n{}\n\n# Schema\n\n```json\n{:#}\n```\n", built.system, built.user, built.schema ), )?; let kind = args.kind; let mut models = args.models.clone(); models.shuffle(&mut rand::rng()); let llm = std::sync::Arc::new(OpenRouter::new(&base_url, &key, None)); let built = std::sync::Arc::new(built); println!("Writing {} analyses in parallel (this takes a minute or two)...", models.len()); let mut tasks = tokio::task::JoinSet::new(); for (i, model) in models.iter().enumerate() { let (llm, built, model) = (llm.clone(), built.clone(), model.clone()); let letter = char::from(b'A' + i as u8); tasks.spawn(async move { let started = Instant::now(); let result = llm.complete(&model, &built).await; (letter, model, result, started.elapsed().as_secs_f64()) }); } let mut key_entries = Vec::new(); let mut total_cost = 0.0; while let Some(joined) = tasks.join_next().await { let (letter, model, result, secs) = joined?; match result { Ok(mut c) => { prompt::polish(&args.language, &mut c.content); let md = prompt::to_markdown(kind, &args.language, &c.content); let words = md.split_whitespace().count(); std::fs::write(args.out.join(format!("{letter}.md")), &md)?; std::fs::write(args.out.join(format!("{letter}.json")), serde_json::to_string_pretty(&c.content)?)?; total_cost += c.cost_usd.unwrap_or(0.0); println!("{letter}: ok, {words} words"); key_entries.push(json!({ "letter": letter.to_string(), "model": model, "model_used": c.model, "ok": true, "words": words, "input_tokens": c.input_tokens, "output_tokens": c.output_tokens, "cost_usd": c.cost_usd, "seconds": (secs * 10.0).round() / 10.0, })); } Err(e) => { println!("{letter}: failed (details in key.json)"); key_entries.push(json!({ "letter": letter.to_string(), "model": model, "ok": false, "error": e.message(), "seconds": (secs * 10.0).round() / 10.0, })); } } } key_entries.sort_by(|a, b| a["letter"].as_str().cmp(&b["letter"].as_str())); std::fs::write(args.out.join("key.json"), serde_json::to_string_pretty(&key_entries)?)?; println!("Total cost: ${total_cost:.4}. Output: {}", args.out.display()); Ok(()) }