compare-models: synastry input; reject placeholder strings in answers

- worker compare-models --synastry FILE takes a /v1/synastry response
  and writes a relationship analysis.
- Answer validation now needs at least three letters per string, so a
  placeholder title such as "x" (seen from Opus 5.5) triggers a retry.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012vLHzh5qUhFtmJhzkcTbXL
This commit is contained in:
feco
2026-11-16 14:38:30 +00:00
co-authored by Claude Opus 5.5
parent e789e5a981
commit 269e3167e2
2 changed files with 44 additions and 15 deletions
+26 -13
View File
@@ -1,12 +1,13 @@
//! `worker compare-models`: writes the same natal analysis with several
//! models for a blind comparison.
//! `worker compare-models`: writes the same natal (or synastry) analysis with
//! several models for a blind comparison.
//!
//! ```text
//! OPENROUTER_API_KEY=... worker compare-models --chart natal.json \
//! --models minimax/minimax-m3,anthropic/claude-sonnet-5.5 [--language hu] [--out DIR]
//! ```
//!
//! `--chart` is a horoscope-api `/v1/natal` response. The models get shuffled
//! `--chart` is a horoscope-api `/v1/natal` response; `--synastry FILE` (instead
//! of `--chart`) takes a `/v1/synastry` response for a relationship analysis. The models get shuffled
//! letters: `A.md`, `B.md`, ... hold the analyses, `key.json` tells which
//! letter is which model (with tokens, cost and time). The output on the
//! console does not reveal the mapping.
@@ -22,26 +23,33 @@ use crate::analysis::openrouter::{DEFAULT_BASE_URL, OpenRouter};
use crate::analysis::{compact, prompt};
struct Args {
chart: PathBuf,
input: PathBuf,
kind: prompt::Kind,
models: Vec<String>,
language: String,
out: PathBuf,
}
fn parse(args: Vec<String>) -> anyhow::Result<Args> {
let (mut chart, mut models, mut language, mut out) = (None, Vec::new(), "hu".to_owned(), None);
let (mut chart, mut synastry, mut models, mut language, mut out) =
(None, None, Vec::new(), "hu".to_owned(), None);
let mut it = args.into_iter();
while let Some(flag) = it.next() {
let mut value = || it.next().with_context(|| format!("{flag} needs a value"));
match flag.as_str() {
"--chart" => chart = Some(PathBuf::from(value()?)),
"--synastry" => synastry = Some(PathBuf::from(value()?)),
"--models" => models = value()?.split(',').map(|m| m.trim().to_owned()).filter(|m| !m.is_empty()).collect(),
"--language" => language = value()?,
"--out" => out = Some(PathBuf::from(value()?)),
other => bail!("unknown argument '{other}'"),
}
}
let chart = chart.context("--chart is required")?;
let (input, kind) = match (chart, synastry) {
(Some(c), None) => (c, prompt::Kind::Natal),
(None, Some(s)) => (s, prompt::Kind::Synastry),
_ => bail!("give either --chart or --synastry"),
};
if models.is_empty() {
bail!("--models is required (comma separated OpenRouter model ids)");
}
@@ -52,7 +60,7 @@ fn parse(args: Vec<String>) -> anyhow::Result<Args> {
let stamp = time::OffsetDateTime::now_utc().unix_timestamp();
PathBuf::from(format!("model-compare-{stamp}"))
});
Ok(Args { chart, models, language, out })
Ok(Args { input, kind, models, language, out })
}
pub async fn main(args: Vec<String>) -> anyhow::Result<()> {
@@ -63,12 +71,16 @@ pub async fn main(args: Vec<String>) -> anyhow::Result<()> {
.context("OPENROUTER_API_KEY is not set")?;
let base_url = std::env::var("OPENROUTER_BASE_URL").unwrap_or_else(|_| DEFAULT_BASE_URL.into());
let raw = std::fs::read_to_string(&args.chart).with_context(|| format!("reading {}", args.chart.display()))?;
let mut chart: Value = serde_json::from_str(&raw).context("--chart is not JSON")?;
if chart["result"].is_object() {
chart = chart["result"].take();
let raw = std::fs::read_to_string(&args.input).with_context(|| format!("reading {}", args.input.display()))?;
let mut input: Value = serde_json::from_str(&raw).context("the input file is not JSON")?;
if input["result"].is_object() {
input = input["result"].take();
}
let built = prompt::build(prompt::Kind::Natal, &args.language, &compact::natal(&chart));
let data = match args.kind {
prompt::Kind::Synastry => compact::synastry(&input["person1"], &input["person2"], &input),
_ => compact::natal(&input),
};
let built = prompt::build(args.kind, &args.language, &data);
std::fs::create_dir_all(&args.out)?;
std::fs::write(
@@ -79,6 +91,7 @@ pub async fn main(args: Vec<String>) -> anyhow::Result<()> {
),
)?;
let kind = args.kind;
let mut models = args.models.clone();
models.shuffle(&mut rand::rng());
let llm = std::sync::Arc::new(OpenRouter::new(&base_url, &key, None));
@@ -103,7 +116,7 @@ pub async fn main(args: Vec<String>) -> anyhow::Result<()> {
match result {
Ok(mut c) => {
prompt::polish(&args.language, &mut c.content);
let md = prompt::to_markdown(prompt::Kind::Natal, &args.language, &c.content);
let md = prompt::to_markdown(kind, &args.language, &c.content);
let words = md.split_whitespace().count();
std::fs::write(args.out.join(format!("{letter}.md")), &md)?;
std::fs::write(args.out.join(format!("{letter}.json")), serde_json::to_string_pretty(&c.content)?)?;