compare-models: synastry input; reject placeholder strings in answers
- worker compare-models --synastry FILE takes a /v1/synastry response and writes a relationship analysis. - Answer validation now needs at least three letters per string, so a placeholder title such as "x" (seen from Opus 5.5) triggers a retry. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012vLHzh5qUhFtmJhzkcTbXL
This commit is contained in:
@@ -170,7 +170,8 @@ pub fn build(kind: Kind, language: &str, chart_text: &str) -> Prompt {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Checks an answer against our schema subset (objects with required
|
/// Checks an answer against our schema subset (objects with required
|
||||||
/// properties, non-empty strings, string arrays with at least two items).
|
/// properties, strings with at least three letters, string arrays with at
|
||||||
|
/// least two items). A placeholder such as "x" counts as missing.
|
||||||
pub fn validate(schema: &Value, value: &Value) -> Result<(), String> {
|
pub fn validate(schema: &Value, value: &Value) -> Result<(), String> {
|
||||||
fn walk(schema: &Value, value: &Value, path: &str) -> Result<(), String> {
|
fn walk(schema: &Value, value: &Value, path: &str) -> Result<(), String> {
|
||||||
match schema["type"].as_str() {
|
match schema["type"].as_str() {
|
||||||
@@ -190,7 +191,7 @@ pub fn validate(schema: &Value, value: &Value) -> Result<(), String> {
|
|||||||
items.iter().enumerate().try_for_each(|(i, v)| walk(&schema["items"], v, &format!("{path}[{i}]")))
|
items.iter().enumerate().try_for_each(|(i, v)| walk(&schema["items"], v, &format!("{path}[{i}]")))
|
||||||
}
|
}
|
||||||
Some("string") => match value.as_str() {
|
Some("string") => match value.as_str() {
|
||||||
Some(s) if !s.trim().is_empty() => Ok(()),
|
Some(s) if s.chars().filter(|c| c.is_alphabetic()).nth(2).is_some() => Ok(()),
|
||||||
_ => Err(format!("{path}: expected a non-empty string")),
|
_ => Err(format!("{path}: expected a non-empty string")),
|
||||||
},
|
},
|
||||||
_ => Ok(()),
|
_ => Ok(()),
|
||||||
@@ -260,6 +261,21 @@ pub fn to_markdown(kind: Kind, language: &str, content: &Value) -> String {
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn validate_rejects_placeholder_titles() {
|
||||||
|
let schema = schema(Kind::Synastry);
|
||||||
|
let section = json!({"title": "Rák-szívek", "body": "Két bekezdés szöveg."});
|
||||||
|
let mut answer = json!({
|
||||||
|
"title": "Két Rák-szív", "summary": "Összefoglaló.", "advice": "Útravaló.",
|
||||||
|
"strengths": ["Egyik erősség.", "Másik erősség."], "challenges": ["Egy kihívás.", "Még egy."],
|
||||||
|
"sections": {"attraction": section, "communication": section, "emotional_bond": section,
|
||||||
|
"friction": section, "long_term": section},
|
||||||
|
});
|
||||||
|
assert_eq!(validate(&schema, &answer), Ok(()));
|
||||||
|
answer["sections"]["attraction"]["title"] = json!("x");
|
||||||
|
assert_eq!(validate(&schema, &answer), Err("$.sections.attraction.title: expected a non-empty string".into()));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn polish_turns_straight_quotes_hungarian() {
|
fn polish_turns_straight_quotes_hungarian() {
|
||||||
let mut v = json!({"a": "a \"hazai pálya\" és a \"sorok\"", "b": ["egy \" pár nélkül"], "c": {"d": "nincs"}});
|
let mut v = json!({"a": "a \"hazai pálya\" és a \"sorok\"", "b": ["egy \" pár nélkül"], "c": {"d": "nincs"}});
|
||||||
|
|||||||
@@ -1,12 +1,13 @@
|
|||||||
//! `worker compare-models`: writes the same natal analysis with several
|
//! `worker compare-models`: writes the same natal (or synastry) analysis with
|
||||||
//! models for a blind comparison.
|
//! several models for a blind comparison.
|
||||||
//!
|
//!
|
||||||
//! ```text
|
//! ```text
|
||||||
//! OPENROUTER_API_KEY=... worker compare-models --chart natal.json \
|
//! OPENROUTER_API_KEY=... worker compare-models --chart natal.json \
|
||||||
//! --models minimax/minimax-m3,anthropic/claude-sonnet-5.5 [--language hu] [--out DIR]
|
//! --models minimax/minimax-m3,anthropic/claude-sonnet-5.5 [--language hu] [--out DIR]
|
||||||
//! ```
|
//! ```
|
||||||
//!
|
//!
|
||||||
//! `--chart` is a horoscope-api `/v1/natal` response. The models get shuffled
|
//! `--chart` is a horoscope-api `/v1/natal` response; `--synastry FILE` (instead
|
||||||
|
//! of `--chart`) takes a `/v1/synastry` response for a relationship analysis. The models get shuffled
|
||||||
//! letters: `A.md`, `B.md`, ... hold the analyses, `key.json` tells which
|
//! letters: `A.md`, `B.md`, ... hold the analyses, `key.json` tells which
|
||||||
//! letter is which model (with tokens, cost and time). The output on the
|
//! letter is which model (with tokens, cost and time). The output on the
|
||||||
//! console does not reveal the mapping.
|
//! console does not reveal the mapping.
|
||||||
@@ -22,26 +23,33 @@ use crate::analysis::openrouter::{DEFAULT_BASE_URL, OpenRouter};
|
|||||||
use crate::analysis::{compact, prompt};
|
use crate::analysis::{compact, prompt};
|
||||||
|
|
||||||
struct Args {
|
struct Args {
|
||||||
chart: PathBuf,
|
input: PathBuf,
|
||||||
|
kind: prompt::Kind,
|
||||||
models: Vec<String>,
|
models: Vec<String>,
|
||||||
language: String,
|
language: String,
|
||||||
out: PathBuf,
|
out: PathBuf,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn parse(args: Vec<String>) -> anyhow::Result<Args> {
|
fn parse(args: Vec<String>) -> anyhow::Result<Args> {
|
||||||
let (mut chart, mut models, mut language, mut out) = (None, Vec::new(), "hu".to_owned(), None);
|
let (mut chart, mut synastry, mut models, mut language, mut out) =
|
||||||
|
(None, None, Vec::new(), "hu".to_owned(), None);
|
||||||
let mut it = args.into_iter();
|
let mut it = args.into_iter();
|
||||||
while let Some(flag) = it.next() {
|
while let Some(flag) = it.next() {
|
||||||
let mut value = || it.next().with_context(|| format!("{flag} needs a value"));
|
let mut value = || it.next().with_context(|| format!("{flag} needs a value"));
|
||||||
match flag.as_str() {
|
match flag.as_str() {
|
||||||
"--chart" => chart = Some(PathBuf::from(value()?)),
|
"--chart" => chart = Some(PathBuf::from(value()?)),
|
||||||
|
"--synastry" => synastry = Some(PathBuf::from(value()?)),
|
||||||
"--models" => models = value()?.split(',').map(|m| m.trim().to_owned()).filter(|m| !m.is_empty()).collect(),
|
"--models" => models = value()?.split(',').map(|m| m.trim().to_owned()).filter(|m| !m.is_empty()).collect(),
|
||||||
"--language" => language = value()?,
|
"--language" => language = value()?,
|
||||||
"--out" => out = Some(PathBuf::from(value()?)),
|
"--out" => out = Some(PathBuf::from(value()?)),
|
||||||
other => bail!("unknown argument '{other}'"),
|
other => bail!("unknown argument '{other}'"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
let chart = chart.context("--chart is required")?;
|
let (input, kind) = match (chart, synastry) {
|
||||||
|
(Some(c), None) => (c, prompt::Kind::Natal),
|
||||||
|
(None, Some(s)) => (s, prompt::Kind::Synastry),
|
||||||
|
_ => bail!("give either --chart or --synastry"),
|
||||||
|
};
|
||||||
if models.is_empty() {
|
if models.is_empty() {
|
||||||
bail!("--models is required (comma separated OpenRouter model ids)");
|
bail!("--models is required (comma separated OpenRouter model ids)");
|
||||||
}
|
}
|
||||||
@@ -52,7 +60,7 @@ fn parse(args: Vec<String>) -> anyhow::Result<Args> {
|
|||||||
let stamp = time::OffsetDateTime::now_utc().unix_timestamp();
|
let stamp = time::OffsetDateTime::now_utc().unix_timestamp();
|
||||||
PathBuf::from(format!("model-compare-{stamp}"))
|
PathBuf::from(format!("model-compare-{stamp}"))
|
||||||
});
|
});
|
||||||
Ok(Args { chart, models, language, out })
|
Ok(Args { input, kind, models, language, out })
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn main(args: Vec<String>) -> anyhow::Result<()> {
|
pub async fn main(args: Vec<String>) -> anyhow::Result<()> {
|
||||||
@@ -63,12 +71,16 @@ pub async fn main(args: Vec<String>) -> anyhow::Result<()> {
|
|||||||
.context("OPENROUTER_API_KEY is not set")?;
|
.context("OPENROUTER_API_KEY is not set")?;
|
||||||
let base_url = std::env::var("OPENROUTER_BASE_URL").unwrap_or_else(|_| DEFAULT_BASE_URL.into());
|
let base_url = std::env::var("OPENROUTER_BASE_URL").unwrap_or_else(|_| DEFAULT_BASE_URL.into());
|
||||||
|
|
||||||
let raw = std::fs::read_to_string(&args.chart).with_context(|| format!("reading {}", args.chart.display()))?;
|
let raw = std::fs::read_to_string(&args.input).with_context(|| format!("reading {}", args.input.display()))?;
|
||||||
let mut chart: Value = serde_json::from_str(&raw).context("--chart is not JSON")?;
|
let mut input: Value = serde_json::from_str(&raw).context("the input file is not JSON")?;
|
||||||
if chart["result"].is_object() {
|
if input["result"].is_object() {
|
||||||
chart = chart["result"].take();
|
input = input["result"].take();
|
||||||
}
|
}
|
||||||
let built = prompt::build(prompt::Kind::Natal, &args.language, &compact::natal(&chart));
|
let data = match args.kind {
|
||||||
|
prompt::Kind::Synastry => compact::synastry(&input["person1"], &input["person2"], &input),
|
||||||
|
_ => compact::natal(&input),
|
||||||
|
};
|
||||||
|
let built = prompt::build(args.kind, &args.language, &data);
|
||||||
|
|
||||||
std::fs::create_dir_all(&args.out)?;
|
std::fs::create_dir_all(&args.out)?;
|
||||||
std::fs::write(
|
std::fs::write(
|
||||||
@@ -79,6 +91,7 @@ pub async fn main(args: Vec<String>) -> anyhow::Result<()> {
|
|||||||
),
|
),
|
||||||
)?;
|
)?;
|
||||||
|
|
||||||
|
let kind = args.kind;
|
||||||
let mut models = args.models.clone();
|
let mut models = args.models.clone();
|
||||||
models.shuffle(&mut rand::rng());
|
models.shuffle(&mut rand::rng());
|
||||||
let llm = std::sync::Arc::new(OpenRouter::new(&base_url, &key, None));
|
let llm = std::sync::Arc::new(OpenRouter::new(&base_url, &key, None));
|
||||||
@@ -103,7 +116,7 @@ pub async fn main(args: Vec<String>) -> anyhow::Result<()> {
|
|||||||
match result {
|
match result {
|
||||||
Ok(mut c) => {
|
Ok(mut c) => {
|
||||||
prompt::polish(&args.language, &mut c.content);
|
prompt::polish(&args.language, &mut c.content);
|
||||||
let md = prompt::to_markdown(prompt::Kind::Natal, &args.language, &c.content);
|
let md = prompt::to_markdown(kind, &args.language, &c.content);
|
||||||
let words = md.split_whitespace().count();
|
let words = md.split_whitespace().count();
|
||||||
std::fs::write(args.out.join(format!("{letter}.md")), &md)?;
|
std::fs::write(args.out.join(format!("{letter}.md")), &md)?;
|
||||||
std::fs::write(args.out.join(format!("{letter}.json")), serde_json::to_string_pretty(&c.content)?)?;
|
std::fs::write(args.out.join(format!("{letter}.json")), serde_json::to_string_pretty(&c.content)?)?;
|
||||||
|
|||||||
Reference in New Issue
Block a user