diff --git a/src/cli.rs b/src/cli.rs index d8d8d78..36f60c4 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -431,7 +431,8 @@ The implementation is a process speaking the conformance ops protocol (as \ stdout, exit when stdin ends. It runs once, with a cleared environment, a \ time limit and an output cap. The report says which profile facts are \ proven and which cases failed, with byte diffs; the exit code is 1 on any \ -failure. +failure. `--json` prints the same report as JSON on stdout instead \ +(`format`, `implementation`, `encodings`, `totals`, `verdict`). `--corpus` runs a corpus laid out like ComlineProject/runtime's \ `runtime/conformance` (`profiles/`, `vectors/`) instead, for the official \ @@ -440,7 +441,8 @@ encodings. Examples: comline conformance --impl ./target/release/acme-lines-conformance comline conformance --impl \"node conformance.js\" --encoding com.acme.lines@1 - comline conformance --impl comline-reference --corpus runtime/conformance --encoding packed@1")] + comline conformance --impl comline-reference --corpus runtime/conformance --encoding packed@1 + comline conformance --impl ./target/release/acme-lines-conformance --json")] Conformance { /// The implementation's command line: a program and its arguments, split on whitespace #[arg(long = "impl", value_name = "COMMAND")] @@ -457,6 +459,10 @@ Examples: /// Seconds the implementation may take for every case together [default: 60] #[arg(long, value_name = "SECS")] timeout: Option, + + /// Print the report as JSON on stdout (same exit code) + #[arg(long)] + json: bool, }, /// Print a shell completion script to stdout diff --git a/src/commands/conformance.rs b/src/commands/conformance.rs index dbc0c75..7c8b33c 100644 --- a/src/commands/conformance.rs +++ b/src/commands/conformance.rs @@ -34,7 +34,7 @@ use comline_runtime_conformance::runner::{ self, judge, parse_reply, questions, Corpus, FactProof, OPS_PROTOCOL, }; use comline_runtime_conformance::{Answer, Case, Outcome, Profile}; -use miette::{miette, Result}; +use miette::{miette, IntoDiagnostic, Result}; use serde_json::{json, Value as Json}; use crate::commands::{core_err, ensure_project}; @@ -53,6 +53,7 @@ pub fn run( encodings: &[String], corpus: Option<&Path>, timeout: Option, + json: bool, ) -> Result<()> { let subjects = match corpus { Some(dir) => from_corpus(dir, encodings)?, @@ -89,11 +90,13 @@ pub fn run( input.push('\n'); } - ui::step(format!( - "Running `{implementation}` against {} case(s) of {} encoding(s)", - all.len(), - subjects.len() - )); + if !json { + ui::step(format!( + "Running `{implementation}` against {} case(s) of {} encoding(s)", + all.len(), + subjects.len() + )); + } let output = native::run(&program, args, &name, input.as_bytes(), timeout).map_err(|e| { if e.starts_with("didn't finish") { miette!( @@ -137,18 +140,115 @@ pub fn run( } let mut failures = 0; - for subject in &subjects { - failures += report(subject, &outcomes, claimed(&subject.profile.name)); + if json { + let mut encodings = Vec::new(); + for subject in &subjects { + let (entry, problems) = report_json(subject, &outcomes, claimed(&subject.profile.name)); + failures += problems; + encodings.push(entry); + } + let cases = all.len(); + let passed = all + .iter() + .filter(|c| outcomes.get(&c.id) == Some(&Outcome::Pass)) + .count(); + let report = json!({ + "format": 1, + "implementation": { + "command": implementation, + "version": version, + }, + "encodings": encodings, + "totals": { + "encodings": subjects.len(), + "cases": cases, + "passed": passed, + "problems": failures, + }, + "verdict": if failures == 0 { "conforms" } else { "fails" }, + }); + println!( + "{}", + serde_json::to_string_pretty(&report).into_diagnostic()? + ); + } else { + for subject in &subjects { + failures += report(subject, &outcomes, claimed(&subject.profile.name)); + } } if failures > 0 { return Err(miette!("conformance failed: {failures} problem(s)")); } - ui::success(format!( - "`{name}` conforms to every encoding it was run against" - )); + if !json { + ui::success(format!( + "`{name}` conforms to every encoding it was run against" + )); + } Ok(()) } +/// One encoding's results as JSON, and its number of problems (counted as +/// [`report`] does). +fn report_json( + subject: &Subject, + outcomes: &BTreeMap, + claimed: bool, +) -> (Json, usize) { + let profile = &subject.profile; + let mut problems = 0; + let case = |id: &str| subject.cases.iter().find(|c| c.id == id); + let facts: Vec = runner::prove(profile, case, outcomes) + .into_iter() + .map( + |FactProof { + fact, + value, + result, + }| { + if result.is_err() { + problems += 1; + } + json!({ + "fact": fact, + "value": value, + "proven": result.is_ok(), + "reason": result.err(), + }) + }, + ) + .collect(); + let cases: Vec = subject + .cases + .iter() + .map(|c| match &outcomes[&c.id] { + Outcome::Pass => json!({ "id": c.id, "outcome": "pass", "detail": null }), + Outcome::Fail(why) => { + problems += 1; + json!({ "id": c.id, "outcome": "fail", "detail": why }) + } + Outcome::NotImplemented(what) => { + problems += 1; + json!({ "id": c.id, "outcome": "not_implemented", "detail": what }) + } + }) + .collect(); + let passed = subject + .cases + .iter() + .filter(|c| outcomes.get(&c.id) == Some(&Outcome::Pass)) + .count(); + let entry = json!({ + "name": profile.name, + "kind": serde_json::to_value(profile.kind()).unwrap_or_default(), + "listed_by_implementation": claimed, + "passed": passed, + "total": subject.cases.len(), + "facts": facts, + "cases": cases, + }); + (entry, problems) +} + /// The answers on one reply line: the batch for case `i`. fn answers(line: &str, i: usize) -> std::result::Result, String> { let reply: Json = diff --git a/src/main.rs b/src/main.rs index e5248c8..555858f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -197,12 +197,14 @@ fn run(cli: Cli) -> Result<()> { encoding, corpus, timeout, + json, } => commands::conformance::run( &work_dir, &implementation, &encoding, corpus.as_deref(), timeout, + json, ), Commands::Completions { shell } => commands::completions::run(shell), Commands::Targets { features, json } => commands::targets::run(features, json), diff --git a/tests/cli/conformance.rs b/tests/cli/conformance.rs index 41cda3e..1729ab3 100644 --- a/tests/cli/conformance.rs +++ b/tests/cli/conformance.rs @@ -176,6 +176,78 @@ fn an_official_name_points_at_the_corpus() { .stderr(predicate::str::contains("--corpus")); } +#[test] +fn json_reports_a_passing_run() { + let output = comline_cmd() + .args([ + "conformance", + "--impl", + &implementation("reference"), + "--json", + ]) + .args(["--encoding", "packed@1", "--corpus"]) + .arg(corpus()) + .assert() + .success() + .get_output() + .stdout + .clone(); + let report: serde_json::Value = serde_json::from_slice(&output).expect("stdout is JSON"); + assert_eq!(report["format"], 1); + assert_eq!(report["verdict"], "conforms"); + assert_eq!(report["totals"]["problems"], 0); + assert_eq!(report["totals"]["cases"], report["totals"]["passed"]); + assert!(report["implementation"]["version"]["protocol"].is_number()); + let encoding = &report["encodings"][0]; + assert_eq!(encoding["name"], "packed@1"); + assert_eq!(encoding["kind"], "format"); + assert_eq!(encoding["listed_by_implementation"], true); + assert!(encoding["facts"] + .as_array() + .unwrap() + .iter() + .all(|f| f["proven"] == true)); + assert!(encoding["cases"] + .as_array() + .unwrap() + .iter() + .all(|c| c["outcome"] == "pass")); +} + +#[test] +fn json_reports_a_failing_run_and_still_exits_1() { + let assert = comline_cmd() + .args([ + "conformance", + "--impl", + &implementation("flipper"), + "--json", + ]) + .args(["--encoding", "datagram@1", "--corpus"]) + .arg(corpus()) + .assert() + .code(1); + let report: serde_json::Value = + serde_json::from_slice(&assert.get_output().stdout).expect("stdout is JSON"); + assert_eq!(report["format"], 1); + assert_eq!(report["verdict"], "fails"); + assert!(report["totals"]["problems"].as_u64().unwrap() > 0); + let encoding = &report["encodings"][0]; + let failed = encoding["cases"] + .as_array() + .unwrap() + .iter() + .find(|c| c["id"] == "framing-datagram/request") + .unwrap(); + assert_eq!(failed["outcome"], "fail"); + assert!(failed["detail"].as_str().unwrap().contains("encodes wrong")); + assert!(encoding["facts"] + .as_array() + .unwrap() + .iter() + .any(|f| f["fact"] == "calls" && f["proven"] == false && f["reason"].is_string())); +} + fn copy_dir_all(from: &Path, to: &Path) { std::fs::create_dir_all(to).unwrap(); for entry in std::fs::read_dir(from).unwrap() {