Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions src/cli.rs
Original file line number Diff line number Diff line change
Expand Up @@ -431,7 +431,8 @@ The implementation is a process speaking the conformance ops protocol (as \
stdout, exit when stdin ends. It runs once, with a cleared environment, a \
time limit and an output cap. The report says which profile facts are \
proven and which cases failed, with byte diffs; the exit code is 1 on any \
failure.
failure. `--json` prints the same report as JSON on stdout instead \
(`format`, `implementation`, `encodings`, `totals`, `verdict`).

`--corpus` runs a corpus laid out like ComlineProject/runtime's \
`runtime/conformance` (`profiles/`, `vectors/`) instead, for the official \
Expand All @@ -440,7 +441,8 @@ encodings.
Examples:
comline conformance --impl ./target/release/acme-lines-conformance
comline conformance --impl \"node conformance.js\" --encoding com.acme.lines@1
comline conformance --impl comline-reference --corpus runtime/conformance --encoding packed@1")]
comline conformance --impl comline-reference --corpus runtime/conformance --encoding packed@1
comline conformance --impl ./target/release/acme-lines-conformance --json")]
Conformance {
/// The implementation's command line: a program and its arguments, split on whitespace
#[arg(long = "impl", value_name = "COMMAND")]
Expand All @@ -457,6 +459,10 @@ Examples:
/// Seconds the implementation may take for every case together [default: 60]
#[arg(long, value_name = "SECS")]
timeout: Option<u64>,

/// Print the report as JSON on stdout (same exit code)
#[arg(long)]
json: bool,
},

/// Print a shell completion script to stdout
Expand Down
122 changes: 111 additions & 11 deletions src/commands/conformance.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ use comline_runtime_conformance::runner::{
self, judge, parse_reply, questions, Corpus, FactProof, OPS_PROTOCOL,
};
use comline_runtime_conformance::{Answer, Case, Outcome, Profile};
use miette::{miette, Result};
use miette::{miette, IntoDiagnostic, Result};
use serde_json::{json, Value as Json};

use crate::commands::{core_err, ensure_project};
Expand All @@ -53,6 +53,7 @@ pub fn run(
encodings: &[String],
corpus: Option<&Path>,
timeout: Option<u64>,
json: bool,
) -> Result<()> {
let subjects = match corpus {
Some(dir) => from_corpus(dir, encodings)?,
Expand Down Expand Up @@ -89,11 +90,13 @@ pub fn run(
input.push('\n');
}

ui::step(format!(
"Running `{implementation}` against {} case(s) of {} encoding(s)",
all.len(),
subjects.len()
));
if !json {
ui::step(format!(
"Running `{implementation}` against {} case(s) of {} encoding(s)",
all.len(),
subjects.len()
));
}
let output = native::run(&program, args, &name, input.as_bytes(), timeout).map_err(|e| {
if e.starts_with("didn't finish") {
miette!(
Expand Down Expand Up @@ -137,18 +140,115 @@ pub fn run(
}

let mut failures = 0;
for subject in &subjects {
failures += report(subject, &outcomes, claimed(&subject.profile.name));
if json {
let mut encodings = Vec::new();
for subject in &subjects {
let (entry, problems) = report_json(subject, &outcomes, claimed(&subject.profile.name));
failures += problems;
encodings.push(entry);
}
let cases = all.len();
let passed = all
.iter()
.filter(|c| outcomes.get(&c.id) == Some(&Outcome::Pass))
.count();
let report = json!({
"format": 1,
"implementation": {
"command": implementation,
"version": version,
},
"encodings": encodings,
"totals": {
"encodings": subjects.len(),
"cases": cases,
"passed": passed,
"problems": failures,
},
"verdict": if failures == 0 { "conforms" } else { "fails" },
});
println!(
"{}",
serde_json::to_string_pretty(&report).into_diagnostic()?
);
} else {
for subject in &subjects {
failures += report(subject, &outcomes, claimed(&subject.profile.name));
}
}
if failures > 0 {
return Err(miette!("conformance failed: {failures} problem(s)"));
}
ui::success(format!(
"`{name}` conforms to every encoding it was run against"
));
if !json {
ui::success(format!(
"`{name}` conforms to every encoding it was run against"
));
}
Ok(())
}

/// One encoding's results as JSON, and its number of problems (counted as
/// [`report`] does).
fn report_json(
subject: &Subject,
outcomes: &BTreeMap<String, Outcome>,
claimed: bool,
) -> (Json, usize) {
let profile = &subject.profile;
let mut problems = 0;
let case = |id: &str| subject.cases.iter().find(|c| c.id == id);
let facts: Vec<Json> = runner::prove(profile, case, outcomes)
.into_iter()
.map(
|FactProof {
fact,
value,
result,
}| {
if result.is_err() {
problems += 1;
}
json!({
"fact": fact,
"value": value,
"proven": result.is_ok(),
"reason": result.err(),
})
},
)
.collect();
let cases: Vec<Json> = subject
.cases
.iter()
.map(|c| match &outcomes[&c.id] {
Outcome::Pass => json!({ "id": c.id, "outcome": "pass", "detail": null }),
Outcome::Fail(why) => {
problems += 1;
json!({ "id": c.id, "outcome": "fail", "detail": why })
}
Outcome::NotImplemented(what) => {
problems += 1;
json!({ "id": c.id, "outcome": "not_implemented", "detail": what })
}
})
.collect();
let passed = subject
.cases
.iter()
.filter(|c| outcomes.get(&c.id) == Some(&Outcome::Pass))
.count();
let entry = json!({
"name": profile.name,
"kind": serde_json::to_value(profile.kind()).unwrap_or_default(),
"listed_by_implementation": claimed,
"passed": passed,
"total": subject.cases.len(),
"facts": facts,
"cases": cases,
});
(entry, problems)
}

/// The answers on one reply line: the batch for case `i`.
fn answers(line: &str, i: usize) -> std::result::Result<Vec<Answer>, String> {
let reply: Json =
Expand Down
2 changes: 2 additions & 0 deletions src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -197,12 +197,14 @@ fn run(cli: Cli) -> Result<()> {
encoding,
corpus,
timeout,
json,
} => commands::conformance::run(
&work_dir,
&implementation,
&encoding,
corpus.as_deref(),
timeout,
json,
),
Commands::Completions { shell } => commands::completions::run(shell),
Commands::Targets { features, json } => commands::targets::run(features, json),
Expand Down
72 changes: 72 additions & 0 deletions tests/cli/conformance.rs
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,78 @@ fn an_official_name_points_at_the_corpus() {
.stderr(predicate::str::contains("--corpus"));
}

#[test]
fn json_reports_a_passing_run() {
let output = comline_cmd()
.args([
"conformance",
"--impl",
&implementation("reference"),
"--json",
])
.args(["--encoding", "packed@1", "--corpus"])
.arg(corpus())
.assert()
.success()
.get_output()
.stdout
.clone();
let report: serde_json::Value = serde_json::from_slice(&output).expect("stdout is JSON");
assert_eq!(report["format"], 1);
assert_eq!(report["verdict"], "conforms");
assert_eq!(report["totals"]["problems"], 0);
assert_eq!(report["totals"]["cases"], report["totals"]["passed"]);
assert!(report["implementation"]["version"]["protocol"].is_number());
let encoding = &report["encodings"][0];
assert_eq!(encoding["name"], "packed@1");
assert_eq!(encoding["kind"], "format");
assert_eq!(encoding["listed_by_implementation"], true);
assert!(encoding["facts"]
.as_array()
.unwrap()
.iter()
.all(|f| f["proven"] == true));
assert!(encoding["cases"]
.as_array()
.unwrap()
.iter()
.all(|c| c["outcome"] == "pass"));
}

#[test]
fn json_reports_a_failing_run_and_still_exits_1() {
let assert = comline_cmd()
.args([
"conformance",
"--impl",
&implementation("flipper"),
"--json",
])
.args(["--encoding", "datagram@1", "--corpus"])
.arg(corpus())
.assert()
.code(1);
let report: serde_json::Value =
serde_json::from_slice(&assert.get_output().stdout).expect("stdout is JSON");
assert_eq!(report["format"], 1);
assert_eq!(report["verdict"], "fails");
assert!(report["totals"]["problems"].as_u64().unwrap() > 0);
let encoding = &report["encodings"][0];
let failed = encoding["cases"]
.as_array()
.unwrap()
.iter()
.find(|c| c["id"] == "framing-datagram/request")
.unwrap();
assert_eq!(failed["outcome"], "fail");
assert!(failed["detail"].as_str().unwrap().contains("encodes wrong"));
assert!(encoding["facts"]
.as_array()
.unwrap()
.iter()
.any(|f| f["fact"] == "calls" && f["proven"] == false && f["reason"].is_string()));
}

fn copy_dir_all(from: &Path, to: &Path) {
std::fs::create_dir_all(to).unwrap();
for entry in std::fs::read_dir(from).unwrap() {
Expand Down
Loading