diff --git a/bench/local/exact_replay.py b/bench/local/exact_replay.py new file mode 100644 index 0000000..43707ab --- /dev/null +++ b/bench/local/exact_replay.py @@ -0,0 +1,192 @@ +"""The exact tag (src/tiers.ts assertsExact) replayed over 2026-10-07's recorded "verified" runs. + + python3 exact_replay.py [--base REF] [--show N] [LANE ...] + +"Verified" now needs a passing independent check that compares the work's +output with an EXACT expected value; a run whose passing checks only test +properties (counts, sortedness, membership, types, two runs agreeing) gets +"passed-checks" with the reason "passed checks that test properties only". + +Population: every -molt-[-] tree under the lanes in +~/.cache/maat-bench/container-work last written on --day (default +2026-10-07), joined to its grader row. The grader row comes from the lane's +results file (named on the first line of ~/.cache/maat-bench/.log), or +its `.regraded.jsonl` twin when there is one: the regraded verdict wins. + +For each run whose last journalled tier is "verified": the checks that could +have carried the word are the passing (`pass`, `pass-vacuous`) sealed checks +in the last accepted receipt that the value rule at --base tags value and +that the worker did not write. The run keeps "verified" under the new rule +when one of them is also tagged exact (the working tree's assertsExact). A +"verified-audit" run keeps it when one of its accepted audit checks is exact. + +The rule ships only if it removes every grader-WRONG verified and keeps at +least two thirds of the grader-RIGHT ones. Two populations are reported: + + A. every recorded "verified" on the day, from any build; + B. those the base rule (--base) still grants: a carrying check whose + receipt names an independent author. Older builds labelled runs + verified on checks with no recorded author, which the base rule already + treats as the worker's; the exact rule cannot remove what is not there. + +B is the gate: it is what the exact rule changes on the build it ships on. +As shipped (2026-10-07 lanes): + + A right A wrong B right B wrong + recorded verified 113 8 79 2 + --literal-only kept 53 (47%) 0 48 (61%) 0 + shipped rule kept 61 (54%) 0 56 (71%) 0 + +The literal-only rule (equality with a literal, diff against literal text, +grep -x) failed the gate; the shipped rule adds two forms (containment of a +worked-out value in a program's output on literal input, and an oracle built +from the input files), and passes. In A, 11 right runs rest only on commands +that also carried a wrong run (the same `find -exec`, `|| echo fail` and +`print(len(header) == 5)` checks), so no command-level rule can keep them. +""" +from __future__ import annotations + +import argparse +import datetime +import json +import subprocess +import sys +from collections import defaultdict +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import value_replay as vr # noqa: E402 + +PASS = ("pass", "pass-vacuous") +_recorded_results = vr.results_file + + +def results_file(lane: Path) -> Path | None: + p = _recorded_results(lane) + if p: + g = p.with_name(p.name.replace(".jsonl", ".regraded.jsonl")) + if g.exists(): + return g + return p + + +def lanes_on(day: str, names: list[str]) -> list[Path]: + out = [] + for p in vr.CW.iterdir(): + if not p.is_dir(): + continue + if names and p.name not in names and not any(p.name.startswith(n + "-") for n in names): + continue + if datetime.date.fromtimestamp(p.stat().st_mtime).isoformat() == day: + out.append(p) + return sorted(out) + + +def tag(src: Path, fn: str, cmds: list[str], arg: str = "") -> dict[str, bool]: + js = ( + f"import {{ {fn} }} from {json.dumps(str(src))};" + f"let s='';process.stdin.on('data',d=>s+=d).on('end',()=>console.log(JSON.stringify(JSON.parse(s).map(r=>{fn}(r{arg})))));" + ) + p = subprocess.run(["node", "--no-warnings", "--input-type=module", "-e", js], input=json.dumps(cmds), capture_output=True, text=True) + if p.returncode: + sys.exit(f"{fn} from {src} failed: {p.stderr[-600:]}") + return dict(zip(cmds, json.loads(p.stdout))) + + +def worker_wrote(author: str | None) -> bool: + return bool(author) and author.startswith("the worker model") + + +def main(argv: list[str]) -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--base", default="origin/claude/value-classifier") + ap.add_argument("--day", default="2026-10-07") + ap.add_argument("--show", type=int, default=0, help="print the carrying checks of up to N kept and N removed right runs") + ap.add_argument("--literal-only", action="store_true", help="the rule as first measured: no containment (form 5), no oracle (form 6)") + ap.add_argument("lanes", nargs="*") + a = ap.parse_args(argv) + vr.results_file = results_file + vr.lanes = lambda _prefixes: lanes_on(a.day, a.lanes) + runs, checks = vr.collect([]) + by_run: dict[str, list[dict]] = defaultdict(list) + for c in checks: + by_run[c["run"]["id"]].append(c) + old_src = vr.classifier(a.base)["old"] + new_src = vr.REPO / "src/tiers.ts" + cmds = sorted({c["cmd"] for c in checks}) + value = tag(old_src, "assertsValue", cmds) + exact = tag(new_src, "assertsExact", cmds, ", { literalOnly: true }" if a.literal_only else "") + + rows = [] + for r in runs: + if r["tier"] not in ("verified", "verified-audit"): + continue + cs = by_run[r["id"]] + if r["tier"] == "verified": + carry = [c for c in cs if c["source"] == "sealed" and c.get("result") in PASS and value[c["cmd"]] and not worker_wrote(c.get("author"))] + else: + carry = [c for c in cs if c["source"] == "audit" and c.get("accepted")] + # Still verified at --base: a carrier whose independent author the receipt names (an unrecorded author is the worker's). + base = r["tier"] == "verified-audit" or any(vr.independent(c.get("author")) for c in carry) + keep = [c for c in carry if exact[c["cmd"]] and (base and (r["tier"] == "verified-audit" or vr.independent(c.get("author"))) or not base)] + rows.append({"run": r, "carry": carry, "keep": keep, "base": base}) + + print(f"runs with a grader row: {len(runs)} recorded verified: {sum(x['run']['tier'] == 'verified' for x in rows)}" + f" verified-audit: {sum(x['run']['tier'] == 'verified-audit' for x in rows)}") + print(f"distinct check commands: {len(cmds)} value (base): {sum(value.values())} exact (new): {sum(exact.values())}" + f" exact but not value (never counted): {sum(exact[c] and not value[c] for c in cmds)}") + + # A command-level rule cannot tell two runs apart that rest on the same commands: a right run whose + # carriers all also carried a WRONG run is removed by any rule that removes every wrong one. + wrong_cmds = {c["cmd"] for x in rows if not x["run"]["passed"] for c in x["carry"]} + for x in rows: + x["forced"] = x["run"]["passed"] and all(c["cmd"] in wrong_cmds for c in x["carry"]) + + verdict = {} + for scope, sel in (("A. every recorded verified (any build)", lambda x: True), + ("B. those --base still grants (an independent author on the receipt)", lambda x: x["base"])): + xs = [x for x in rows if sel(x)] + right = [x for x in xs if x["run"]["passed"]] + wrong = [x for x in xs if not x["run"]["passed"]] + kept_right = sum(bool(x["keep"]) for x in right) + kept_wrong = sum(bool(x["keep"]) for x in wrong) + forced = sum(x["forced"] for x in right) + print(f"\n== {scope}") + print(f" {'':16} {'total':>6} {'kept':>6} {'removed':>8}") + print(f" {'grader-right':16} {len(right):6} {kept_right:6} {len(right) - kept_right:8}") + print(f" {'grader-wrong':16} {len(wrong):6} {kept_wrong:6} {len(wrong) - kept_wrong:8}") + kept_all = kept_right + kept_wrong + print(f" precision of 'verified': {len(right)}/{len(xs)} = {len(right) / max(1, len(xs)):.1%} recorded -> " + f"{kept_right}/{kept_all} = {kept_right / max(1, kept_all):.1%} under the exact rule") + print(f" right runs whose every carrying command also carried a wrong run (no command rule can keep them): {forced}") + ok = kept_wrong == 0 and 3 * kept_right >= 2 * len(right) + print(f" gate: wrong removed {len(wrong) - kept_wrong}/{len(wrong)}, right kept {kept_right}/{len(right)}" + f" ({kept_right / max(1, len(right)):.0%}, need >= 67%) -> {'PASS' if ok else 'FAIL'}") + verdict[scope[0]] = ok + ok = verdict["B"] + right = [x for x in rows if x["run"]["passed"]] + wrong = [x for x in rows if not x["run"]["passed"]] + + print("\n-- grader-wrong verifieds") + for x in wrong: + print(f" {'KEPT ' if x['keep'] else 'removed'} {x['run']['id']}{'' if x['base'] else ' (not granted at --base)'}") + for c in x["keep"] or x["carry"]: + print(f" [{'exact' if exact[c['cmd']] else 'property'}] {c['cmd'][:200]!r}") + print("\n-- grader-right verifieds removed") + for x in right: + if x["keep"]: + continue + print(f" removed {x['run']['id']}{'' if x['base'] else ' (not granted at --base)'}{' (forced)' if x['forced'] else ''}") + for c in x["carry"][: max(a.show, 2)]: + print(f" [property] {c['cmd'][:200]!r}") + if a.show: + print("\n-- grader-right verifieds kept (the exact check that carries each)") + for x in right: + if x["keep"]: + print(f" kept {x['run']['id']}: {x['keep'][0]['cmd'][:200]!r}") + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/src/ask.ts b/src/ask.ts index 3dffb8e..650c502 100644 --- a/src/ask.ts +++ b/src/ask.ts @@ -113,7 +113,10 @@ async function askSized(opts: AskOptions): Promise { // never started the answer. Once, with room: a model that empties a four // times larger ceiling the same way is not going to answer, and the second // failure says exactly that instead of "the reply was empty". - if (first.ok && first.cutOff && first.text.trim() === "" && (leftMs(opts.deadlineAt) ?? 1) > 0) { + // Cut off with text written is retried the same way: a draft of exact, + // edge-case checks is longer than the ceiling allowed for, and half a JSON + // reply proposes nothing (2026-10-07: 3 of 20 runs drafted no checks). + if (first.ok && first.cutOff && (leftMs(opts.deadlineAt) ?? 1) > 0) { const bigger = (opts.maxTokens ?? ASK_MAX_TOKENS) * ASK_RETRY_FACTOR; const second = await askOnce(opts, bigger); if (second.ok && second.cutOff && second.text.trim() === "") { diff --git a/src/criteria.ts b/src/criteria.ts index 7a47c9b..b06772c 100644 --- a/src/criteria.ts +++ b/src/criteria.ts @@ -44,7 +44,12 @@ import type { CheckAuthor } from "./types.js"; * appears, it compiles) rather than running the deliverable. Kept, but such a * check alone cannot carry a "verified" once the ones that ran it are retired. */ -export type DraftedCheck = { name: string; run: string; surface?: true }; +/** + * A drafted check. `expect` is where its expected output comes from: the task + * words that state it, or the steps that derive it from a rule the task states. + * Shown to the critic; it never runs. + */ +export type DraftedCheck = { name: string; run: string; surface?: true; expect?: string }; export type Draft = { checks: DraftedCheck[]; notes: string[]; @@ -71,6 +76,8 @@ export const CRITERIA_MAX_NAME = 40; */ export const CRITERIA_MAX_RUN = 1_000; export const CRITERIA_MAX_NOTE = 200; +/** The longest `expect` (where a check's expected output comes from) kept on a drafted check. */ +export const CRITERIA_MAX_EXPECT = 300; /** * Shape a renderer-supplied payload into checks the engine may run. @@ -100,6 +107,7 @@ export function sanitizeCriteria(raw: unknown): Draft { name: c.name.trim().slice(0, CRITERIA_MAX_NAME), run: c.run.trim(), ...(c.surface === true ? { surface: true as const } : {}), + ...(typeof c.expect === "string" && c.expect.trim() ? { expect: c.expect.trim().slice(0, CRITERIA_MAX_EXPECT) } : {}), })) : []; const notes: string[] = Array.isArray(o.notes) @@ -500,13 +508,13 @@ export function drafterInputsHash(d: DrafterInputs): string { return h.digest("hex").slice(0, 16); } -const SYSTEM = [ +export const DRAFT_SYSTEM = [ "You draft acceptance criteria for one coding task. You are not doing the task", "and you will not judge whether it was done — a person approves what you write", "and it is sealed before the work starts.", "", "Return JSON only, matching:", - ' {"checks":[{"name":"kebab-name","run":"shell command"}],"notes":["sentence"]}', + ' {"checks":[{"name":"kebab-name","run":"shell command","expect":"where the expected output comes from"}],"notes":["sentence"]}', "", "checks are commands that MUST already work in this project. Prefer the scripts", "listed below verbatim, optionally narrowed (npm test -- ). Never invent", @@ -525,16 +533,22 @@ const SYSTEM = [ "present. A task that says accuracy above 0.62 gets a check of accuracy above 0.62.", "At least one check must run the thing and test its result; a check that only", "asks whether a file exists, a name appears, or output parses shows nothing.", + "For each thing the task says the work must do, at least one check runs it on a", + "concrete input and compares what it produces with the EXACT expected output:", + "[ \"$(python3 tool.py 3 4)\" = \"7\" ], assert f('a b') == 'a-b', a diff against a", + "heredoc, or grep -qx of a whole expected line. Work the expected output out from", + "the task text only: a value it states, or one you get by applying a rule it states", + "to an input your check picks, step by step. Put where it comes from in the check's", + "\"expect\": the task words quoted, or the derivation (\"*/15 from 00:07: 00:15,", + "00:30, 00:45\"). A property (the line count, sorted, each minute a multiple of 15,", + "a prefix, two runs agreeing) holds for wrong output too: it may come with an exact", + "check, never instead of one. Pick inputs at the edges: a boundary, empty input, a", + "value just outside a rule (1.2.3.4.5 for an IP rule), a start that is not on a step,", + "and each side of an either/or rule.", "When the right answer cannot be known before the work (a fitted value, a model's", "accuracy on held-out data, the fastest query), check the closest thing the task", "provides instead: the stated threshold on the data that IS here (accuracy >= 0.62", - "on the provided test split), the fit's error against the measured points, the new", - "query's time against the original's. Never hard-code an answer you had to guess.", - "Never write an expected value you worked out yourself — a date, a total, a count,", - "an output line the task does not state. Your own arithmetic is where checks go", - "wrong. Expected values come only from the task text; otherwise test what the task", - "states as a property: the stated number of lines, each strictly after the start,", - "the stated format, totals that add up, the same answer from two equivalent inputs.", + "on the provided test split). Never hard-code an answer you had to guess.", "Checks must leave nothing behind: build and write into $(mktemp -d), never into", "a directory the task names.", "Every path is relative to the working directory shown below. Never write an", @@ -545,7 +559,7 @@ const SYSTEM = [ "feels. They are recorded on the receipt as stated intent and never reported as", "verified. Do not write a note that pretends to be a check.", "", - "Two or three checks and at most two notes. Fewer is better. If the task needs", + "Two to four checks and at most two notes. If the task needs", "no criterion beyond the project's own bar, return empty lists.", ].join("\n"); @@ -751,7 +765,7 @@ export async function draftCriteria(opts: { baseUrl: opts.baseUrl, apiKey: opts.apiKey, model: opts.model, - system: SYSTEM, + system: DRAFT_SYSTEM, prompt: prompted, cwd: askCwd, what: "drafting criteria", @@ -772,7 +786,7 @@ export async function draftCriteria(opts: { baseUrl: opts.baseUrl, apiKey: opts.apiKey, model: opts.model, - system: SYSTEM, + system: DRAFT_SYSTEM, prompt: `${prompted}\n\nYour last reply could not be parsed as JSON. Reply with the JSON object only.`, cwd: askCwd, what: "drafting criteria", @@ -797,7 +811,7 @@ export async function draftCriteria(opts: { baseUrl: opts.baseUrl, apiKey: opts.apiKey, model: opts.model, - system: SYSTEM, + system: DRAFT_SYSTEM, prompt: context, cwd: askCwd, what: "drafting criteria", @@ -850,7 +864,8 @@ export const CRITIC_SYSTEM = [ " only have guessed (a model name, a count, a result the work must find out).", ' "surface" — it only looks: a file exists, a word appears in source, it compiles, --help works.', ' "runs" — it runs the deliverable on an input and checks what it produces.', - "For invents and guesses, quote the task words the check gets wrong, verbatim.", + "For invents and guesses, quote the task words the check gets wrong, verbatim. A value", + "derived step by step from a rule the task states, shown in the check's expect, is not a guess.", "", "Then list hard requirements no check would catch if the work got them wrong. A hard", "requirement is a concrete fact a shell command could test on this machine, right now:", @@ -869,6 +884,12 @@ export const CRITIC_SYSTEM = [ ' "uncovered":["verbatim task words"], "requirements":["verbatim task words"]}', ].join("\n"); +/** What the critic reads: the task, the project view, and each check with where its expected output comes from. */ +export function criticPrompt(task: string, view: string | undefined, checks: readonly DraftedCheck[]): string { + const line = (c: DraftedCheck) => `- ${c.name}: ${c.run}${c.expect ? `\n expect: ${c.expect}` : ""}`; + return `TASK TEXT:\n${task}\n\n${view ? `${view}\n\n` : ""}CHECKS:\n${checks.map(line).join("\n")}`; +} + export type Critique = { kept: DraftedCheck[]; dropped: { name: string; verdict: "invents" | "guesses"; quote: string }[]; @@ -1132,7 +1153,7 @@ async function critiqued( apiKey: opts.apiKey, model: opts.model, system: CRITIC_SYSTEM, - prompt: `TASK TEXT:\n${opts.task}\n\n${view ? `${view}\n\n` : ""}CHECKS:\n${d.checks.map((c) => `- ${c.name}: ${c.run}`).join("\n")}`, + prompt: criticPrompt(opts.task, view, d.checks), cwd: opts.askCwd ?? opts.cwd, what: "reviewing the drafted checks", fetchFn: opts.fetchFn, diff --git a/src/post-audit.ts b/src/post-audit.ts index 65fecef..c1f0bd0 100644 --- a/src/post-audit.ts +++ b/src/post-audit.ts @@ -41,7 +41,7 @@ import { parseLenient } from "./lenient-json.js"; import { interfaceView, isCode, type Changed } from "./interface-view.js"; import { bashPath, runCommand } from "./run.js"; import { copyTree } from "./scratch.js"; -import { assertsValue } from "./tiers.js"; +import { assertsExact, assertsValue } from "./tiers.js"; export const AUDIT_MAX_CHECKS = 4; export const AUDIT_MIN_QUOTE = 8; @@ -54,32 +54,39 @@ export const AUDIT_SYSTEM = [ "it added or changed, their signatures, how to run them, and the project listing.", "", "Write 2 to 4 shell checks. Each runs the deliverable (a script, a function, a query) on an input", - "and compares what it produces with an expected value, exiting non-zero on a mismatch. Almost", + "and compares what it produces with the EXACT expected output, exiting non-zero on a mismatch:", + "equality with a literal, a diff against a heredoc, or grep -x of a whole expected line. A count,", + "an order, a format, a prefix or two runs agreeing is a property wrong output also has: such a", + "check is dropped. Each requirement the task states gets at least one exact check. Almost", "every task states something checkable: an output for a given input, a format, a count, a rule", "(\"blank lines are ignored\", \"sorted by total, highest first\"), a file it must write.", "Return an empty list only when the task states nothing a command could test.", "", "The expected value must come from the task text: either stated outright, or the result of", "applying a rule the task states to a small input your check builds itself (write it under", - "$(mktemp -d)). For each check give `quote`: the task words that state that value or rule,", + "$(mktemp -d)). Pick inputs at the edges: a boundary, empty input, a value just outside a rule", + "(1.2.3.4.5 for an IP rule), a start that is not on a step, and each side of an either/or rule.", + "For each check give `quote`: the task words that state that value or rule,", "copied character for character from the task text (a short span is best, 5 to 20 words).", "A quote that is not word for word in the task, even a close paraphrase, drops the check. Never", - "use a value you would only know by running the work or by guessing.", + "use a value you would only know by running the work or by guessing. When the value is derived,", + "give the steps in `expect` (\"*/15 from 00:07: 00:15, 00:30, 00:45\").", "", "Rules: one line each, under 500 characters, run under bash from the project root on a", "throwaway copy. A check that only prints is useless: end every check in a comparison, e.g.", " [ \"$(python3 tool.py 3 4)\" = \"7\" ]", " python3 -c \"import json; d=json.load(open('out.json')); assert d['total'] == 12, d\"", - " [ \"$(grep -c '^ERROR' report.txt)\" -eq 3 ]", + " python3 tool.py 'x y' | diff - <(printf 'x-y\\n')", "A wrong result must make the check exit non-zero: no `|| true`, no `|| echo`, no", "printing PASS/FAIL with exit 0. Do not write into the project; scratch files go under", "$(mktemp -d). Use the file names, flags and signatures shown; do not invent others.", "", "Reply with JSON only:", - '{"checks":[{"name":"kebab-name","run":"shell command","quote":"verbatim task words"}]}', + '{"checks":[{"name":"kebab-name","run":"shell command","quote":"verbatim task words","expect":"how the expected output follows"}]}', ].join("\n"); -export type AuditCheck = { name: string; run: string; quote: string }; +/** `expect`: how the expected output follows from the quote, when it is derived rather than stated. */ +export type AuditCheck = { name: string; run: string; quote: string; expect?: string }; export type AuditDrop = { name: string; run: string; why: string }; /** The judge's reply as checks, with every ungrounded or malformed one dropped and said. */ @@ -116,6 +123,7 @@ export function parseAuditChecks(reply: string, task: string): { kept: AuditChec let name = String(o.name ?? "").trim().replace(/\s+/g, "-").slice(0, CRITERIA_MAX_NAME) || `audit-${kept.length + dropped.length + 1}`; const run = String(o.run ?? "").trim(); const quote = String(o.quote ?? "").trim(); + const expect = typeof o.expect === "string" ? o.expect.trim().slice(0, 300) : ""; while (seen.has(name)) name = `${name}-2`; seen.add(name); if (!run) dropped.push({ name, run, why: "no command" }); @@ -124,7 +132,7 @@ export function parseAuditChecks(reply: string, task: string): { kept: AuditChec else if (quote.replace(/\s+/g, " ").length < AUDIT_MIN_QUOTE || !quotedIn(quote, task)) dropped.push({ name, run, why: `its quote is not in the task text: "${quote.slice(0, 80)}"` }); else if (kept.length >= AUDIT_MAX_CHECKS) dropped.push({ name, run, why: `over ${AUDIT_MAX_CHECKS} checks` }); - else kept.push({ name, run, quote }); + else kept.push({ name, run, quote, ...(expect ? { expect } : {}) }); } return { kept, dropped }; } @@ -508,6 +516,7 @@ export async function auditGates( const never = cannotFail(c.run); if (never) return no("L16-cannot-fail", never); if (!assertsValue(c.run)) return no("V-no-value", "it does not assert an expected value"); + if (!assertsExact(c.run)) return no("V-property-only", "it tests a property of the output (a count, an order, a format, membership, two runs agreeing), not an exact expected value"); const work = await runOnCopy(ctx.workDir, c.run, { timeoutMs: ctx.timeoutMs, signal: ctx.signal }); if (work.printedFail) return no("L16-printed-fail", "it printed FAIL and still exited 0 on the work", { work: work.status }); if (work.status !== "pass") return no("A-fails-on-work", `it does not pass on the work (${work.status}): ${work.output.split("\n").slice(-1)[0]?.slice(0, 160) ?? ""}`, { work: work.status }); diff --git a/src/reference.ts b/src/reference.ts index 579989d..cf57565 100644 --- a/src/reference.ts +++ b/src/reference.ts @@ -486,8 +486,9 @@ async function writeAndTry(opts: Ask, prompt: string): Promise return assertsBehaviour(run); } -/** The tags a drafted check carries: surface from the critic, value from the command. */ +// ---------------------------------------------------------------- the exact tag + +/** + * Does this command compare the deliverable's output with an EXACT expected + * value, or only test a property of it? + * + * The 2026-10-07 lanes earned a wrong "verified" on cron-next with four + * passing judge checks that all tested properties: 8 lines, each minute a + * multiple of 15, sorted, unique; "every date is the 13th or a Friday"; two + * runs of the worker agree; "starts with 2024-06-1". Output that skips every + * other occurrence has all of those properties. An exact check would not + * have passed: from 00:07, `*\/15` gives 00:15 first, and the work gave 00:45. + * + * Exact, any one of these: + * 1. An equality (`==`, `===`, `-eq`, `=` in a test bracket, a failing `if + * ... != ...` / `!==`, `assertEqual`/`strictEqual`/`deepEqual`) between + * what the work produced and a literal: a number, a quoted string, a + * list/dict/set of literals, true/false/null, `"$(printf '...')"`, or a + * name bound to one (a table of cases). Asserted: under `assert`, `if ...: + * exit/raise/throw`, `exit(0 if ...)`, a test bracket, or `jq -e`. + * 2. `diff`/`cmp` against a heredoc, a here-string, `<(printf ...)` / + * `<(echo ...)`, or an expected file (the tier's pre-work record still + * has to vouch for the file: `valueUnproven`). + * 3. `grep -x` (or a `^...$` pattern) of a literal line, with no regex + * wildcards unless `-F`, over the work's output or a file it wrote. + * 4. ` is None`, asserted. + * 5. A specific worked-out value with a digit in it found in what a program + * printed on literal input: `nextrun.py '0 9 * * *' '2023-01-01 08:00' 1 + * | grep -q '2023-01-01 09:00'` (containsValue). + * 6. An oracle: the work's output equal to a value the check built from the + * task's input files without running anything (oracleEquality). + * 7. An exit or HTTP status other than 0/200 the task names (`rc=$?; [ $rc + * -eq 2 ]`), not read only on the path where the command failed. + * Forms 5 and 6 narrow the rule as first measured (literal equality, diff, + * grep -x only): over the 2026-10-07 lanes that kept 48 of 79 grader-right + * verifieds the base rule grants, under the two-thirds bar; with them, 56 of + * 79, and every grader-wrong verified is still removed + * (bench/local/exact_replay.py). + * Property, never exact, whatever the literal: + * - counts and sizes: `len()`, `.length`, `wc`, `grep -c`, `count`, `sum()`, + * jq `length`; + * - types and shapes: `type()`, `typeof`, `isinstance`, `keys`; + * - modulo (`% 15 == 0`), prefixes/suffixes and slices, membership in a + * file, a format pattern, ordering; + * - exit status 0 and HTTP 200: they say only that it ran; + * - the same constant compared across every item (`all(...)`, `any(...)`, + * a comprehension, a `for` loop, `find -exec`, `xargs`, jq `.[]`): each + * item having the property is still a property; + * - two unknowns compared (`[ "$a" = "$b" ]`, `L == sorted(set(L))`, two + * runs of the work agreeing); + * - a command that cannot fail (`... || echo fail`, `...; true`, `print(x == 1)`); + * - a pattern the work wrote, tried on lines the check chose (`re.search(p, ...)`). + * A constant compared inside a loop over a literal table of cases + * (`for inp, want in CASES: assert f(inp) == want`) is exact: `want` changes + * with the input. + * + * `literalOnly` turns forms 5 and 6 off: the rule as first measured, for + * bench/local/exact_replay.py. + */ +export function assertsExact(run: string, opts: { literalOnly?: boolean } = {}): boolean { + // A pattern the work wrote, tried on lines the check chose: the judge's own examples + // missed the grader's near-miss (see "Not widened" under assertsValue). + if (cannotFail(run) || PATTERN_DELIVERABLE.test(run)) return false; + const names = exactNames(run); + const narrow = !opts.literalOnly; + if (exactEquality(run, names, narrow) || exactHelper(run, names) || exactDiff(run) || exactGrepLine(run) || exactNone(run, names) || (narrow && containsValue(run))) return true; + // `bash -c '