diff --git a/bench/local/value_replay.py b/bench/local/value_replay.py new file mode 100644 index 0000000..1c3c28e --- /dev/null +++ b/bench/local/value_replay.py @@ -0,0 +1,459 @@ +"""The value tag (src/tiers.ts assertsValue) on 2026-10-07's checks: what it rejects, and what widening it flips. + + python3 value_replay.py [--base REF] [--samples N] [--audit-gates --image maat-bench:-u] [LANE_PREFIX ...] + +"Verified" needs a passing independent check that runs the work AND asserts an +expected value; the post-work audit drops a check outright with rule +V-no-value. Both read assertsValue. This replays it, offline, with two builds +of the rule: OLD is src/tiers.ts at --base (default origin/claude/post-work-audit), +NEW is the working tree's src/tiers.ts. Both are loaded straight from the .ts +source (node strips the types), so no build is needed for either. + +Population: every -molt-[-] tree under today's lanes in +~/.cache/maat-bench/container-work whose grader row exists (the lane's results +file is named on the first line of ~/.cache/maat-bench/.log). + +Collected per check command, with its source: + sealed a sealed drafted check, its command from the released list, the + last full receipt, or (older builds) the bar run's detail + audit a post-work audit check in the journal (`post-work-audit` note) + replay a post-work audit check from audit_replay.py's caches + (~/.cache/maat-bench/audit-replay*), judged live on the same trees + +1. CATEGORIES. Every command OLD rejects is sorted into one category by the + first matching pattern below (a heuristic, for the measurement only), with + NEW's verdict per category. + +2. TIER REPLAY (sealed checks). A run whose last tier note was "passed-checks: + no check that ran the work asserted an expected value" (the evidence + blocker, nothing else) earns "verified" under NEW when a passing drafted + check in its last accepted receipt is now tagged value and was written by + an independent author (the receipt's "written by: the judge model ..." / + "the reference writer ..."). Counted two ways: any such check (the + default: no lane today ran with --require-discriminating, no + `passed-untested` tier is journalled), and only one that FAILED before the + work (the receipt's "before the work: failed"), which is what the + discrimination gate would ask. The critic's `surface` tag is not + journalled: when the receipt's strongest passing class is "runs" and the + run has more than one passing drafted check, the newly-value one may have + been surface, so the count can only overstate. + +3. AUDIT REPLAY (--audit-gates). An audit check OLD dropped with V-no-value + and NEW tags value goes on to the remaining gates (runs on the work, fails + before it, kills a mutant) in a bench container with NEW's dist mounted + (REPLAY_DIST, default /dist: run `npm run build` first). A run flips + to "verified-audit" when one clears every gate and the run was not already + verified. + + Gate results are cached per (run, command) under ~/.cache/maat-bench/value-replay/; + clear it after changing anything past the V-no-value gate. + +The change ships only if no grader-FAILED run flips to verified. As first +widened (a pattern the work wrote, matched against the check's own strings, +counted too), 4 of the 7 runs that alone would have flipped were grader +failures, every one a regex task whose judge examples missed the grader's +`x 1.2.3.4.5 on 2024-01-05`; that form was withdrawn (src/tiers.ts +PATTERN_DELIVERABLE), and this script reports the rule as shipped. +""" +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +import tempfile +from collections import Counter, defaultdict +from pathlib import Path + +HERE = Path(__file__).resolve().parent +REPO = HERE.parents[1] if len(HERE.parents) > 1 else HERE +CACHE = Path.home() / ".cache/maat-bench" +CW = CACHE / "container-work" +RESULTS = CACHE / "container-results" +TODAY = ["ab1", "dsbig", "dsha", "dshj", "dsv4", "ga3b", "gh", "gnuc", "goc", "gp", "gx", "hk55", "hkbig", + "ml4", "ml4b", "ml4p", "mm3", "mmhd", "mmhj", "mmsj", "nemj", "q235", "scm", "scr"] +RUN_DIR = re.compile(r"^(?P.+)-molt-(?P\d+)(?:-(?P[a-z0-9-]+))?$") +RESULTS_LINE = re.compile(r"container-results/(results-[\w.-]+\.jsonl)") +NO_VALUE_REASON = "no check that ran the work asserted an expected value" + + +# ---------------------------------------------------------------- the two classifiers + +def classifier(base: str) -> dict[str, Path]: + """{"old": tiers.ts at `base`, "new": the working tree's}, as loadable .ts files.""" + d = Path(tempfile.mkdtemp(prefix="value-replay-")) + old = d / "tiers-old.ts" + old.write_text(subprocess.run(["git", "-C", str(REPO), "show", f"{base}:src/tiers.ts"], capture_output=True, text=True, check=True).stdout) + return {"old": old, "new": REPO / "src/tiers.ts"} + + +def tag_all(src: Path, runs: list[str]) -> list[bool]: + js = ( + f"import {{ assertsValue }} from {json.dumps(str(src))};" + "let s='';process.stdin.on('data',d=>s+=d).on('end',()=>console.log(JSON.stringify(JSON.parse(s).map(r=>assertsValue(r)))));" + ) + p = subprocess.run(["node", "--no-warnings", "--input-type=module", "-e", js], input=json.dumps(runs), capture_output=True, text=True) + if p.returncode: + sys.exit(f"classifier {src} failed: {p.stderr[-600:]}") + return json.loads(p.stdout) + + +# ---------------------------------------------------------------- reading the runs + +def results_file(lane: Path) -> Path | None: + log = CACHE / f"{lane.name}.log" + if log.exists(): + m = RESULTS_LINE.search(log.read_text(errors="replace")[:4000]) + if m and (RESULTS / m.group(1)).exists(): + return RESULTS / m.group(1) + return None + + +def grader_rows(path: Path) -> dict[tuple[str, str, int], dict]: + rows: dict[tuple[str, str, int], dict] = {} + for line in path.read_text().splitlines(): + try: + r = json.loads(line) + except ValueError: + continue + if r.get("agent") == "molt": + rows[(r["task"], r.get("arm") or "", int(r["rep"]))] = r # the last row for a trial wins + return rows + + +def journal(tree: Path) -> list[dict]: + out: list[dict] = [] + for f in sorted((tree / ".maat/log").glob("*.jsonl")): + for line in f.read_text(errors="replace").splitlines(): + try: + out.append(json.loads(line)) + except ValueError: + pass + out.sort(key=lambda e: (e.get("iso", ""), e.get("seq", 0))) + return out + + +def tname(n: str) -> str: + return n if n.startswith("task:") else f"task:{n}" + + +def receipt(tree: Path) -> dict: + """The last accepted receipt (its full text when withheld): status line and, per check, command/result/author/before.""" + full = sorted((tree / ".maat/receipts/full").glob("*-accepted.md")) or sorted((tree / ".maat/receipts").glob("*-accepted.md")) + if not full: + return {} + text = full[-1].read_text(errors="replace") + checks: dict[str, dict] = {} + cur: dict | None = None + key = None + for line in text.splitlines(): + m = re.match(r"^(check|kind|written by|before the work|command|exit|result|ran|duration_ms|role): ?(.*)$", line) + if m: + key = m.group(1) + if key == "check": + cur = checks.setdefault(m.group(2).strip(), {}) + elif cur is not None: + cur[key] = m.group(2) + continue + if key == "command" and cur is not None and line and not line.startswith(("#", "|")): + cur["command"] += "\n" + line # a multi-line command runs on until the next field + else: + key = None if line.startswith(("#", "|")) else key + strongest = re.search(r"strongest passing check is (\w+)", text) + return {"file": str(full[-1]), "checks": checks, "strongest": strongest.group(1) if strongest else None} + + +def independent(author: str | None) -> bool: + return bool(author) and author.startswith(("the judge model", "the reference writer")) + + +def lanes(prefixes: list[str]) -> list[Path]: + return sorted(p for p in CW.iterdir() if p.is_dir() and any(re.fullmatch(rf"{re.escape(x)}-(?:r)?\d+", p.name) for x in prefixes)) + + +def collect(prefixes: list[str]) -> tuple[list[dict], list[dict]]: + """(runs, checks). A check row: {run, source, name, cmd, rule?, ...}.""" + runs: list[dict] = [] + checks: list[dict] = [] + for lane in lanes(prefixes): + rf = results_file(lane) + if not rf: + print(f"# {lane.name}: no results file, skipped", file=sys.stderr) + continue + rows = grader_rows(rf) + for tree in sorted(lane.iterdir()): + m = RUN_DIR.match(tree.name) + if not m or not tree.is_dir() or not (tree / ".maat/log").is_dir(): + continue + row = rows.get((m["task"], m["arm"] or "", int(m["rep"]))) + if not row: + continue + ev = journal(tree) + notes = [e["data"] for e in ev if e.get("kind") == "note"] + tier = next((d for d in reversed(notes) if str(d.get("text", "")).startswith("tier:")), None) + rc = receipt(tree) + run = {"id": f"{lane.name}/{tree.name}", "lane": lane.name, "task": m["task"], "arm": m["arm"] or "", "tree": str(tree), + "passed": bool(row.get("passed")), "claim": row.get("claim") or "", "said_done": row.get("said_done"), + "tier": tier.get("text", "")[6:] if tier else None, "tier_reason": (tier or {}).get("reason"), + "evidence": (tier or {}).get("evidence"), "receipt": rc, "audit": []} + runs.append(run) + # sealed checks: commands from the released list, then the receipt, then a journalled bar-run detail + cmds: dict[str, str] = {} + for e in ev: + if e.get("kind") == "bar_run": + for c in e["data"].get("checks", []): + if c.get("name", "").startswith("task:") and c.get("detail") and not str(c["detail"]).startswith("[withheld"): + cmds.setdefault(c["name"], c["detail"]) + for n, c in rc.get("checks", {}).items(): + if n.startswith("task:") and c.get("command") and not c["command"].startswith("[withheld"): + cmds[n] = c["command"] + for d in notes: + if d.get("kind") == "checks-released": + for c in d.get("checks", []): + if c.get("run"): + cmds[tname(c["name"])] = c["run"] + run["cmds"] = cmds + for n, cmd in cmds.items(): + rcx = rc.get("checks", {}).get(n, {}) + checks.append({"run": run, "source": "sealed", "name": n, "cmd": cmd, "result": rcx.get("result"), + "author": rcx.get("written by"), "before": rcx.get("before the work")}) + for d in notes: + if d.get("kind") == "post-work-audit": + for c in d.get("checks", []): + checks.append({"run": run, "source": "audit", "name": c["name"], "cmd": c["run"], "rule": c.get("rule"), + "accepted": c.get("accepted"), "quote": c.get("quote"), "changed": d.get("changed", [])}) + for rd in sorted(CACHE.glob("audit-replay*")): + f = rd / (run["id"].replace("/", "__") + ".json") + if f.exists(): + try: + rep = json.loads(f.read_text()) + except ValueError: + continue + for c in rep.get("checks", []): + checks.append({"run": run, "source": f"replay:{rd.name}", "name": c["name"], "cmd": c["run"], "rule": c.get("rule"), + "accepted": c.get("accepted"), "quote": c.get("quote"), + "changed": [x["path"] if isinstance(x, dict) else x for x in rep.get("changed", [])]}) + return runs, checks + + +# ---------------------------------------------------------------- categories + +PY_CALL_LIT = r"\w+(?:\.\w+)*\((?:[^()]*?)(?:-?\d|['\"\[{])" +CATEGORIES: list[tuple[str, re.Pattern[str]]] = [ + ("exception expectation", re.compile(r"except\s+\(?[\w.]*(?:Error|Exception)\b|assertRaises|pytest\.raises|assert\.throws|\.toThrow")), + ("match / non-match of a literal", re.compile(r"re\.(?:search|match|fullmatch|findall)\([^,()]+,\s*['\"]|\.test\(\s*['\"]")), + ("is None / is not None on a literal call", re.compile(PY_CALL_LIT + r"[^\n]*?\bis\s+(?:not\s+)?None\b")), + ("literal in / not in the output", re.compile(r"['\"][^'\"]{2,}['\"]\s+(?:not\s+)?in\s+\w|\bnot\s+in\s+['\"]")), + ("grep -q of a literal in the deliverable's output", re.compile(r"\|\s*grep\s+(?:-\w+\s+)*-\w*q")), + ("grep -q of a literal in a file", re.compile(r"\bgrep\s+(?:-\w+\s+)*-\w*q")), + ("exit code with literal stdin", re.compile(r"(?:<<<|<<|printf|echo)[^\n]*\|[^\n]*(?:\$\?|\|\||&&|^!|\bif\b)")), + ("truthiness of a call on literal input", re.compile(r"\bassert\s+(?:not\s+)?" + PY_CALL_LIT)), + ("ordering comparison with a literal", re.compile(r"(?:<=?|>=?)\s*-?\d|-(?:lt|le|gt|ge)\s+-?\d")), + ("structural only (exists, type, non-empty)", re.compile(r"isinstance|test\s+-[fesdx]|\[\s+-[fesdx]|len\([^)]*\)\s*>|hasattr|callable\(|py_compile|--help|\bnode\s+--check")), +] + + +def category(cmd: str) -> str: + timed = bool(re.search(r"\btimeout\s+\d|time\.(?:time|perf_counter)\(", cmd)) + for name, pat in CATEGORIES: + if pat.search(cmd): + return f"{name}{' (+timing)' if timed else ''}" + return "timing only" if timed else "other" + + +# ---------------------------------------------------------------- audit gates (--audit-gates) + +GATE_CACHE = CACHE / "value-replay" + + +def gate_one(c: dict, image: str, dist: str) -> dict: + """The audit gates for one check, cached per (run, command) under ~/.cache/maat-bench/value-replay/.""" + import hashlib # noqa: PLC0415 + key = GATE_CACHE / (c["run"]["id"].replace("/", "__") + "-" + hashlib.sha256(c["cmd"].encode()).hexdigest()[:16] + ".json") + if key.exists(): + return json.loads(key.read_text()) + r = gate_run(c, image, dist) + if not r.get("error"): + GATE_CACHE.mkdir(parents=True, exist_ok=True) + key.write_text(json.dumps(r)) + return r + + +def gate_run(c: dict, image: str, dist: str) -> dict: + run = c["run"] + entry = {"task": run["task"], "tree_name": Path(run["tree"]).name, "checks": [{"name": c["name"], "run": c["cmd"], "quote": c.get("quote") or ""}], + "changed": c.get("changed") or []} + cmd = ["docker", "run", "--rm", "-i", "-e", "PYTHONDONTWRITEBYTECODE=1", + "-v", f"{HERE}:/bench:ro", "-v", f"{run['tree']}:/tree:ro", + "-v", f"{dist}:/usr/local/lib/node_modules/@solvyx/molt/dist:ro", image, + "sh", "-c", "export MOLT_DIST_ABS=$(npm root -g)/@solvyx/molt/dist; python3 /bench/value_replay.py --inside"] + try: + p = subprocess.run(cmd, input=json.dumps(entry), capture_output=True, text=True, timeout=600) + line = (p.stdout.strip().splitlines() or [""])[-1] + return json.loads(line) if line.startswith("{") else {"error": f"exit {p.returncode}: {p.stderr[-400:]}"} + except subprocess.TimeoutExpired: + return {"error": "timed out"} + + +def inside() -> int: + """In the container: rebuild the pre-work tree, plan mutants of the changed files, run the audit gates.""" + import shutil # noqa: PLC0415 + sys.path.insert(0, str(HERE)) + from tasks import TASKS # noqa: PLC0415 + from tasks2 import TASKS2 # noqa: PLC0415 + from tasks3 import TASKS3 # noqa: PLC0415 + by = {T.name: T for T in TASKS + TASKS2 + TASKS3} + e = json.loads(sys.stdin.read()) + root = Path(tempfile.mkdtemp(prefix="value-")) + pre = root / "pre" / e["tree_name"] + work = root / "work" / e["tree_name"] + pre.mkdir(parents=True) + by[e["task"]].setup(pre) + shutil.copytree("/tree", work, symlinks=True, ignore=shutil.ignore_patterns(".maat", ".molt")) + js = r""" +const { auditGates, planAuditMutants } = await import(`${process.env.MOLT_DIST_ABS}/post-audit.js`); +const fs = await import('node:fs'); const path = await import('node:path'); +let s=''; for await (const c of process.stdin) s+=c; const a = JSON.parse(s); +const files = []; +for (const p of a.changed) { + let text = '', before; + try { text = fs.readFileSync(path.join(a.work, p), 'utf8'); } catch {} + try { before = fs.readFileSync(path.join(a.pre, p), 'utf8'); } catch {} + if (text && !text.includes('\u0000')) files.push({ path: p, text, ...(before !== undefined ? { before } : {}) }); +} +const mutants = await planAuditMutants(files); +const out = []; +for (const c of a.checks) out.push(await auditGates(c, { workDir: a.work, preWorkDir: a.pre, mutants })); +console.log(JSON.stringify({ mutants: mutants.length, checks: out })); +""" + inp = {"work": str(work), "pre": str(pre), "changed": e["changed"], "checks": e["checks"]} + p = subprocess.run(["node", "--input-type=module", "-e", js], input=json.dumps(inp), capture_output=True, text=True, timeout=500) + print((p.stdout.strip().splitlines() or [json.dumps({"error": p.stderr[-500:]})])[-1]) + return 0 + + +# ---------------------------------------------------------------- report + +def pct(a: int, b: int) -> str: + return f"{100 * a / b:.0f}%" if b else "n/a" + + +def main(argv: list[str]) -> int: + if argv[:1] == ["--inside"]: + return inside() + ap = argparse.ArgumentParser() + ap.add_argument("--base", default="origin/claude/post-work-audit") + ap.add_argument("--samples", type=int, default=3) + ap.add_argument("--audit-gates", action="store_true") + ap.add_argument("--image") + ap.add_argument("--jobs", type=int, default=4) + ap.add_argument("prefixes", nargs="*") + a = ap.parse_args(argv) + runs, checks = collect(a.prefixes or TODAY) + src = classifier(a.base) + cmds = sorted({c["cmd"] for c in checks}) + old = dict(zip(cmds, tag_all(src["old"], cmds))) + new = dict(zip(cmds, tag_all(src["new"], cmds))) + for c in checks: + c["old"], c["new"] = old[c["cmd"]], new[c["cmd"]] + print(f"runs with a grader row: {len(runs)} check rows: {len(checks)} distinct commands: {len(cmds)}") + print(f"tagged value: OLD {sum(old.values())}, NEW {sum(new.values())} of {len(cmds)} distinct commands") + lost = [x for x in cmds if old[x] and not new[x]] + if lost: + print(f"!! NEW drops the value tag from {len(lost)} command(s) OLD tagged:") + for x in lost[:10]: + print(" " + x[:200].replace("\n", "\\n")) + + # 1. categories of what OLD rejects (distinct commands; audit V-no-value rows called out) + rejected = sorted({c["cmd"] for c in checks if not c["old"]}) + vnv = {c["cmd"] for c in checks if c.get("rule") == "V-no-value"} + cat = {x: category(x) for x in rejected} + by_cat: dict[str, list[str]] = defaultdict(list) + for x in rejected: + by_cat[cat[x]].append(x) + print(f"\n== 1. what OLD tags no-value: {len(rejected)} distinct commands ({len(vnv)} of them audit V-no-value rejections)") + print(f" {'category':58} {'all':>5} {'V-no-v':>7} {'NEW value':>10}") + for k, xs in sorted(by_cat.items(), key=lambda kv: -len(kv[1])): + print(f" {k:58} {len(xs):5} {sum(x in vnv for x in xs):7} {sum(new[x] for x in xs):10}") + for k, xs in sorted(by_cat.items(), key=lambda kv: -len(kv[1])): + print(f"\n -- {k}") + for x in sorted(xs, key=lambda x: (x not in vnv, len(x)))[: a.samples]: + print(f" [{'NEW value' if new[x] else 'no value '}] {x[:230]!r}") + + # 2. tier replay over the sealed checks + print("\n== 2. tier replay (sealed checks; last tier note per run)") + tiers = Counter((r["tier"], r["tier_reason"] == NO_VALUE_REASON) for r in runs) + print(" runs by last tier note: " + ", ".join(f"{t or 'none'}{' [value blocker]' if b else ''}: {n}" for (t, b), n in tiers.most_common())) + blocked = [r for r in runs if r["tier"] == "passed-checks" and r["tier_reason"] == NO_VALUE_REASON] + flips_any: list[tuple[dict, list[dict]]] = [] + flips_disc: list[tuple[dict, list[dict]]] = [] + own_only = 0 + for r in blocked: + mine = [c for c in checks if c["run"] is r and c["source"] == "sealed" and c.get("result") in ("pass", "pass-vacuous")] + newv = [c for c in mine if c["new"] and not c["old"]] + ind = [c for c in newv if independent(c.get("author"))] + if newv and not ind: + own_only += 1 + if ind: + flips_any.append((r, ind)) + disc = [c for c in ind if str(c.get("before") or "").startswith("failed")] + if disc: + flips_disc.append((r, disc)) + for label, fl in (("any independent newly-value check (as the lanes ran)", flips_any), + ("one that also FAILED before the work (the discrimination gate)", flips_disc)): + right = [r for r, _ in fl if r["passed"]] + wrong = [r for r, _ in fl if not r["passed"]] + print(f" blocked only by the value tag: {len(blocked)} runs ({sum(r['passed'] for r in blocked)} grader-passed)") + print(f" flip to verified, {label}: {len(fl)} right {len(right)} WRONG {len(wrong)}") + for r, cs in fl: + many = sum(1 for c in checks if c['run'] is r and c['source'] == 'sealed' and c.get('result') in ('pass', 'pass-vacuous')) > 1 + print(f" {'right' if r['passed'] else 'WRONG'} {r['id']}{' (surface unknown)' if many else ''}") + for c in cs: + print(f" {c['name']} [{c.get('before') or 'no pre-work try'}] {c['cmd'][:170]!r}") + print(f" runs whose newly-value checks were all the worker's own: {own_only} (-> passed-own-checks, not verified)") + old_v = [r for r in runs if r["tier"] == "verified"] + print(f" recorded 'verified' on these lanes: {sum(r['passed'] for r in old_v)}/{len(old_v)} right; with NEW: " + f"{sum(r['passed'] for r in old_v) + sum(r['passed'] for r, _ in flips_any)}/{len(old_v) + len(flips_any)} " + f"({pct(sum(r['passed'] for r in old_v) + sum(r['passed'] for r, _ in flips_any), len(old_v) + len(flips_any))})") + + # 3. audit checks OLD dropped with V-no-value + cand = [c for c in checks if c.get("rule") == "V-no-value" and c["new"]] + stay = [c for c in checks if c.get("rule") == "V-no-value" and not c["new"]] + print(f"\n== 3. audit checks dropped with V-no-value: {len(cand) + len(stay)}; NEW tags value: {len(cand)} " + f"(runs: {len({c['run']['id'] for c in cand})}, {sum(1 for i in {c['run']['id']: c['run'] for c in cand}.values() if not i['passed'])} grader-failed)") + if a.audit_gates: + if not a.image: + sys.exit("--audit-gates needs --image maat-bench:-u") + dist = os.environ.get("REPLAY_DIST") or str(REPO / "dist") + from concurrent.futures import ThreadPoolExecutor # noqa: PLC0415 + with ThreadPoolExecutor(a.jobs) as ex: + res = list(ex.map(lambda c: gate_one(c, a.image, dist), cand)) + per_run: dict[str, list[tuple[dict, dict]]] = defaultdict(list) + rules = Counter() + for c, g in zip(cand, res): + gr = (g.get("checks") or [{}])[0] if not g.get("error") else {"rule": "replay-error", "why": g["error"]} + rules[gr.get("rule") or "accepted"] += 1 + per_run[(c["source"], c["run"]["id"])].append((c, gr)) + print(" gate outcomes: " + ", ".join(f"{k} {v}" for k, v in rules.most_common())) + fl = [(k, cs) for k, cs in per_run.items() if any(g.get("accepted") for _, g in cs)] + already = {(c["source"], c["run"]["id"]) for c in checks if c.get("accepted")} + newfl = [(k, cs) for k, cs in fl if k not in already and cs[0][0]["run"]["tier"] != "verified"] + right = [k for k, cs in newfl if cs[0][0]["run"]["passed"]] + wrong = [k for k, cs in newfl if not cs[0][0]["run"]["passed"]] + print(f" runs that flip to verified-audit: {len(newfl)} right {len(right)} WRONG {len(wrong)}") + for k, cs in newfl: + run = cs[0][0]["run"] + for c, g in cs: + if g.get("accepted"): + print(f" {'right' if run['passed'] else 'WRONG'} {k[0]} {k[1]} {c['name']}: {c['cmd'][:160]!r}") + for k, cs in per_run.items(): + for c, g in cs: + if g.get("rule") == "replay-error": + print(f" error {k[1]} {c['name']}: {g['why'][:200]}") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/src/tiers.ts b/src/tiers.ts index 43bf441..ee2b9c9 100644 --- a/src/tiers.ts +++ b/src/tiers.ts @@ -17,8 +17,8 @@ */ import type { CheckAuthor, CheckResult } from "./types.js"; -/** An operand that is a literal: a number, a quoted string without a variable in it, a list/dict opener, or a keyword value. */ -const LIT_AFTER = /^(?:\\?["'](?![^"']*\$)[^"']|-?\d|[[{]|(?:True|False|None|true|false|null)\b)/; +/** An operand that is a literal: a number, a quoted string without a variable in it (or a `$'...'` ANSI-C string), a list/dict opener, or a keyword value. */ +const LIT_AFTER = /^(?:\\?["'](?![^"']*\$)[^"']|\$'[^']|-?\d|[[{]|(?:True|False|None|true|false|null)\b)/; /** The operand just before an operator, when it is wholly a literal and not the tail of a name like `x2`. */ const LIT_BEFORE = /(?:(?')` / `except KeyError:` + * where only the path without the exception exits non-zero, or + * `pytest.raises(E)` / `assertRaises(E, ...)` / `assert.throws(...)` + * around a call with a literal argument. `except Exception` does not + * count, nor a handler that only prints. + * c. `grep` (plain, `-q`, `-x`; never `-v`, `-c`, `-l`) of a literal at + * the end of a pipeline that runs a program (python3, node, bash, sh, + * make, curl, `./tool`, `tool.py`, ...): `python3 slugify.py <<< '!!' + * | grep -q untitled`. A pipeline that only reads a file or git + * (`head -1 out.csv | grep -q id,name`, `git log | grep -q msg`) does + * not count: that is the shape of an output, not its behaviour. + * Structural checks (exists, `isinstance`, `hasattr`, `len(...) > 0`, + * non-empty), the truthiness of an arbitrary call, and timing bounds + * (`time.time() - t < 5`, `timeout 5`) never count. A specific exit code + * with literal stdin already counts through rule 1 (`[ $? -eq 2 ]`). + * + * Not widened: a pattern the work wrote, tried on strings the check + * chose (`re.search(p, '10.0.0.1 2024-01-05')`, `p = re.compile(open( + * 'regex.txt').read())`). Replayed over 2026-10-07's lanes, every run + * such a check alone would have verified was a regex task, and 4 of 7 + * were grader failures: the judges' example lines all missed the case the + * grader tried (`x 1.2.3.4.5 on 2024-01-05`). A command that does this + * gets none of rule 5, and rule 1 reads its `[ ... = ... ]` as before + * (bench/local/value_replay.py). * Conservative on purpose: a check missed here costs one claim the word * "verified"; a check counted wrongly costs the word its meaning. */ @@ -87,14 +117,444 @@ export function goldenOperands(run: string): string[] { return out; } +// ---------------------------------------------------------------- rule 5: behavioural assertions + +/** + * The shell nesting depth before each character of `s`: 0 at top level, more + * inside quotes, `$(...)`, `(...)` and backticks. A heuristic scanner, not a + * shell parser: it only has to tell `a; b` from `"$(x; y)"`. + */ +function shellDepths(s: string): number[] { + const out: number[] = new Array(s.length); + const stack: string[] = []; + for (let i = 0; i < s.length; i++) { + out[i] = stack.length; + const ch = s[i]!; + const top = stack[stack.length - 1]; + if (top === "'") { + if (ch === "'") stack.pop(); + continue; + } + if (ch === "\\") { + if (i + 1 < s.length) out[i + 1] = stack.length; + i++; + continue; + } + if (top === "$'") { + if (ch === "'") stack.pop(); + continue; + } + if (top === '"') { + if (ch === '"') stack.pop(); + else if (ch === "$" && s[i + 1] === "(") (stack.push("("), (out[i + 1] = stack.length - 1), i++); + else if (ch === "`") stack.push("`"); + continue; + } + if (top === "`" && ch === "`") { + stack.pop(); + continue; + } + if (ch === "'") stack.push(s[i - 1] === "$" ? "$'" : "'"); + else if (ch === '"') stack.push('"'); + else if (ch === "`") stack.push("`"); + else if (ch === "(") stack.push("("); + else if (ch === ")" && top === "(") stack.pop(); + } + return out; +} + +/** Top-level command segments of `s` (split on `;`, newline, `&&`, `||` outside quotes and substitutions), as [start, end). */ +function shellSegments(s: string): [number, number][] { + const d = shellDepths(s); + const out: [number, number][] = []; + let start = 0; + for (let i = 0; i < s.length; i++) { + if (d[i] !== 0) continue; + const two = s.slice(i, i + 2); + const sep = two === "&&" || two === "||" ? 2 : s[i] === ";" || s[i] === "\n" ? 1 : 0; + if (!sep) continue; + out.push([start, i]); + start = i + sep; + i += sep - 1; + } + out.push([start, s.length]); + return out; +} + +/** Asks what type or shape a thing has: `type(x).__name__`, `isinstance`, `hasattr`, `callable`. */ +const TYPE_PROBE = /\btype\([^()]*(?:\([^()]*\)[^()]*)*\)\.__name__|\.__class__|\b(?:isinstance|issubclass|hasattr|callable)\(/; +const BRACKET = /(?:^|\s)(?:\[\[?|test)\s/; +/** A whole shell word that is a literal: a non-empty quoted string with no expansion in it, `$'...'`, a number, or a keyword value. */ +const SHELL_LITERAL = /^(?:"[^"$`]+"|'[^']+'|\$'(?:[^'\\]|\\.)+'|-?\d+(?:\.\d+)?|true|false|null|True|False|None)$/; + +/** The shell word ending just before `at` (or starting just after `at + len`), quotes and substitutions kept whole. */ +function shellWordAround(s: string, d: number[], at: number, len: number): [string, string] { + let i = at - 1; + while (i >= 0 && /\s/.test(s[i]!) && d[i] === 0) i--; + let a = i; + while (a >= 0 && !(d[a] === 0 && /\s/.test(s[a]!))) a--; + let j = at + len; + while (j < s.length && /\s/.test(s[j]!) && d[j] === 0) j++; + let b = j; + while (b < s.length && !(d[b] === 0 && /\s/.test(s[b]!))) b++; + return [s.slice(a + 1, i + 1), s.slice(j, b)]; +} + +/** A top-level `=` inside `[ ]`/`[[ ]]`/`test`, with a literal shell word on either side. */ +function bracketLiteral(s: string, at: number): boolean { + const d = shellDepths(s); + if (d[at] !== 0) return false; + const seg = shellSegments(s).find(([a, b]) => a <= at && at <= b); + if (!seg || !BRACKET.test(s.slice(seg[0], at))) return false; + const [left, right] = shellWordAround(s, d, at, 1); + return SHELL_LITERAL.test(left) || SHELL_LITERAL.test(right); +} + +/** The stages of one top-level command split on `|` (never `||`). */ +function pipeStages(cmd: string): string[] { + const d = shellDepths(cmd); + const out: string[] = []; + let start = 0; + for (let i = 0; i < cmd.length; i++) { + if (d[i] === 0 && cmd[i] === "|" && cmd[i + 1] !== "|" && cmd[i - 1] !== "|" && cmd[i - 1] !== ">") { + out.push(cmd.slice(start, i)); + start = i + 1; + } + } + out.push(cmd.slice(start)); + return out; +} + +/** The program a pipeline stage runs: leading `!`, `VAR=x`, `timeout 5`, `env`, `time` taken off. */ +function stageProgram(stage: string): string { + let t = stage.trim(); + for (;;) { + const n = t.replace(/^(?:!\s*|\w+=\S*\s+|(?:env|time|nice|exec|command)\s+|timeout\s+(?:-\S+\s+)*\S+\s+)/, ""); + if (n === t) break; + t = n; + } + return t.split(/\s+/)[0] ?? ""; +} + +/** A program that runs something: an interpreter, a build tool, a client of a server the work runs, or a script by path. */ +const RUNNER = /^(?:python[\d.]*|node|bash|sh|zsh|dash|ruby|perl|php|deno|bun|npx|npm|make|gmake|java|go|cargo|sqlite3|curl|\.{1,2}\/\S+|\S+\.(?:py|js|mjs|cjs|ts|sh|rb|pl))$/; + +/** Rule 5c: ` | ... | grep [-q] ''`. */ +function grepsRunOutput(run: string): boolean { + for (const [a, b] of shellSegments(run)) { + const stages = pipeStages(run.slice(a, b)); + if (stages.length < 2) continue; + const last = stages[stages.length - 1]!.trim(); + const words: string[] = last.match(/"(?:[^"\\]|\\.)*"|'[^']*'|\S+/g) ?? []; + if (!/^[ef]?grep$/.test(words[0] ?? "")) continue; + const args = words.slice(1); + const flags = args.filter((w) => w.startsWith("-")); + if (hasFlag(flags, "vcLl") || flags.some((f) => /^--(?:invert|count|files)/.test(f))) continue; + const e = args.indexOf("-e"); + const pat = e >= 0 ? args[e + 1] : args.find((w) => !w.startsWith("-")); + if (!pat || pat.startsWith(">") || pat.startsWith("<")) continue; + const body = pat.replace(/^(["'])(.*)\1$/s, "$2"); + if (/\$[\w{(]/.test(body) || (body.match(/\w/g) ?? []).length < 2) continue; + if (stages.slice(0, -1).some((s) => RUNNER.test(stageProgram(s)))) return true; + } + return false; +} + +/** Python/JS names whose call is never "the deliverable on an input": builtins, the stdlib, string and container methods. */ +const NOT_DELIVERABLE = new Set( + ( + "all any bool list tuple set frozenset dict sorted reversed sum min max next iter zip map filter enumerate range str int float abs round " + + "strip lstrip rstrip split rsplit splitlines join replace lower upper casefold startswith endswith find rfind index count get keys values items " + + "read readline readlines write encode decode format append extend pop add update copy compile escape print open repr chr ord " + + "isinstance issubclass hasattr getattr setattr callable len type id dir vars hash super object exit quit" + ).split(" "), +); +/** Calls whose contents never count: they ask about a thing's shape, not what the work does. */ +const OPAQUE = /^(?:isinstance|issubclass|hasattr|getattr|callable|len|type|id|dir|vars|open|print|repr|os(?:\.\w+)*|glob(?:\.\w+)*|json\.\w+|csv\.\w+|inspect\.\w+|time\.\w+|shutil\.\w+|subprocess\.\w+|Path|pathlib\.\w+)$/; +/** + * A pattern the work wrote, tried on strings the check chose: `re.search(p, ...)` + * or `re.compile(open('regex.txt').read())` with a pattern that is not a literal. + * See "Not widened" under assertsValue. + */ +const PATTERN_DELIVERABLE = /(? { + const names = new Set(); + for (const m of run.matchAll(/(?])=(?!=)\s*/g)) { + if (LITERAL_ARG.test(run.slice((m.index ?? 0) + m[0].length, (m.index ?? 0) + m[0].length + 12))) names.add(m[1]!); + } + for (let pass = 0; pass < 2; pass++) { + for (const m of run.matchAll(/\bfor\s+\(?\s*([A-Za-z_][\w\s,]*?)\s*\)?\s+in\s+(?:(?:zip|enumerate|sorted|list)\(\s*)?([A-Za-z_][\w]*|[[({])/g)) { + const src = m[2]!; + if (/^[[({]$/.test(src) || names.has(src)) for (const v of m[1]!.split(",")) if (v.trim()) names.add(v.trim()); + } + } + return names; +} + +/** Is this argument text literal input: a literal, or a name bound to one? */ +function literalInput(arg: string, names: ReadonlySet): boolean { + const a = arg.trim().replace(/^[A-Za-z_]\w*\s*=(?!=)\s*/, ""); + return LITERAL_ARG.test(a) || names.has(a); +} + +/** A call: its name and top-level arguments. */ +type Call = { name: string; args: string[] }; + +/** The calls in `expr` outside opaque ones (isinstance, len, open, ...), with their top-level arguments. */ +function callsIn(expr: string): Call[] { + const out: Call[] = []; + const re = /([A-Za-z_][\w.]*)\s*\(/g; + let m: RegExpExecArray | null; + while ((m = re.exec(expr))) { + const open = m.index + m[0].length - 1; + const end = closeParen(expr, open); + if (end < 0) continue; + const name = m[1]!; + if (OPAQUE.test(name)) { + re.lastIndex = end; + continue; + } + out.push({ name, args: splitTop(expr.slice(open + 1, end - 1), /,/y) }); + } + return out; +} + +/** A call into the work's code (not a builtin, the stdlib or a string method) handed literal input. */ +function deliverableCall(c: Call, names: ReadonlySet): boolean { + const last = c.name.split(".").pop() ?? c.name; + if (NOT_DELIVERABLE.has(last) || /^(?:re|json|csv|os|sys|time|math|random|itertools|collections|datetime)\./.test(c.name)) return false; + return c.args.some((a) => literalInput(a, names)); +} + +const COMPARISON = /==|!=|<=|>=|<|>|\s(?:not\s+)?in\s/y; + +/** One `and`-conjunct of an asserted condition: does it assert the work's behaviour on literal input? */ +function conjunctAsserts(raw: string, names: ReadonlySet): boolean { + let c = raw.trim(); + for (;;) { + const n = c.replace(/^not\s+/, "").replace(/^\((.*)\)$/s, (all, inner: string) => (closeParen(all, 0) === all.length ? inner : all)).trim(); + if (n === c) break; + c = n; + } + // `'x' in f('lit')` / `'x' not in f('lit')`: a literal in the work's answer. + const member = /^([rbuf]{0,2}\\?["'][^"']*\\?["']|-?\d+)\s+(?:not\s+)?in\s+(.+)$/s.exec(c); + if (member) return callsIn(member[2]!).some((k) => deliverableCall(k, names)); + const none = /\s+is\s+(?:not\s+)?None\s*$/.test(c); + const body = c.replace(/\s+is\s+(?:not\s+)?None\s*$/, ""); + // A comparison with no literal side is rule 1's to refuse (`f(x) == g(x)`), and an ordering is a bound, not a value. + if (!none || splitTop(body, COMPARISON).length > 1) return false; + return callsIn(body).some((k) => deliverableCall(k, names)); +} + +/** The condition text starting at `from`: up to a top-level `,`, `;`, `:`, newline, closing bracket, or `stop`. */ +function conditionAt(s: string, from: number, stop?: RegExp): string { + let depth = 0; + let q: string | undefined; + for (let i = from; i < s.length; i++) { + const ch = s[i]!; + if (q) { + if (ch === "\\") i++; + else if (ch === q) q = undefined; + else if (ch === "\n") return s.slice(from, i); + continue; + } + if (ch === "\\" && (s[i + 1] === '"' || s[i + 1] === "'")) { + q = s[i + 1]; + i++; + continue; + } + if (ch === "'" || ch === '"') { + // A quote that never closes on this line is the shell's, not Python's: the code ends here. + const rest = s.slice(i + 1); + const close = rest.search(new RegExp(`(?= 0 && nl < close)) return s.slice(from, i); + q = ch; + continue; + } + if ("([{".includes(ch)) depth++; + else if (")]}".includes(ch)) { + if (depth === 0) return s.slice(from, i); + depth--; + } else if (depth === 0 && (ch === "," || ch === ";" || ch === "\n" || ch === ":")) return s.slice(from, i); + else if (depth === 0 && stop) { + stop.lastIndex = i; + if (stop.test(s)) return s.slice(from, i); + } + } + return s.slice(from); +} + +/** Rule 5a: an asserted condition (`assert`, `sys.exit(0 if ... else 1)`, `if ...: sys.exit(1)`) on the work's behaviour. */ +function assertsCondition(run: string, names: ReadonlySet): boolean { + const conditions: string[] = []; + for (const m of run.matchAll(/(? { + // `a or b` asserts neither: one may be the always-true branch. + if (splitTop(cond, /\sor\s/y).length > 1) return false; + return splitTop(cond, /\sand\s/y).some((c) => conjunctAsserts(c, names)); + }); +} + +/** Exits non-zero: what makes the path that did NOT raise fail the check. */ +const FAIL_EXIT = /(?:(?): boolean { + const literalCall = (text: string) => callsIn(text).some((k) => deliverableCall(k, names)); + // pytest.raises(KeyError) / assertRaises(KeyError, f, 'x') / assert.throws(() => f('x')) + for (const m of run.matchAll(/(?:pytest\.raises|assertRaises)\(\s*([\w.]+)/g)) { + if (/^(?:Base)?Exception$/.test(m[1]!)) continue; + const tail = run.slice(m.index ?? 0, (m.index ?? 0) + 300); + if (literalCall(tail) || /assertRaises\(\s*[\w.]+\s*,\s*[\w.]+\s*,\s*(?:[rbuf]{0,2}\\?["']|-?\d|[[({])/.test(tail)) return true; + } + for (const m of run.matchAll(/assert\.(?:throws|rejects)\(/g)) { + const open = (m.index ?? 0) + m[0].length - 1; + const end = closeParen(run, open); + if (end > 0 && literalCall(run.slice(open, end))) return true; + } + // try: f('x') / except KeyError: ... with a failing exit outside the handler. + for (const m of run.matchAll(/(? deliverableCall(k, names)); + if (hit < 0) continue; + // In the try block after the call: reached only when the call did not raise. + if (FAIL_EXIT.test(body.slice(body.indexOf(calls[hit]!.name)))) return true; + // After the handler: a line indented no deeper than `except` (or an `else:`). + const lineStart = run.lastIndexOf("\n", e.index) + 1; + const indent = e.index - lineStart; + const handlerEnd = run.indexOf("\n", e.index + e[0].length); + if (handlerEnd < 0) continue; + const handler: string[] = []; + let after = ""; + const lines = run.slice(handlerEnd + 1).split("\n"); + for (let i = 0; i < lines.length; i++) { + const line = lines[i]!; + const ind = line.length - line.trimStart().length; + if (line.trim() && ind <= indent) { + after = lines.slice(i).join("\n"); + break; + } + handler.push(line); + } + const handlerText = `${run.slice(e.index + e[0].length, handlerEnd)}\n${handler.join("\n")}`; + if (!FAIL_EXIT.test(handlerText) && FAIL_EXIT.test(after.split(/\n\S*(?:except|finally)\b/)[0] ?? "")) return true; + } + return false; +} + +/** + * Rule 5 of assertsValue: the deliverable is called or run on literal input + * and the outcome is asserted: `is None`, a literal in its answer, a specific + * exception, or a literal grepped from its output. Never for a pattern the + * work wrote tried on the check's own strings (PATTERN_DELIVERABLE). + * Exported for the tests. + */ +export function assertsBehaviour(run: string): boolean { + if (PATTERN_DELIVERABLE.test(run)) return false; + if (grepsRunOutput(run)) return true; + const names = literalNames(run); + return assertsCondition(run, names) || assertsException(run, names); +} + export function assertsValue(run: string, goldenPredates?: (operand: string) => boolean): boolean { const ops = /(==|!=|(?<=\s)-eq(?=\s)|(?<=\s)-ne(?=\s)|(?<=\s)=(?=\s))/g; for (const m of run.matchAll(ops)) { const at = m.index ?? 0; if (m[1] === "=") { - // A lone `=` is an assignment unless it sits in a test bracket. + // A lone `=` is an assignment unless it sits in a test bracket. A `;` or + // newline inside `$(python3 -c "a; b")` does not end the bracket's command. const segment = run.slice(0, at).split(/[;\n]|&&|\|\|/).pop() ?? ""; - if (!/(?:^|\s)(?:\[\[?|test)\s/.test(segment)) continue; + if (!BRACKET.test(segment)) { + // A `;` or newline inside `[ "$(python3 -c "a; b")" = "2" ]` does not end the bracket's + // command: read that one shell-aware, its operands as whole shell words. + // Not when what is compared is a type probe (`print(type(x).__name__)" = "int"`): that is a shape. + if (!PATTERN_DELIVERABLE.test(run) && !TYPE_PROBE.test(run) && bracketLiteral(run, at)) return true; + continue; + } } if (literalAround(run, at, m[1]!.length)) return true; } @@ -121,7 +581,7 @@ export function assertsValue(run: string, goldenPredates?: (operand: string) => if (!body || /\$[\w{(]/.test(body)) continue; if (STANDALONE_NUMBER.test(body) || /^\^.{3,}\$$/s.test(body) || (hasFlag(flags, "x") && /\S/.test(body))) return true; } - return false; + return assertsBehaviour(run); } /** The tags a drafted check carries: surface from the critic, value from the command. */ diff --git a/test/tiers.test.ts b/test/tiers.test.ts index 9da9dda..cc14fd8 100644 --- a/test/tiers.test.ts +++ b/test/tiers.test.ts @@ -53,6 +53,63 @@ describe("the value tag", () => { for (const c of yes) it(`asserts a value: ${c.split("\n")[0]}`, () => assert.equal(assertsValue(c), true, c)); for (const c of no) it(`asserts no value: ${c}`, () => assert.equal(assertsValue(c), false, c)); + // Rule 5 and the bracket fix, on commands the 2026-10-07 lanes drafted and V-no-value refused. + const behaviour = [ + // a specific exception, and only the path without it exits non-zero + `python3 -c "import pricing\ntry:\n pricing.price('zz-no-such-item-xyz', 1)\nexcept KeyError:\n raise SystemExit(0)\nraise SystemExit(1)"`, + `python3 -c "import pricing\ntry:\n pricing.price('zz', 1)\n raise SystemExit('no KeyError')\nexcept KeyError:\n pass"`, + `python3 -c "import pytest, pricing\nwith pytest.raises(KeyError):\n pricing.price('zz', 1)"`, + `cd $(mktemp -d) && cp ../pricing.py . && python3 -c "from pricing import price; price('widget', 1, 'UNKNOWN')" 2>&1 | grep -q 'ValueError' && exit 0 || exit 1`, + // is None / a literal in the answer, on a call with literal input + `python3 -c "import pricing; assert pricing.best_coupon('item1', 1) is not None"`, + `python3 -c "import pricing; assert pricing.best_coupon('zz', 1) is None, 'no coupon for an unknown item'"`, + `python3 -c "import sys, pricing; sys.exit(0 if 'SAVE10' in pricing.coupons_for('mug', 2) else 1)"`, + // a literal grepped from what running the work printed + `echo -e '!!!\n!!!' | python3 slugify.py | grep -q 'untitled$'`, + `python3 nextrun.py '* * * * 1' '2023-01-01 12:00' 1 | grep '2023-01-02'`, + `node summarize.js nonexistent.json 2>&1 | grep -q 'Error reading file'`, + `d=$(mktemp -d) && cp -r md2html.py src Makefile "$d"/ && cd "$d" && make -s && ! make | grep -q md2html`, + `./rotate.sh 2>&1 | grep -q 'usage'`, + // rule 1 inside a test bracket whose $(...) holds a ; or a newline, and $'...' literals + `[ "$(python3 -c "from pricing import best_coupon; print(best_coupon('A', 5))")" = 'SAVE10' ]`, + `[ "$(python3 -c "import json; d=json.load(open('configs/api.json')); print(d['version'])")" = "2" ]`, + `[ $(node summarize.js nonexistent.json 2>&1; echo $?) = 1 ]`, + `output=$(python3 wc.py f1.txt f2.txt) && [ "$output" = $'2 2 4 f1.txt\n4 5 10 total' ]`, + ]; + const notBehaviour = [ + // Not widened: the work's pattern tried on the check's own strings (4 of 7 such flips were grader failures). + `python3 -c "import re; p=open('regex.txt').read().strip(); assert re.search(p, 'GET 10.1.2.3 served 2024-05-06 ok'), p"`, + `python3 -c "import re; p=open('regex.txt').read().strip(); assert re.search(p, 'host 192.168.01.1 seen 2024-05-06') is None, p"`, + `[ "$(python3 -c "\nimport re\nregex = re.compile(open('regex.txt').read().strip())\nprint('PASS' if not regex.search('192.168.01.1 2023-01-01') else 'FAIL')\n")" = "PASS" ]`, + // a timing bound with only a type check + `timeout 5 python3 -c "from pairs import find_pairs; import random; random.seed(1); nums=[random.randint(-10**6,10**6) for _ in range(200000)]; r=find_pairs(nums, 7); assert isinstance(r, list)"`, + `python3 -c "import time,random,pairs as p;a=[random.randint(0,10**9) for _ in range(200000)];t=time.time();p.find_pairs(a);assert time.time()-t<5"`, + `python3 -c "import time; from pairs import find_pairs; t0=time.time(); r=find_pairs(list(range(200000)), 199999); assert len(r)>0 and time.time()-t0<5"`, + // structural: existence, type, non-empty + `python3 -c 'import pricing; assert hasattr(pricing, "COUPONS"), "COUPONS dict missing"'`, + `[ "$(python3 -c "import json; d=json.load(open('configs/web.json')); print(type(d['debug']).__name__)")" = "bool" ]`, + `[ "$(python3 -c "import json; d=json.load(open('configs/w.json')); print(isinstance(d['features'], list))")" = "True" ]`, + `head -n 1 clean.csv | grep -q 'id,name,email,age,city'`, + `git log --oneline -n 1 | grep -q 'Revert.*Add experimental cache'`, + `make clean && ! [ -d build ]`, + // the outcome is printed, never asserted + `python3 -c "import pricing; print(pricing.best_coupon('item1', 1) is not None)"`, + `python3 -c "import pricing; try: pricing.price('item1', 1, 'UNKNOWN'); print('No error'); except ValueError: print('ValueError raised')"`, + // an exception that is not specific, or only the raising path fails + `python3 -c "import pricing\ntry:\n pricing.price('zz', 1)\nexcept Exception:\n raise SystemExit(0)\nraise SystemExit(1)"`, + `python3 -c "import pricing\ntry:\n pricing.price('mug', 1)\nexcept KeyError:\n raise SystemExit(1)"`, + // truthiness of an arbitrary call; a comparison of two unknowns; or-ed conditions + `python3 -c "import tool; assert tool.ok('x')"`, + `python3 -c "import pricing; assert pricing.price('mug', 2, 'SAVE10') == pricing.price('mug', 2, None) * 0.9"`, + `python3 -c "import pricing; assert pricing.best_coupon('a', 1) is None or True"`, + `cd $(mktemp -d) && [ "$(python3 -c "from pricing import price; print(price('w', 2, 'S'))")" = "$(python3 -c "from pricing import price; print(price('w', 2, 'S'))")" ]`, + // grep that inverts or counts + `echo -e 'hello\n\nworld' | python3 wrap.py 10 | grep -c '^$'`, + `python3 tool.py | grep -v 'error'`, + ]; + for (const c of behaviour) it(`asserts behaviour: ${c.split("\n")[0]}`, () => assert.equal(assertsValue(c), true, c)); + for (const c of notBehaviour) it(`asserts no behaviour: ${c.split("\n")[0]}`, () => assert.equal(assertsValue(c), false, c)); + it("reaches the sealed check as a tag beside the critic's surface reading", () => { const sealed = taskChecksFrom( {