From d99e25e13cdf8cee6a62591a332450772cbb3a0d Mon Sep 17 00:00:00 2001 From: Tyler Skelton Date: Wed, 7 Oct 2026 08:55:59 -0700 Subject: [PATCH 01/87] Sync local work: separate judge model, OpenCode backend, bench container runner Squash of the local work since 2026-10-05 onto main (after PRs #14-#22): - --judge / --judge-url and MAAT_JUDGE_MODEL / MAAT_JUDGE_URL: checks and reviews run on a separate model; receipts name the judge. Short names `grok` and `opencode`; removed backends are refused at parse time. - OpenCode as an ACP worker and judge (opencode://subscription). - Check quality: lint + redraft, edge checks, late-seal and lazy-seal fixes, leftover-files nudge, reveal/bend guard, provider-error retries, lenient tool-argument JSON, provider-cap stop, local speed profile. - Grok Build fixes: seal gate inside the subprocess, late-join wait. - Bench: container runner with the answer-key lock (agent runs unprivileged), paired arms, 42 local tasks with reference solutions, analysis scripts. - finetune/extract.mjs excludes runs with a Grok worker or judge (xAI AUP). --- CHANGELOG.md | 2 +- bench/harbor/compare.py | 13 + bench/harbor/molt_agent.py | 13 +- bench/local/ambiguity-results/others16.jsonl | 16 + .../run1-tools-narration-INVALID.jsonl | 20 + .../run2-tool-stalls-partial.jsonl | 20 + bench/local/ambiguity-results/traps3.jsonl | 12 + bench/local/ambiguity_probe.py | 85 + bench/local/ambiguity_probe_driver.mjs | 19 + bench/local/ambiguity_probe_prompt.txt | 14 + bench/local/checkquality_analysis.py | 223 ++ bench/local/checkquality_lint.py | 202 ++ bench/local/checkquality_lintts.mjs | 20 + bench/local/checkquality_lintts.py | 70 + bench/local/checkquality_load.py | 136 + bench/local/checkquality_run.py | 100 + bench/local/checkquality_taxonomy.py | 126 + bench/local/checkquality_tiers.py | 128 + bench/local/checkquality_trees.py | 121 + bench/local/container/Dockerfile | 17 + bench/local/container/Dockerfile.grok | 9 + bench/local/container/Dockerfile.opencode | 9 + bench/local/container/run-in-container.sh | 83 + bench/local/crosscheck.py | 476 +++ bench/local/denials.py | 101 + bench/local/env_values.py | 414 +++ bench/local/frame_intact.py | 221 ++ bench/local/oracle_prompt.txt | 69 + bench/local/oracle_replay.py | 235 ++ bench/local/path_rule_count.py | 70 + .../_wrong/async-pool-node/1/pool.js | 14 + .../_wrong/async-pool-node/2/pool.js | 19 + .../_wrong/bash-backup/1/backup.sh | 15 + .../_wrong/bash-backup/2/backup.sh | 15 + .../_wrong/csv-to-json-node/1/csv2json.js | 29 + .../_wrong/dedupe-contacts/1/dedupe.py | 24 + .../_wrong/dedupe-contacts/2/dedupe.py | 35 + .../_wrong/fix-daily-buckets/1/bucket.py | 16 + .../_wrong/fix-daily-buckets/2/bucket.py | 20 + .../_wrong/git-find-culprit/1/solve.sh | 7 + .../_wrong/git-find-culprit/2/solve.sh | 6 + .../_wrong/git-split-lib-history/1/solve.sh | 6 + .../_wrong/git-split-lib-history/2/solve.sh | 8 + .../_wrong/ini-to-json/1/ini2json.py | 37 + .../_wrong/ini-to-json/2/solve.sh | 8 + .../_wrong/json-diff-cli/1/jdiff.py | 40 + .../_wrong/ledger-balance/1/ledger.py | 37 + .../_wrong/make-incremental/1/Makefile | 18 + .../_wrong/make-incremental/2/Makefile | 18 + .../_wrong/money-split/1/alloc.py | 11 + .../_wrong/money-split/2/alloc.py | 15 + .../_wrong/perf-asof-join/1/join.py | 23 + .../_wrong/refactor-invoice/1/solve.sh | 8 + .../_wrong/refactor-invoice/2/solve.sh | 7 + .../_wrong/rename-api-signature/1/solve.sh | 9 + .../_wrong/rename-api-signature/2/solve.sh | 4 + .../_wrong/semver-sort/1/vsort.py | 19 + .../_wrong/sessionize-logs/1/sessions.py | 30 + .../_wrong/sessionize-logs/2/solve.sh | 8 + .../_wrong/slugify-batch/1/slugify.py | 13 + .../_wrong/sql-longest-gaps/1/solve.sh | 12 + .../_wrong/sql-longest-gaps/2/solve.sh | 26 + .../_wrong/sqlite-migrate/1/migrate.py | 28 + .../_wrong/sqlite-migrate/2/solve.sh | 8 + .../_wrong/text-wrap-cli/1/wrap.py | 11 + .../_wrong/text-wrap-cli/2/wrap.py | 30 + .../_wrong/vendor-units-workaround/1/solve.sh | 6 + .../vendor-units-workaround/2/app/retry.py | 17 + .../async-pool-node/pool.js | 52 + .../reference_solutions/bash-backup/backup.sh | 42 + .../csv-to-json-node/csv2json.js | 78 + .../dedupe-contacts/dedupe.py | 48 + .../dedupe-contacts/solve.sh | 1 + .../fix-daily-buckets/bucket.py | 19 + .../fix-daily-buckets/report.py | 27 + .../git-find-culprit/solve.sh | 6 + .../git-split-lib-history/solve.sh | 7 + .../ini-to-json/ini2json.py | 105 + .../json-diff-cli/jdiff.py | 62 + .../ledger-balance/ledger.py | 93 + .../make-incremental/Makefile | 18 + .../reference_solutions/money-split/alloc.py | 35 + .../perf-asof-join/join.py | 21 + .../refactor-invoice/format.py | 11 + .../refactor-invoice/invoice.py | 42 + .../refactor-invoice/tax.py | 12 + .../rename-api-signature/README.md | 3 + .../rename-api-signature/docs/API.md | 11 + .../rename-api-signature/scripts/audit.py | 11 + .../rename-api-signature/shop/billing.py | 14 + .../rename-api-signature/shop/db.py | 20 + .../rename-api-signature/shop/notify.py | 10 + .../rename-api-signature/shop/plugins.json | 1 + .../rename-api-signature/shop/views.py | 19 + .../rename-api-signature/tests/test_shop.py | 36 + .../reference_solutions/semver-sort/vsort.py | 39 + .../sessionize-logs/sessions.py | 60 + .../slugify-batch/slugify.py | 39 + .../sql-longest-gaps/solve.sh | 26 + .../sqlite-migrate/migrate.py | 50 + .../reference_solutions/text-wrap-cli/wrap.py | 42 + .../vendor-units-workaround/app/retry.py | 29 + bench/local/regrade.py | 3 +- bench/local/rejudge.py | 156 + bench/local/replay_common.py | 119 + bench/local/run.py | 166 +- bench/local/scoreboard.py | 197 ++ bench/local/stats.py | 81 + bench/local/tasks3.py | 2681 +++++++++++++++++ bench/local/tests/test_provider_cap.py | 24 + bench/local/tests/test_scoreboard.py | 177 ++ bench/local/validate3.py | 108 + docs/missions.md | 6 +- docs/prior-art.md | 6 +- finetune/extract.mjs | 32 +- src/acp.ts | 131 +- src/ask.ts | 128 +- src/autonomy.ts | 89 +- src/background.ts | 2 +- src/bar.ts | 6 +- src/brief.ts | 2 +- src/checklint.ts | 366 +++ src/cli.tsx | 225 +- src/criteria.ts | 359 ++- src/dispute.ts | 4 +- src/endpoint.ts | 8 +- src/engine.ts | 1040 ++++++- src/gitbend.ts | 57 + src/judge.ts | 31 + src/leftovers.ts | 21 + src/lenient-json.ts | 64 + src/localgate.ts | 90 + src/mission.ts | 2 +- src/opencode.ts | 208 ++ src/providers.ts | 65 +- src/receipts.ts | 102 +- src/reference.ts | 416 +++ src/run.ts | 27 +- src/signout.ts | 106 + src/stream.ts | 209 +- src/tiers.ts | 200 ++ src/toolargs.ts | 100 + src/transcript.ts | 15 +- src/types.ts | 43 + src/watchdog.ts | 109 +- test/acp-agent.ts | 84 +- test/acp-seal-gate.test.ts | 65 + test/acp.test.ts | 123 + test/ask.test.ts | 9 +- test/autonomy-probe.test.ts | 4 +- test/autonomy.test.ts | 2 +- test/checklint.test.ts | 263 ++ test/drafter-view.test.ts | 142 + test/ended-with-verdict.test.ts | 220 ++ test/exit-codes.test.ts | 2 +- test/finetune-extract.test.ts | 83 + test/gate-unattended.test.ts | 166 + test/harness.test.ts | 83 +- test/judge.test.ts | 60 + test/judgment.test.ts | 2 +- test/late-criteria.test.ts | 122 + test/late-seal.test.ts | 89 + test/latency-learner.test.ts | 102 + test/lazy-seal.test.ts | 2 +- test/leftover-files.test.ts | 59 + test/lenient-json.test.ts | 39 + test/local-ask.test.ts | 143 + test/local-speed.test.ts | 35 + test/localgate.test.ts | 152 + test/model-quirks.test.ts | 206 ++ test/opencode.test.ts | 231 ++ test/openrouter-provider.test.ts | 19 + test/provider-errors.test.ts | 187 ++ test/quota-wall.test.ts | 35 + test/reference.test.ts | 245 ++ test/reveal-default.test.ts | 272 ++ test/review-advisory.test.ts | 51 + test/review.test.ts | 5 +- test/sharper-checks.test.ts | 135 +- test/signout.test.ts | 182 ++ test/tiers.test.ts | 169 ++ test/toolargs.test.ts | 64 + test/turn-loss.test.ts | 10 +- 183 files changed, 16371 insertions(+), 262 deletions(-) create mode 100644 bench/local/ambiguity-results/others16.jsonl create mode 100644 bench/local/ambiguity-results/run1-tools-narration-INVALID.jsonl create mode 100644 bench/local/ambiguity-results/run2-tool-stalls-partial.jsonl create mode 100644 bench/local/ambiguity-results/traps3.jsonl create mode 100644 bench/local/ambiguity_probe.py create mode 100644 bench/local/ambiguity_probe_driver.mjs create mode 100644 bench/local/ambiguity_probe_prompt.txt create mode 100644 bench/local/checkquality_analysis.py create mode 100644 bench/local/checkquality_lint.py create mode 100644 bench/local/checkquality_lintts.mjs create mode 100644 bench/local/checkquality_lintts.py create mode 100644 bench/local/checkquality_load.py create mode 100644 bench/local/checkquality_run.py create mode 100644 bench/local/checkquality_taxonomy.py create mode 100644 bench/local/checkquality_tiers.py create mode 100644 bench/local/checkquality_trees.py create mode 100644 bench/local/container/Dockerfile create mode 100644 bench/local/container/Dockerfile.grok create mode 100644 bench/local/container/Dockerfile.opencode create mode 100755 bench/local/container/run-in-container.sh create mode 100644 bench/local/crosscheck.py create mode 100644 bench/local/denials.py create mode 100644 bench/local/env_values.py create mode 100644 bench/local/frame_intact.py create mode 100644 bench/local/oracle_prompt.txt create mode 100644 bench/local/oracle_replay.py create mode 100644 bench/local/path_rule_count.py create mode 100644 bench/local/reference_solutions/_wrong/async-pool-node/1/pool.js create mode 100644 bench/local/reference_solutions/_wrong/async-pool-node/2/pool.js create mode 100644 bench/local/reference_solutions/_wrong/bash-backup/1/backup.sh create mode 100644 bench/local/reference_solutions/_wrong/bash-backup/2/backup.sh create mode 100644 bench/local/reference_solutions/_wrong/csv-to-json-node/1/csv2json.js create mode 100644 bench/local/reference_solutions/_wrong/dedupe-contacts/1/dedupe.py create mode 100644 bench/local/reference_solutions/_wrong/dedupe-contacts/2/dedupe.py create mode 100644 bench/local/reference_solutions/_wrong/fix-daily-buckets/1/bucket.py create mode 100644 bench/local/reference_solutions/_wrong/fix-daily-buckets/2/bucket.py create mode 100644 bench/local/reference_solutions/_wrong/git-find-culprit/1/solve.sh create mode 100644 bench/local/reference_solutions/_wrong/git-find-culprit/2/solve.sh create mode 100644 bench/local/reference_solutions/_wrong/git-split-lib-history/1/solve.sh create mode 100644 bench/local/reference_solutions/_wrong/git-split-lib-history/2/solve.sh create mode 100644 bench/local/reference_solutions/_wrong/ini-to-json/1/ini2json.py create mode 100644 bench/local/reference_solutions/_wrong/ini-to-json/2/solve.sh create mode 100644 bench/local/reference_solutions/_wrong/json-diff-cli/1/jdiff.py create mode 100644 bench/local/reference_solutions/_wrong/ledger-balance/1/ledger.py create mode 100644 bench/local/reference_solutions/_wrong/make-incremental/1/Makefile create mode 100644 bench/local/reference_solutions/_wrong/make-incremental/2/Makefile create mode 100644 bench/local/reference_solutions/_wrong/money-split/1/alloc.py create mode 100644 bench/local/reference_solutions/_wrong/money-split/2/alloc.py create mode 100644 bench/local/reference_solutions/_wrong/perf-asof-join/1/join.py create mode 100644 bench/local/reference_solutions/_wrong/refactor-invoice/1/solve.sh create mode 100644 bench/local/reference_solutions/_wrong/refactor-invoice/2/solve.sh create mode 100644 bench/local/reference_solutions/_wrong/rename-api-signature/1/solve.sh create mode 100644 bench/local/reference_solutions/_wrong/rename-api-signature/2/solve.sh create mode 100644 bench/local/reference_solutions/_wrong/semver-sort/1/vsort.py create mode 100644 bench/local/reference_solutions/_wrong/sessionize-logs/1/sessions.py create mode 100644 bench/local/reference_solutions/_wrong/sessionize-logs/2/solve.sh create mode 100644 bench/local/reference_solutions/_wrong/slugify-batch/1/slugify.py create mode 100644 bench/local/reference_solutions/_wrong/sql-longest-gaps/1/solve.sh create mode 100644 bench/local/reference_solutions/_wrong/sql-longest-gaps/2/solve.sh create mode 100644 bench/local/reference_solutions/_wrong/sqlite-migrate/1/migrate.py create mode 100644 bench/local/reference_solutions/_wrong/sqlite-migrate/2/solve.sh create mode 100644 bench/local/reference_solutions/_wrong/text-wrap-cli/1/wrap.py create mode 100644 bench/local/reference_solutions/_wrong/text-wrap-cli/2/wrap.py create mode 100644 bench/local/reference_solutions/_wrong/vendor-units-workaround/1/solve.sh create mode 100644 bench/local/reference_solutions/_wrong/vendor-units-workaround/2/app/retry.py create mode 100644 bench/local/reference_solutions/async-pool-node/pool.js create mode 100644 bench/local/reference_solutions/bash-backup/backup.sh create mode 100644 bench/local/reference_solutions/csv-to-json-node/csv2json.js create mode 100644 bench/local/reference_solutions/dedupe-contacts/dedupe.py create mode 100644 bench/local/reference_solutions/dedupe-contacts/solve.sh create mode 100644 bench/local/reference_solutions/fix-daily-buckets/bucket.py create mode 100644 bench/local/reference_solutions/fix-daily-buckets/report.py create mode 100644 bench/local/reference_solutions/git-find-culprit/solve.sh create mode 100644 bench/local/reference_solutions/git-split-lib-history/solve.sh create mode 100644 bench/local/reference_solutions/ini-to-json/ini2json.py create mode 100644 bench/local/reference_solutions/json-diff-cli/jdiff.py create mode 100644 bench/local/reference_solutions/ledger-balance/ledger.py create mode 100644 bench/local/reference_solutions/make-incremental/Makefile create mode 100644 bench/local/reference_solutions/money-split/alloc.py create mode 100644 bench/local/reference_solutions/perf-asof-join/join.py create mode 100644 bench/local/reference_solutions/refactor-invoice/format.py create mode 100644 bench/local/reference_solutions/refactor-invoice/invoice.py create mode 100644 bench/local/reference_solutions/refactor-invoice/tax.py create mode 100644 bench/local/reference_solutions/rename-api-signature/README.md create mode 100644 bench/local/reference_solutions/rename-api-signature/docs/API.md create mode 100644 bench/local/reference_solutions/rename-api-signature/scripts/audit.py create mode 100644 bench/local/reference_solutions/rename-api-signature/shop/billing.py create mode 100644 bench/local/reference_solutions/rename-api-signature/shop/db.py create mode 100644 bench/local/reference_solutions/rename-api-signature/shop/notify.py create mode 100644 bench/local/reference_solutions/rename-api-signature/shop/plugins.json create mode 100644 bench/local/reference_solutions/rename-api-signature/shop/views.py create mode 100644 bench/local/reference_solutions/rename-api-signature/tests/test_shop.py create mode 100644 bench/local/reference_solutions/semver-sort/vsort.py create mode 100644 bench/local/reference_solutions/sessionize-logs/sessions.py create mode 100644 bench/local/reference_solutions/slugify-batch/slugify.py create mode 100644 bench/local/reference_solutions/sql-longest-gaps/solve.sh create mode 100644 bench/local/reference_solutions/sqlite-migrate/migrate.py create mode 100644 bench/local/reference_solutions/text-wrap-cli/wrap.py create mode 100644 bench/local/reference_solutions/vendor-units-workaround/app/retry.py create mode 100644 bench/local/rejudge.py create mode 100644 bench/local/replay_common.py create mode 100644 bench/local/scoreboard.py create mode 100644 bench/local/stats.py create mode 100644 bench/local/tasks3.py create mode 100644 bench/local/tests/test_provider_cap.py create mode 100644 bench/local/tests/test_scoreboard.py create mode 100644 bench/local/validate3.py create mode 100644 src/checklint.ts create mode 100644 src/gitbend.ts create mode 100644 src/judge.ts create mode 100644 src/lenient-json.ts create mode 100644 src/localgate.ts create mode 100644 src/opencode.ts create mode 100644 src/reference.ts create mode 100644 src/signout.ts create mode 100644 src/tiers.ts create mode 100644 src/toolargs.ts create mode 100644 test/acp-seal-gate.test.ts create mode 100644 test/checklint.test.ts create mode 100644 test/drafter-view.test.ts create mode 100644 test/ended-with-verdict.test.ts create mode 100644 test/finetune-extract.test.ts create mode 100644 test/gate-unattended.test.ts create mode 100644 test/judge.test.ts create mode 100644 test/late-criteria.test.ts create mode 100644 test/late-seal.test.ts create mode 100644 test/latency-learner.test.ts create mode 100644 test/leftover-files.test.ts create mode 100644 test/lenient-json.test.ts create mode 100644 test/local-ask.test.ts create mode 100644 test/local-speed.test.ts create mode 100644 test/localgate.test.ts create mode 100644 test/model-quirks.test.ts create mode 100644 test/opencode.test.ts create mode 100644 test/openrouter-provider.test.ts create mode 100644 test/provider-errors.test.ts create mode 100644 test/quota-wall.test.ts create mode 100644 test/reference.test.ts create mode 100644 test/reveal-default.test.ts create mode 100644 test/review-advisory.test.ts create mode 100644 test/signout.test.ts create mode 100644 test/tiers.test.ts create mode 100644 test/toolargs.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 72209de..9bcdda4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ written. Stale `claude-code://`, `antigravity://`, `agy://`, and `gemini-cli://` configs now fail with a clear removal message. Grok Build (ACP) and the metered xAI API provider remain. -- **Droid benchmarking** from the harbor/local bench path, and **xAI rows** +- **Competitor benchmarking** from the harbor/local bench path, and **xAI rows** from the committed fine-tune dataset (provider-terms alignment). See `docs/provider-terms.md`. diff --git a/bench/harbor/compare.py b/bench/harbor/compare.py index c88c997..2cd9195 100755 --- a/bench/harbor/compare.py +++ b/bench/harbor/compare.py @@ -30,6 +30,10 @@ from datetime import datetime from pathlib import Path +# Shared with bench/local/scoreboard.py (Wilson intervals, one definition). +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "local")) +from stats import rate # noqa: E402 + def outcome_of(trial: Path) -> str | None: """ @@ -215,6 +219,15 @@ def prec(h): return f"{100 * h['claimed_right'] / (h['claimed_right'] + h['claimed_wrong']):.0f}%" print(f"{' done was true':16} {prec(ha):>{w}} {prec(hb):>{w}}") + def pr(h, kind): + if h is None: + return "—" + r, wr, un = h["claimed_right"], h["claimed_wrong"], h["passed_unclaimed"] + return rate(r, r + wr) if kind == "p" else rate(r, r + un) + + # Wilson 95% intervals: at 89 tasks a 5-task gap is noise, so say how wide. + print(f"{'precision':16} {pr(ha, 'p')} | {pr(hb, 'p')} P(pass | said done)") + print(f"{'recall':16} {pr(ha, 'r')} | {pr(hb, 'r')} P(said done | pass)") only_a, only_b = sorted(set(ta) - set(tb)), sorted(set(tb) - set(ta)) if only_a or only_b: print(f"not compared — only in {na}: {len(only_a)} task(s); only in {nb}: {len(only_b)} task(s)") diff --git a/bench/harbor/molt_agent.py b/bench/harbor/molt_agent.py index 7af684c..4a7d6ae 100644 --- a/bench/harbor/molt_agent.py +++ b/bench/harbor/molt_agent.py @@ -75,6 +75,7 @@ def __init__( steps: int = 300, batch: bool | str = False, reveal_stuck: bool | str = False, + reference: bool | str = False, reasoning_checks: str | None = None, reasoning_retry: str | None = None, review: int | str | None = None, @@ -104,6 +105,8 @@ def __init__( # Batch mode: one act call per reply carrying a list of actions. self._batch = str(batch).lower() in ("1", "true", "yes", "on") self._reveal_stuck = str(reveal_stuck).lower() in ("1", "true", "yes", "on") + # An independent reference check (src/reference.ts); needs python3 in the image. + self._reference = str(reference).lower() in ("1", "true", "yes", "on") # Effort for drafting checks only, and for steps after a refusal. self._reasoning_checks = reasoning_checks self._reasoning_retry = reasoning_retry @@ -230,10 +233,14 @@ async def run( flags.append("--criteria auto") if self._for: flags.append(f"--for {shlex.quote(str(self._for))}") - elif self._pace: + else: + # Every run gets a deadline: a run the harness kills at its limit + # ends with no verdict, while one stopped by --for is judged on the + # tree it left. Paced runs keep a 15% margin; the rest stop a + # minute before the limit, enough to run the sealed checks. limit = self._task_timeout_sec() if limit: - flags.append(f"--for {max(60, int(limit * 0.85) - 60)}s") + flags.append(f"--for {max(60, int(limit * 0.85) - 60) if self._pace else max(60, int(limit) - 60)}s") if self._map_tokens is not None: flags.append("--no-map" if int(self._map_tokens) <= 0 else f"--map {int(self._map_tokens)}") if self._max_tokens is not None: @@ -247,6 +254,8 @@ async def run( flags.append(f"--steps {self._steps}") if self._batch: flags.append("--batch") + if self._reference: + flags.append("--reference") if self._reveal_stuck: flags.append("--reveal-stuck") if self._reasoning_checks: diff --git a/bench/local/ambiguity-results/others16.jsonl b/bench/local/ambiguity-results/others16.jsonl new file mode 100644 index 0000000..83f029f --- /dev/null +++ b/bench/local/ambiguity-results/others16.jsonl @@ -0,0 +1,16 @@ +{"id":"duration-bug#0","secs":18,"ok":true,"text":"{\"probes\":[]}","cutOff":false} +{"id":"fix-git#0","secs":122,"ok":true,"text":"{\"probes\":[{\"input\":\"master: about.md='Original about\\\\n', index.md='Home\\\\n'. Other branch commits: (1) about.md='Updated about\\\\n'; (2) about.md='Original about\\\\n' and index.md='Home updated\\\\n'. Tip is commit 2.\",\"readingA\":\"Bring onto master only the edit to the about page, using the commit that changed about.md.\",\"outputA\":\"about.md='Updated about\\\\n', index.md='Home\\\\n'\",\"readingB\":\"Merge that whole branch into master, so the tip result is what master gets.\",\"outputB\":\"about.md='Original about\\\\n', index.md='Home updated\\\\n'\",\"quote\":\"changes to my site's about page ... merge them into master\",\"dimension\":\"about-only edit vs whole branch tip\"},{\"input\":\"stash@{0} about.md='From stash\\\\n'; branch topic about.md='From branch\\\\n'; master about.md='Original\\\\n'.\",\"readingA\":\"The missing edits were stashed when master was checked out, so stash@{0} is applied onto master.\",\"outputA\":\"about.md='From stash\\\\n'\",\"readingB\":\"Checking out master left the edits on branch topic, so topic is merged into master.\",\"outputB\":\"about.md='From branch\\\\n'\",\"quote\":\"checked out master, and now I can't find those changes. Find them\",\"dimension\":\"stash vs other branch\"},{\"input\":\"stash@{0} sets only index.md to 'X\\\\n'. stash@{1} sets about.md to 'Recovered about\\\\n'. master: about.md='Original\\\\n', index.md='Home\\\\n'.\",\"readingA\":\"The lost work is the newest stash, so stash@{0} is applied.\",\"outputA\":\"about.md='Original\\\\n', index.md='X\\\\n'\",\"readingB\":\"The lost work is the stash that edits the about page, so stash@{1} is applied.\",\"outputB\":\"about.md='Recovered about\\\\n', index.md='Home\\\\n'\",\"quote\":\"changes to my site's about page\",\"dimension\":\"which stash wins\"},{\"input\":\"master about.md line is 'We ship software.\\\\n'; the recovered about.md line is 'We ship great software.\\\\n' (same line on both sides).\",\"readingA\":\"The recovered about-page text is the version that ends up on master.\",\"outputA\":\"We ship great software.\\\\n\",\"readingB\":\"Merge stops with both sides kept as conflict markers because no resolution was specified.\",\"outputB\":\"<<<<<<< HEAD\\\\nWe ship software.\\\\n=======\\\\nWe ship great software.\\\\n>>>>>>> recovered\\\\n\",\"quote\":\"merge them into master\",\"dimension\":\"conflict resolution\"}]}","cutOff":false} +{"id":"wc-tool#0","secs":394,"ok":true,"text":"{\"probes\":[{\"input\":\"f contains hello (bytes 68 65 6c 6c 6f, no newline)\",\"readingA\":\"Line count is the number of LF bytes, so this file has zero lines, one word, and five bytes.\",\"outputA\":\"0 1 5 f\",\"readingB\":\"Line count is the number of text lines read, and the final unterminated record counts as a line.\",\"outputB\":\"1 1 5 f\",\"quote\":\"line count\",\"dimension\":\"partial final line\"},{\"input\":\"f contains hello\\\\rworld (11 bytes, no LF)\",\"readingA\":\"Only LF ends a line. CR is a word separator, so the file has zero lines and two words.\",\"outputA\":\"0 2 11 f\",\"readingB\":\"CR is a line break (universal newlines), so the file has two lines and two words.\",\"outputB\":\"2 2 11 f\",\"quote\":\"line count\",\"dimension\":\"CR as line break\"},{\"input\":\"f contains hello\\\\fworld\\\\n (12 bytes)\",\"readingA\":\"Line count counts LF only. Form feed separates words and does not end a line.\",\"outputA\":\"1 2 12 f\",\"readingB\":\"Line count follows str.splitlines(), which treats form feed as a line boundary.\",\"outputB\":\"2 2 12 f\",\"quote\":\"line count\",\"dimension\":\"form feed as line break\"},{\"input\":\"f contains UTF-8 hello\\\\xc2\\\\xa0world\\\\n (U+00A0 between the words, 13 bytes)\",\"readingA\":\"A word is a maximal run outside POSIX whitespace (space, tab, LF, VT, FF, CR). NBSP is inside the word, so one word.\",\"outputA\":\"1 1 13 f\",\"readingB\":\"A word is a maximal run split on Unicode whitespace, as in str.split(). NBSP yields two words.\",\"outputB\":\"1 2 13 f\",\"quote\":\"word count\",\"dimension\":\"NBSP as word separator\"},{\"input\":\"f contains foo-bar\\\\n\",\"readingA\":\"A word is a maximal non-whitespace run, so foo-bar is one word.\",\"outputA\":\"1 1 8 f\",\"readingB\":\"A word is a maximal alphanumeric run, so the hyphen splits it into two words.\",\"outputB\":\"1 2 8 f\",\"quote\":\"word count\",\"dimension\":\"hyphen as word boundary\"},{\"input\":\"python3 wc.py sub/notes.txt with sub/notes.txt containing hi\\\\n\",\"readingA\":\"The printed file name is the operand path as given.\",\"outputA\":\"1 1 3 sub/notes.txt\",\"readingB\":\"The printed file name is the basename of that path.\",\"outputB\":\"1 1 3 notes.txt\",\"quote\":\"then the file name\",\"dimension\":\"path vs basename\"},{\"input\":\"python3 wc.py f f with f containing a\\\\n\",\"readingA\":\"Each operand is reported, including a repeated path, and two operands produce a total that sums both passes.\",\"outputA\":\"1 1 2 f\\n1 1 2 f\\n2 2 4 total\",\"readingB\":\"Each distinct file is reported once, so a repeated path is one file and there is no total line.\",\"outputB\":\"1 1 2 f\",\"quote\":\"for each file; with more than one file, a final line with the totals\",\"dimension\":\"repeated path operand\"}]}","cutOff":false} +{"id":"log-summary#0","secs":415,"ok":true,"text":"{\"probes\":[{\"input\":\"logs/2025-08-12.log contains only `[INFO] started`; every other file in logs/ is empty\",\"readingA\":\"Emit the full cross product: one row for every period and every listed severity, with count 0 when none occur\",\"outputA\":\"today,ERROR,0 / today,WARNING,0 / today,INFO,1 and the same 0,0,1 pattern for last_7_days and for total\",\"readingB\":\"Emit a row only for severities that actually occur\",\"outputB\":\"today,INFO,1 / last_7_days,INFO,1 / total,INFO,1 and no ERROR or WARNING rows\",\"quote\":\"one row per period and severity, for the periods today, last_7_days ... and total, in that order, severities in the order ERROR, WARNING, INFO\",\"dimension\":\"zero-count rows\"},{\"input\":\"`[INFO] saw [ERROR]` is the only line in logs/2025-08-12.log; every other log file is empty\",\"readingA\":\"A line has one severity, the first bracketed tag, so the line counts only as INFO\",\"outputA\":\"ERROR count 0 and INFO count 1 for today, for last_7_days, and for total\",\"readingB\":\"The line counts once toward every distinct bracketed severity it contains\",\"outputB\":\"ERROR count 1 and INFO count 1 for today, for last_7_days, and for total\",\"quote\":\"each line has a severity in brackets, [ERROR], [WARNING] or [INFO]\",\"dimension\":\"multiple severities on one line\"},{\"input\":\"`[ERROR] [ERROR]` is the only line in logs/2025-08-12.log; every other log file is empty\",\"readingA\":\"Count matching lines, so one line contributes 1\",\"outputA\":\"ERROR count 1 for today, last_7_days, and total\",\"readingB\":\"Count each bracketed occurrence on the line\",\"outputB\":\"ERROR count 2 for today, last_7_days, and total\",\"quote\":\"each line has a severity in brackets\",\"dimension\":\"lines versus occurrences\"},{\"input\":\"`[ERROR] boundary` is the only line in logs/2025-08-05.log; every other log file is empty\",\"readingA\":\"total counts every daily file in logs/, while last_7_days is only 2025-08-06 through 2025-08-12\",\"outputA\":\"today ERROR 0, last_7_days ERROR 0, total ERROR 1\",\"readingB\":\"total is the same span as today and the 6 days before it, so 2025-08-05 is outside total\",\"outputB\":\"today ERROR 0, last_7_days ERROR 0, total ERROR 0\",\"quote\":\"The logs/ directory holds one file per day ... periods today, last_7_days (today and the 6 days before it) and total\",\"dimension\":\"scope of total\"},{\"input\":\"`[ERROR] disk full` is the only line in logs/2025-08-12.log; every other log file is empty\",\"readingA\":\"The severity column uses the bare names from the order list\",\"outputA\":\"today,ERROR,1\",\"readingB\":\"The severity column uses the bracketed forms given for log lines\",\"outputB\":\"today,[ERROR],1\",\"quote\":\"severities in the order ERROR, WARNING, INFO vs [ERROR], [WARNING] or [INFO]\",\"dimension\":\"severity label format\"},{\"input\":\"`[error] disk full` is the only line in logs/2025-08-12.log; every other log file is empty\",\"readingA\":\"Match the uppercase bracket form exactly, so `[error]` is not a severity\",\"outputA\":\"ERROR count 0 for today, last_7_days, and total\",\"readingB\":\"The names ERROR, WARNING, and INFO match case-insensitively\",\"outputB\":\"ERROR count 1 for today, last_7_days, and total\",\"quote\":\"[ERROR], [WARNING] or [INFO]\",\"dimension\":\"severity case\"},{\"input\":\"logs/2025-08-12.log has `[ERROR] a`, `[WARNING] b`, and `[INFO] c`; logs/2025-08-11.log has `[ERROR] d`; every other file is empty\",\"readingA\":\"Period is the outer key: for each period in order, severities ERROR, WARNING, INFO\",\"outputA\":\"today,ERROR,1 then today,WARNING,1 then today,INFO,1 then last_7_days,ERROR,2 then last_7_days,WARNING,1 then last_7_days,INFO,1 then total,ERROR,2 then total,WARNING,1 then total,INFO,1\",\"readingB\":\"Severity is the outer key: for each severity in order, periods today, last_7_days, total\",\"outputB\":\"today,ERROR,1 then last_7_days,ERROR,2 then total,ERROR,2 then today,WARNING,1 then last_7_days,WARNING,1 then total,WARNING,1 then today,INFO,1 then last_7_days,INFO,1 then total,INFO,1\",\"quote\":\"periods today, last_7_days (today and the 6 days before it) and total, in that order, severities in the order ERROR, WARNING, INFO\",\"dimension\":\"row order primary key\"}]}","cutOff":false} +{"id":"http-json-server#0","secs":420,"ok":true,"text":"{\"probes\":[{\"input\":\"items.json is [] and POST /items has body {\\\"name\\\": \\\"a\\\"}\",\"readingA\":\"When no id exists, the next id is 1.\",\"outputA\":\"201 {\\\"id\\\": 1, \\\"name\\\": \\\"a\\\"} and items.json becomes [{\\\"id\\\": 1, \\\"name\\\": \\\"a\\\"}]\",\"readingB\":\"When no id exists, the next id is 0.\",\"outputB\":\"201 {\\\"id\\\": 0, \\\"name\\\": \\\"a\\\"} and items.json becomes [{\\\"id\\\": 0, \\\"name\\\": \\\"a\\\"}]\",\"quote\":\"highest existing id + 1\",\"dimension\":\"first id on an empty list\"},{\"input\":\"items.json is [{\\\"id\\\": 1, \\\"name\\\": \\\"a\\\"}] and the request is GET /items/01\",\"readingA\":\"The segment is parsed as an integer, so 01 selects id 1.\",\"outputA\":\"200 {\\\"id\\\": 1, \\\"name\\\": \\\"a\\\"}\",\"readingB\":\"The segment must equal the stored id, and 01 is a different id.\",\"outputB\":\"404 {\\\"error\\\": \\\"not found\\\"}\",\"quote\":\"GET /items/ returns that one item, or status 404\",\"dimension\":\"leading zeros in the id\"},{\"input\":\"items.json is [{\\\"id\\\": 3, \\\"name\\\": \\\"c\\\"}] and POST /items/ has body {\\\"name\\\": \\\"a\\\"}\",\"readingA\":\"A trailing slash still selects POST /items and creates an item.\",\"outputA\":\"201 {\\\"id\\\": 4, \\\"name\\\": \\\"a\\\"} and the file contains that item\",\"readingB\":\"Only the exact path /items creates an item, so /items/ leaves the file unchanged.\",\"outputB\":\"404 and items.json stays [{\\\"id\\\": 3, \\\"name\\\": \\\"c\\\"}]\",\"quote\":\"POST /items with a JSON body\",\"dimension\":\"trailing slash on create\"},{\"input\":\"items.json is [{\\\"id\\\": 1, \\\"name\\\": \\\"a\\\"}, {\\\"id\\\": 1, \\\"name\\\": \\\"b\\\"}] and the request is GET /items/1\",\"readingA\":\"The earliest stored object with that id is returned.\",\"outputA\":\"200 {\\\"id\\\": 1, \\\"name\\\": \\\"a\\\"}\",\"readingB\":\"The latest stored object with that id is returned.\",\"outputB\":\"200 {\\\"id\\\": 1, \\\"name\\\": \\\"b\\\"}\",\"quote\":\"GET /items/ returns that one item\",\"dimension\":\"duplicate ids\"},{\"input\":\"items.json is [{\\\"id\\\": 1, \\\"name\\\": \\\"a\\\"}] and the request is GET /items/1?x=1\",\"readingA\":\"The query string is ignored and is 1.\",\"outputA\":\"200 {\\\"id\\\": 1, \\\"name\\\": \\\"a\\\"}\",\"readingB\":\"The target /items/1?x=1 does not select stored id 1.\",\"outputB\":\"404 {\\\"error\\\": \\\"not found\\\"}\",\"quote\":\"GET /items/ returns that one item\",\"dimension\":\"query string on item URL\"},{\"input\":\"items.json is [{\\\"id\\\": 1, \\\"name\\\": \\\"a\\\"}] and the request is GET /items/%31\",\"readingA\":\"The segment is percent-decoded, so is 1.\",\"outputA\":\"200 {\\\"id\\\": 1, \\\"name\\\": \\\"a\\\"}\",\"readingB\":\"The segment is the literal text %31, which matches no stored id.\",\"outputB\":\"404 {\\\"error\\\": \\\"not found\\\"}\",\"quote\":\"GET /items/\",\"dimension\":\"percent-encoding of the id\"},{\"input\":\"items.json is [{\\\"id\\\": 1, \\\"name\\\": \\\"a\\\"}] and the request is GET /items/1.0\",\"readingA\":\"The segment is numeric and 1.0 equals stored id 1.\",\"outputA\":\"200 {\\\"id\\\": 1, \\\"name\\\": \\\"a\\\"}\",\"readingB\":\"The segment must be the canonical integer text 1, so 1.0 matches nothing.\",\"outputB\":\"404 {\\\"error\\\": \\\"not found\\\"}\",\"quote\":\"GET /items/ returns that one item, or status 404\",\"dimension\":\"decimal spelling of an integer id\"},{\"input\":\"items.json is [{\\\"id\\\": 1, \\\"name\\\": \\\"lamp\\\", \\\"qty\\\": 2}] and the request is GET /items/1\",\"readingA\":\"The stored object is returned with every field.\",\"outputA\":\"200 {\\\"id\\\": 1, \\\"name\\\": \\\"lamp\\\", \\\"qty\\\": 2}\",\"readingB\":\"The response object contains only id and name.\",\"outputB\":\"200 {\\\"id\\\": 1, \\\"name\\\": \\\"lamp\\\"}\",\"quote\":\"returns that one item\",\"dimension\":\"extra stored fields\"}]}","cutOff":false} +{"id":"sql-top-customers#0","secs":493,"ok":true,"text":"{\"probes\":[{\"input\":\"Ann has one paid order dated 2024-06-01 with a single item qty=1, unit_price=2.225; no other customers\",\"readingA\":\"Format the sum with exactly two decimals using round-half-away-from-zero\",\"outputA\":\"name,total\\nAnn,2.23\",\"readingB\":\"Format the sum with exactly two decimals using round-half-to-even\",\"outputB\":\"name,total\\nAnn,2.22\",\"quote\":\"write total with exactly two decimals (e.g. 1234.50)\",\"dimension\":\"rounding mode at exact half\"},{\"input\":\"Paid 2024-06-01 orders only: Ann total 50, Bo 40, Cy 30, Di 20, Eve sum 9.996, Fay sum 10.004\",\"readingA\":\"Rank by the exact sum of qty*unit_price, then print two decimals\",\"outputA\":\"name,total\\nAnn,50.00\\nBo,40.00\\nCy,30.00\\nDi,20.00\\nFay,10.00\",\"readingB\":\"Rank by the already-rounded two-decimal total, and break remaining ties by name ascending\",\"outputB\":\"name,total\\nAnn,50.00\\nBo,40.00\\nCy,30.00\\nDi,20.00\\nEve,10.00\",\"quote\":\"Order rows by total descending, ties by name ascending, and write total with exactly two decimals\",\"dimension\":\"rank on raw sum vs rounded total\"},{\"input\":\"Customers Ana, Bo, Cy, Di, Ed. Only Ana has an order: status paid, date 2024-03-01, qty=2, unit_price=20. The other four have no orders\",\"readingA\":\"Candidate set is customers who have at least one paid 2024 order\",\"outputA\":\"name,total\\nAna,40.00\",\"readingB\":\"Every customer has a spend on those orders (0 if none) and the top 5 of that score are written\",\"outputB\":\"name,total\\nAna,40.00\\nBo,0.00\\nCy,0.00\\nDi,0.00\\nEd,0.00\",\"quote\":\"one row for each of the 5 customers who spent the most on orders with status 'paid' placed in 2024\",\"dimension\":\"fewer than 5 qualifiers vs zero-filled top 5\"},{\"input\":\"Paid 2024 orders: customer id 1 name Alex spend 100, id 2 name Alex spend 50, Blake 80, Casey 70, Drew 60, Evan 40\",\"readingA\":\"One row per customer id, labeled with that customer's name\",\"outputA\":\"name,total\\nAlex,100.00\\nBlake,80.00\\nCasey,70.00\\nDrew,60.00\\nAlex,50.00\",\"readingB\":\"Group and rank by the name string, summing every customer who shares it\",\"outputB\":\"name,total\\nAlex,150.00\\nBlake,80.00\\nCasey,70.00\\nDrew,60.00\\nEvan,40.00\",\"quote\":\"the 5 customers who spent the most\",\"dimension\":\"identity is customer id vs name\"},{\"input\":\"Five paid 2024 orders of qty=1, unit_price=100 for customers named amy, Bob, cy, Di, and Eve\",\"readingA\":\"Name ascending is binary/ASCII order, so uppercase letters sort before lowercase\",\"outputA\":\"name,total\\nBob,100.00\\nDi,100.00\\nEve,100.00\\namy,100.00\\ncy,100.00\",\"readingB\":\"Name ascending is case-insensitive alphabetical order\",\"outputB\":\"name,total\\namy,100.00\\nBob,100.00\\ncy,100.00\\nDi,100.00\\nEve,100.00\",\"quote\":\"ties by name ascending\",\"dimension\":\"case sensitivity of name order\"},{\"input\":\"Sam has one order with status paid, order_date 2023-12-15, paid_at 2024-02-01, qty=1, unit_price=80; no other orders\",\"readingA\":\"Placed in 2024 filters the order placement date, so this order is outside 2024\",\"outputA\":\"name,total\",\"readingB\":\"The 2024 filter applies to when the paid order was paid (paid_at)\",\"outputB\":\"name,total\\nSam,80.00\",\"quote\":\"orders with status 'paid' placed in 2024\",\"dimension\":\"placement date vs paid date\"},{\"input\":\"One paid 2024 order for customer name Doe, Jane, qty=1, unit_price=10\",\"readingA\":\"CSV-quote a name that contains a comma\",\"outputA\":\"name,total\\n\\\"Doe, Jane\\\",10.00\",\"readingB\":\"Join name and total with a comma and no extra quoting\",\"outputB\":\"name,total\\nDoe, Jane,10.00\",\"quote\":\"Write top_customers.csv with the header name,total and one row\",\"dimension\":\"CSV quoting when name contains a comma\"}]}","cutOff":false} +{"id":"refactor-pricing#0","secs":551,"ok":true,"text":"{\"probes\":[{\"input\":\"best_coupon(\\\"widget\\\", 1) with no-coupon price 100 and COUPONS prices {\\\"SAVE\\\": 100}\",\"readingA\":\"None unless some coupon price is strictly less than the no-coupon price; an equal price does not win\",\"outputA\":\"None\",\"readingB\":\"An equal price is a tie, and ties are awarded to the alphabetically first code rather than None\",\"outputB\":\"SAVE\",\"quote\":\"or None if no coupon is strictly cheaper than no coupon; ties go to the alphabetically first code\",\"dimension\":\"equality vs strictly cheaper than no coupon\"},{\"input\":\"best_coupon(\\\"widget\\\", 1) with no-coupon price 100 and COUPONS prices {\\\"a\\\": 80, \\\"B\\\": 80}\",\"readingA\":\"Alphabetical order is case-sensitive code-point order, so \\\"B\\\" precedes \\\"a\\\"\",\"outputA\":\"B\",\"readingB\":\"Alphabetical order is case-insensitive, so \\\"a\\\" precedes \\\"B\\\"\",\"outputB\":\"a\",\"quote\":\"ties go to the alphabetically first code\",\"dimension\":\"case sensitivity of alphabetical tie-break\"},{\"input\":\"best_coupon(\\\"widget\\\", 1) with no-coupon price 100 and COUPONS prices {\\\"SAVE10\\\": 80, \\\"SAVE2\\\": 80}\",\"readingA\":\"Alphabetical means lexicographic character order, so \\\"SAVE10\\\" < \\\"SAVE2\\\" because \\\"1\\\" < \\\"2\\\"\",\"outputA\":\"SAVE10\",\"readingB\":\"Alphabetical means natural order, comparing digit runs as integers, so \\\"SAVE2\\\" < \\\"SAVE10\\\"\",\"outputB\":\"SAVE2\",\"quote\":\"ties go to the alphabetically first code\",\"dimension\":\"lexicographic vs natural sort of codes\"}]}","cutOff":false} +{"id":"csv-clean#0","secs":454,"ok":true,"text":"{\"probes\":[{\"input\":\"3,Mia,mia@ex.com, 40,Dallas\",\"readingA\":\"Trim every field first and then apply the age rule. Age becomes 40, which is inside 0 to 120, so the row is written with trimmed fields.\",\"outputA\":\"3,Mia,mia@ex.com,40,Dallas\",\"readingB\":\"Apply the age rule to the raw field text. The field is the two characters space and 40, so it fails the digits rule and the row is absent.\",\"outputB\":\"row absent\",\"quote\":\"trim whitespace around every field; age isn't a whole number written in digits from 0 to 120\",\"dimension\":\"trim before vs after age check\"},{\"input\":\"3,Mia, @ex.com,40,Dallas\",\"readingA\":\"Trim the email first. It becomes @ex.com, the local part is empty, and the row is absent.\",\"outputA\":\"row absent\",\"readingB\":\"Judge the raw email before trimming. The local part is one space, hence non-empty, and the domain ex.com has an internal dot, so the row is kept and then trimmed.\",\"outputB\":\"3,Mia,@ex.com,40,Dallas\",\"quote\":\"trim whitespace around every field; non-empty local part\",\"dimension\":\"trim before vs after email check\"},{\"input\":\"3,Mia,mia@ex.com,007,Dallas\",\"readingA\":\"An all-digit field whose integer value lies from 0 to 120 stays, with the trimmed spelling left as written.\",\"outputA\":\"3,Mia,mia@ex.com,007,Dallas\",\"readingB\":\"A whole number written in digits from 0 to 120 must be the canonical spelling, so a leading zero makes the row absent.\",\"outputB\":\"row absent\",\"quote\":\"a whole number written in digits from 0 to 120\",\"dimension\":\"leading zeros in age\"},{\"input\":\"3,Mia,mia@ex.com,25.0,Dallas\",\"readingA\":\"The age must be written using digit characters only, so 25.0 fails and the row is absent.\",\"outputA\":\"row absent\",\"readingB\":\"25.0 is a whole number in digit notation whose value is from 0 to 120, so the row is kept with age 25.0.\",\"outputB\":\"3,Mia,mia@ex.com,25.0,Dallas\",\"quote\":\"whole number written in digits from 0 to 120\",\"dimension\":\"decimal spelling of an integral age\"},{\"input\":\"3,Mia,mia@ex.com.,40,Dallas\",\"readingA\":\"Domain ex.com. contains a dot before com that is neither the domain's first nor last character, so the row is kept.\",\"outputA\":\"3,Mia,mia@ex.com.,40,Dallas\",\"readingB\":\"The domain must contain a dot and its first and last characters must not be dots, so a trailing dot removes the row.\",\"outputB\":\"row absent\",\"quote\":\"a domain containing a dot that is neither its first nor last character\",\"dimension\":\"trailing dot together with an internal dot\"},{\"input\":\"1,Ann,Ann@X.com,25,NYC\\n2,Bob,ann@x.com,30,LA\",\"readingA\":\"Lowercase the email before the duplicate pass. Both remaining rows have ann@x.com, so the later row is removed.\",\"outputA\":\"1,Ann,ann@x.com,25,NYC\",\"readingB\":\"Duplicate removal compares the email strings from the file, Ann@X.com and ann@x.com, and both rows stay; lowercasing is applied to each kept row.\",\"outputB\":\"1,Ann,ann@x.com,25,NYC\\n2,Bob,ann@x.com,30,LA\",\"quote\":\"lowercase the email; drop any row whose email already appeared in an earlier remaining row\",\"dimension\":\"dedup before vs after lowercasing\"},{\"input\":\"4,\\\"Jane\\\",jane@x.com,40,Austin\",\"readingA\":\"Commas alone separate fields and quote marks are field text, so the name cell stays quoted.\",\"outputA\":\"4,\\\"Jane\\\",jane@x.com,40,Austin\",\"readingB\":\"Quote marks are CSV syntax, the name field is Jane, and the plain comma-separated row has no quote marks.\",\"outputB\":\"4,Jane,jane@x.com,40,Austin\",\"quote\":\"exactly 5 fields; write plain comma-separated values\",\"dimension\":\"quotes as data vs CSV syntax\"},{\"input\":\"4,\\\"Doe, Jane\\\",jane@x.com,40,Austin\",\"readingA\":\"Every comma separates a field, giving six fields, so the row is absent.\",\"outputA\":\"row absent\",\"readingB\":\"Quoted commas stay inside one field, leaving five fields; those fields are joined with plain commas.\",\"outputB\":\"4,Doe, Jane,jane@x.com,40,Austin\",\"quote\":\"rows that don't have exactly 5 fields; write plain comma-separated values\",\"dimension\":\"quoted embedded comma vs raw split\"}]}","cutOff":false} +{"id":"perf-pairs#0","secs":23,"ok":true,"text":"{\"probes\":[]}","cutOff":false} +{"id":"git-revert-one#0","secs":254,"ok":true,"text":"{\"probes\":[{\"input\":\"Two commits on master both have subject exactly Add experimental cache. The older one only adds the line cache=1 to config.ini and that line is still at HEAD. The newer one only creates cache.py containing enabled = True\\\\n and that file is unchanged since.\",\"readingA\":\"The quoted name selects only the newest matching commit, and that one commit is undone.\",\"outputA\":\"cache.py is gone. config.ini still contains the line cache=1.\",\"readingB\":\"The quoted name selects only the oldest matching commit, and that one commit is undone.\",\"outputB\":\"cache.py still contains enabled = True\\\\n. config.ini no longer contains the line cache=1.\",\"quote\":\"the commit 'Add experimental cache'\",\"dimension\":\"which duplicate subject\"},{\"input\":\"No subject equals Add experimental cache. The only related commit has subject Add experimental cache warmup and only adds the line warmup=1 to config.ini; that line is still at HEAD.\",\"readingA\":\"The subject must equal the quoted text as a whole token, so that commit is left in place.\",\"outputA\":\"config.ini still contains the line warmup=1.\",\"readingB\":\"The quoted text may match as a substring of the subject, so that commit is undone.\",\"outputB\":\"config.ini does not contain the line warmup=1.\",\"quote\":\"the commit 'Add experimental cache'\",\"dimension\":\"whole subject vs substring\"},{\"input\":\"The only commit containing those words has subject Fix startup and a body line Add experimental cache. It changed log.py from level = 1\\\\n to level = 2\\\\n, and HEAD log.py is still level = 2\\\\n.\",\"readingA\":\"The commit is identified by a subject line equal to the quoted text, so this commit is left in place.\",\"outputA\":\"log.py is level = 2\\\\n.\",\"readingB\":\"A body line equal to the quoted text identifies the commit, so that commit is undone.\",\"outputB\":\"log.py is level = 1\\\\n.\",\"quote\":\"the commit 'Add experimental cache'\",\"dimension\":\"subject line vs commit body\"},{\"input\":\"HEAD config.ini is exactly mode=a\\\\ncache=1\\\\ndebug=2\\\\n. The parent of Add experimental cache was mode=a\\\\n. That commit added the lines cache=1 and debug=1. A later commit changed debug=1 to debug=2.\",\"readingA\":\"Undo exactly the lines that commit introduced, including the line a later commit edited, so both added lines are removed.\",\"outputA\":\"config.ini is exactly mode=a\\\\n.\",\"readingB\":\"Keep the later commit's edit, so debug=2 stays, and only the still-intact line cache=1 is removed.\",\"outputB\":\"config.ini is exactly mode=a\\\\ndebug=2\\\\n.\",\"quote\":\"Undo exactly that commit's changes ... keeping every change made by the other commits\",\"dimension\":\"later edit of the same line\"},{\"input\":\"Add experimental cache created cache.py as def init():\\\\n return 1\\\\n. A later commit changed that file to def init():\\\\n return 2\\\\n. HEAD cache.py is the later text.\",\"readingA\":\"Undoing the commit that added the file deletes cache.py.\",\"outputA\":\"cache.py is absent.\",\"readingB\":\"The later commit's change to the file is kept, so cache.py stays at the later text.\",\"outputB\":\"cache.py is def init():\\\\n return 2\\\\n.\",\"quote\":\"Undo exactly that commit's changes ... keeping every change made by the other commits\",\"dimension\":\"file added then modified\"},{\"input\":\"Add experimental cache created cache.py containing x = 1\\\\n. A later commit renamed cache.py to store.py with the same contents. HEAD has store.py containing x = 1\\\\n and no cache.py.\",\"readingA\":\"Undoing the add follows the rename and deletes the file that commit introduced.\",\"outputA\":\"store.py is absent.\",\"readingB\":\"The other commit's rename is kept, so the file remains at the new path.\",\"outputB\":\"store.py contains x = 1\\\\n.\",\"quote\":\"Undo exactly that commit's changes ... keeping every change made by the other commits\",\"dimension\":\"rename after the added file\"}]}","cutOff":false} +{"id":"size-parse-bug#0","secs":17,"ok":true,"text":"{\"probes\":[]}","cutOff":false} +{"id":"bash-rotate#0","secs":589,"ok":true,"text":"{\"probes\":[{\"input\":\"./rotate.sh -n 2 DIR with regular files a.log, b.log, and c.log, each mtime 1000\",\"readingA\":\"Keep a set of exactly 2 files, breaking equal mtimes by filename ascending, so a.log and b.log stay and c.log is deleted.\",\"outputA\":\"c.log\\\\nexit 0\",\"readingB\":\"Keep every file at least as new as the 2nd newest mtime, so all three at mtime 1000 stay.\",\"outputB\":\"empty stdout, exit 0, nothing deleted\",\"quote\":\"keeps the N newest (by modification time)\",\"dimension\":\"ties at the N cutoff\"},{\"input\":\"./rotate.sh -n 5 DIR with a.log and b.log both mtime 1000, plus c.log mtime 2000, d.log 3000, e.log 4000, f.log 5000, g.log 6000\",\"readingA\":\"The two deleted files share an mtime, so oldest-first lists them by filename ascending.\",\"outputA\":\"a.log\\\\nb.log\\\\nexit 0\",\"readingB\":\"Oldest-first via a fully reversed time sort (as with ls -tr) lists equal mtimes by filename descending.\",\"outputB\":\"b.log\\\\na.log\\\\nexit 0\",\"quote\":\"prints the name (basename only) of each deleted file on its own line, oldest first\",\"dimension\":\"equal-mtime print order\"},{\"input\":\"./rotate.sh -n 2 DIR with a.log mtime 1000.1, b.log mtime 1000.9, c.log mtime 2000\",\"readingA\":\"Compare full modification timestamps; c.log and b.log are the two newest, so only a.log is deleted.\",\"outputA\":\"a.log\\\\nexit 0\",\"readingB\":\"Compare whole-second mtimes and break the 1000-second tie by filename ascending, keeping c.log and a.log.\",\"outputB\":\"b.log\\\\nexit 0\",\"quote\":\"newest (by modification time)\",\"dimension\":\"mtime subsecond rounding\"},{\"input\":\"./rotate.sh -n 1 DIR with regular files .app.log mtime 1 and new.log mtime 2\",\"readingA\":\"The pattern *.log is a shell glob, so the dotfile .app.log is outside the set and only new.log is seen.\",\"outputA\":\"empty stdout, exit 0, .app.log remains\",\"readingB\":\"A name ending in .log matches, so .app.log is the older of two candidates and is deleted.\",\"outputB\":\".app.log\\\\nexit 0\",\"quote\":\"regular files named *.log\",\"dimension\":\"dotfile versus glob\"},{\"input\":\"./rotate.sh DIR (default N=5) with regular a.log..e.log at mtimes 10,20,30,40,50 and link.log, a symlink to a regular file, whose own mtime and target mtime are both 1\",\"readingA\":\"Only lstat-regular files count, so the five *.log files are kept and the symlink is left in place.\",\"outputA\":\"empty stdout, exit 0, link.log remains\",\"readingB\":\"A symlink to a regular file counts as a regular *.log file, and as the oldest of six it is deleted.\",\"outputB\":\"link.log\\\\nexit 0\",\"quote\":\"regular files named *.log\",\"dimension\":\"symlink counted as regular\"},{\"input\":\"./rotate.sh DIRLINK where DIRLINK is a symlink to a directory that holds regular a.log (oldest) through f.log (newest), six files, default N=5\",\"readingA\":\"A path that resolves to a directory is accepted, so the oldest log inside the target, a.log, is deleted.\",\"outputA\":\"a.log\\\\nexit 0\",\"readingB\":\"A symlink is not itself a directory, so this is the failure path.\",\"outputB\":\"usage on stderr, empty stdout, exit 2, nothing deleted\",\"quote\":\"If DIR is missing or not a directory\",\"dimension\":\"symlink as DIR\"},{\"input\":\"./rotate.sh -n +5 DIR with regular a.log (mtime 1) through f.log (mtime 6)\",\"readingA\":\"+5 is the positive integer 5, so the oldest file a.log is deleted.\",\"outputA\":\"a.log\\\\nexit 0\",\"readingB\":\"+5 is not a digit-only integer token, so N is rejected.\",\"outputB\":\"usage on stderr, empty stdout, exit 2, nothing deleted\",\"quote\":\"N is not a positive integer\",\"dimension\":\"plus-sign integer form\"},{\"input\":\"./rotate.sh -n 1 -n 5 DIR with regular a.log (mtime 1) through f.log (mtime 6)\",\"readingA\":\"The synopsis allows a single -n, so a repeated -n is a usage failure and deletes nothing.\",\"outputA\":\"usage on stderr, empty stdout, exit 2, all six files remain\",\"readingB\":\"The last -n wins, so N=5 and only a.log is deleted.\",\"outputB\":\"a.log\\\\nexit 0\",\"quote\":\"used as ./rotate.sh [-n N] DIR\",\"dimension\":\"repeated -n flag\"}]}","cutOff":false} +{"id":"config-migrate#0","secs":562,"ok":true,"text":"{\"probes\":[{\"input\":\"\\\"debug\\\": 1\",\"readingA\":\"Debug is true only for JSON boolean true or a string equal to yes, true, or 1. The JSON number 1 matches neither.\",\"outputA\":\"\\\"debug\\\": false\",\"readingB\":\"The trailing \\\"or 1\\\" is its own accepted value, so numeric 1 makes debug true.\",\"outputB\":\"\\\"debug\\\": true\",\"quote\":\"true only if v1 debug is JSON true or a string equal to yes, true or 1 ignoring case\",\"dimension\":\"debug 1 type and OR-binding\"},{\"input\":\"\\\"debug\\\": \\\" yes \\\"\",\"readingA\":\"Equality is case-insensitive and exact, so surrounding spaces do not match yes.\",\"outputA\":\"\\\"debug\\\": false\",\"readingB\":\"The string is trimmed before the case-insensitive comparison, so it matches yes.\",\"outputB\":\"\\\"debug\\\": true\",\"quote\":\"a string equal to yes, true or 1 ignoring case\",\"dimension\":\"debug match versus trim\"},{\"input\":\"\\\"db_port\\\": 5432.6\",\"readingA\":\"\\\"as an integer\\\" truncates toward zero.\",\"outputA\":\"\\\"port\\\": 5432\",\"readingB\":\"\\\"as an integer\\\" rounds to the nearest integer.\",\"outputB\":\"\\\"port\\\": 5433\",\"quote\":\"db_port as an integer, 5432 if missing\",\"dimension\":\"non-integer port rounding\"},{\"input\":\"{\\\"version\\\":1,\\\"db_port\\\":5432,\\\"debug\\\":false,\\\"features\\\":\\\"\\\",\\\"users\\\":[]}\",\"readingA\":\"Only port has a missing default, so a missing host is JSON null.\",\"outputA\":\"\\\"database\\\": {\\\"host\\\": null, \\\"port\\\": 5432}\",\"readingB\":\"A missing host is written as an empty string.\",\"outputB\":\"\\\"database\\\": {\\\"host\\\": \\\"\\\", \\\"port\\\": 5432}\",\"quote\":\"\\\"host\\\": db_host, \\\"port\\\": db_port as an integer, 5432 if missing\",\"dimension\":\"missing db_host value\"},{\"input\":\"{\\\"version\\\":\\\"1\\\",\\\"db_host\\\":\\\"h\\\",\\\"db_port\\\":5432,\\\"debug\\\":false,\\\"features\\\":\\\"a\\\",\\\"users\\\":[]}\",\"readingA\":\"Version 1 means the JSON number 1, so a string version is left untouched.\",\"outputA\":\"unchanged file, still \\\"version\\\": \\\"1\\\"\",\"readingB\":\"The string \\\"1\\\" counts as version 1, so the file is rewritten to the v2 schema.\",\"outputB\":\"rewritten file with \\\"version\\\": 2 and \\\"features\\\": [\\\"a\\\"]\",\"quote\":\"every configs/*.json file whose \\\"version\\\" is 1\",\"dimension\":\"what counts as version 1\"},{\"input\":\"\\\"users\\\": [{\\\"name\\\":\\\"amy\\\",\\\"admin\\\":\\\"1\\\"},{\\\"name\\\":\\\"bob\\\",\\\"admin\\\":true}]\",\"readingA\":\"Admin status is only JSON number 1 or JSON boolean true, so amy is not an admin.\",\"outputA\":\"\\\"users\\\": [\\\"amy\\\", \\\"bob\\\"], \\\"admins\\\": [\\\"bob\\\"]\",\"readingB\":\"The admin value 1 includes the string \\\"1\\\", so amy is an admin.\",\"outputB\":\"\\\"users\\\": [\\\"amy\\\", \\\"bob\\\"], \\\"admins\\\": [\\\"amy\\\", \\\"bob\\\"]\",\"quote\":\"whose admin is 1 or true, in order\",\"dimension\":\"admin flag JSON type\"},{\"input\":\"\\\"features\\\": \\\"red, blue, red\\\"\",\"readingA\":\"Repeats are removed by keeping the first occurrence in original order.\",\"outputA\":\"[\\\"red\\\", \\\"blue\\\"]\",\"readingB\":\"Repeats are removed by keeping the last occurrence, ordered by last appearance.\",\"outputB\":\"[\\\"blue\\\", \\\"red\\\"]\",\"quote\":\"trimmed, non-empty names in original order with repeats removed\",\"dimension\":\"which duplicate feature is kept\"}]}","cutOff":false} +{"id":"makefile-site#0","secs":423,"ok":true,"text":"{\"probes\":[{\"input\":\"src/guide/intro.md exists; run make\",\"readingA\":\"NAME is one path segment, so only src/*.md is a source; a file under a subdirectory is not src/NAME.md\",\"outputA\":\"build/guide/intro.html is not created\",\"readingB\":\"NAME may be the relative path guide/intro, so every .md under src/ is built to the matching build path\",\"outputB\":\"build/guide/intro.html is created by python3 md2html.py src/guide/intro.md build/guide/intro.html\",\"quote\":\"builds build/NAME.html from every src/NAME.md\",\"dimension\":\"flat vs recursive sources\"},{\"input\":\"src/.draft.md exists; run make\",\"readingA\":\"new .md files are picked up with a normal src/*.md glob, which does not match names that start with a dot\",\"outputA\":\"build/.draft.html is not created\",\"readingB\":\"every src/NAME.md includes NAME=.draft\",\"outputB\":\"build/.draft.html is created\",\"quote\":\"every src/NAME.md, picking up new .md files\",\"dimension\":\"whether dotfiles count\"},{\"input\":\"make build/contact.html when build/ does not exist and src/about.md, src/contact.md, and src/index.md do\",\"readingA\":\"each build/NAME.html is its own target, so the requested page is the only goal\",\"outputA\":\"only build/contact.html is created; build/about.html and build/index.html are absent\",\"readingB\":\"only the default make goal runs one recipe that builds every page; build/NAME.html paths are not targets\",\"outputB\":\"make stops with no rule to make target build/contact.html and creates no html files\",\"quote\":\"make builds build/NAME.html from every src/NAME.md\",\"dimension\":\"per-file targets vs one recipe\"},{\"input\":\"a regular file named clean exists and build/index.html exists; run make clean\",\"readingA\":\"clean is always the removal action, so it runs even if a file named clean is present\",\"outputA\":\"build/ is removed\",\"readingB\":\"clean is an ordinary target with no prerequisites, so an existing clean file is already up to date\",\"outputB\":\"build/index.html remains\",\"quote\":\"make clean removes build/\",\"dimension\":\"phony clean vs a file named clean\"}]}","cutOff":false} +{"id":"redact-secrets#0","secs":573,"ok":true,"text":"{\"probes\":[{\"input\":\"-----BEGIN PRIVATE KEY-----\\nABC\\n-----END PRIVATE KEY-----\\n\",\"readingA\":\"The ellipsis may be empty, so a PKCS#8 block with no type word is a private-key block and is replaced through the END marker's last dash.\",\"outputA\":\"[REDACTED]\\n\",\"readingB\":\"The ellipsis is a required non-empty type label between BEGIN/END and PRIVATE, so this block is not a secret.\",\"outputB\":\"-----BEGIN PRIVATE KEY-----\\nABC\\n-----END PRIVATE KEY-----\\n\",\"quote\":\"-----BEGIN ... PRIVATE KEY----- through the matching -----END ... PRIVATE KEY-----\",\"dimension\":\"empty PEM type label\"},{\"input\":\"-----BEGIN RSA PRIVATE KEY-----\\nABC\\n-----END EC PRIVATE KEY-----\\n\",\"readingA\":\"matching means the next END ... PRIVATE KEY marker; the two ellipses are independent, so RSA may close on EC.\",\"outputA\":\"[REDACTED]\\n\",\"readingB\":\"matching means the END type text must be the same text as the BEGIN type text, so RSA does not close on EC.\",\"outputB\":\"-----BEGIN RSA PRIVATE KEY-----\\nABC\\n-----END EC PRIVATE KEY-----\\n\",\"quote\":\"the matching -----END ... PRIVATE KEY-----\",\"dimension\":\"BEGIN/END type labels must be equal\"},{\"input\":\"-----BEGIN RSA PRIVATE KEY-----\\nA\\n-----END RSA PRIVATE KEY-----\\n-----BEGIN RSA PRIVATE KEY-----\\nB\\n-----END RSA PRIVATE KEY-----\\n\",\"readingA\":\"Each BEGIN pairs with the nearest following END, so there are two whole-block replacements.\",\"outputA\":\"[REDACTED]\\n[REDACTED]\\n (N=2)\",\"readingB\":\"The first BEGIN runs through the last END that fits the pattern, so both blocks are one replacement.\",\"outputB\":\"[REDACTED]\\n (N=1)\",\"quote\":\"through the matching -----END ... PRIVATE KEY----- line's last dash, replaced as a whole\",\"dimension\":\"nearest END vs greedy span\"},{\"input\":\"-----BEGIN RSA PRIVATE KEY-----\\nABC\\n-----END RSA PRIVATE KEY------\\n\",\"readingA\":\"The closing marker is the five dashes written in the pattern; the sixth dash is outside the span and remains.\",\"outputA\":\"[REDACTED]-\\n\",\"readingB\":\"The span ends at that END line's final dash character, so all six trailing dashes are inside the replacement.\",\"outputB\":\"[REDACTED]\\n\",\"quote\":\"-----END ... PRIVATE KEY----- line's last dash\",\"dimension\":\"five-dash marker vs line's last dash\"},{\"input\":\"AKIAIOSFODNN7EXAMPLEé\",\"readingA\":\"letter or digit means ASCII alphanumerics; é is outside that set, so the 20-character id is replaced and é stays.\",\"outputA\":\"[REDACTED]é\",\"readingB\":\"letter means any Unicode letter; é immediately follows the id, so the id is not a secret.\",\"outputB\":\"AKIAIOSFODNN7EXAMPLEé\",\"quote\":\"neither may be immediately preceded or followed by another letter or digit\",\"dimension\":\"ASCII vs Unicode boundary\"},{\"input\":\" -----BEGIN RSA PRIVATE KEY-----\\nABC\\n-----END RSA PRIVATE KEY-----\\n\",\"readingA\":\"The replaced span starts at the first dash of -----BEGIN, so the leading space stays.\",\"outputA\":\" [REDACTED]\\n\",\"readingB\":\"The block is the full BEGIN line through the END line's last dash, so the leading space is inside the one replacement.\",\"outputB\":\"[REDACTED]\\n\",\"quote\":\"from -----BEGIN ... PRIVATE KEY----- through the matching -----END ... PRIVATE KEY----- line's last dash\",\"dimension\":\"span starts at dashes vs start of line\"},{\"input\":\"-----BEGIN RSA PRIVATE KEY-----\\nAKIAIOSFODNN7EXAMPLE\\n-----END RSA PRIVATE KEY-----\\n\",\"readingA\":\"A PEM block is one secret replaced as a whole; the inner AKIA is not an additional replaced secret.\",\"outputA\":\"[REDACTED]\\n and redactions.txt N=1\",\"readingB\":\"The AKIA rule and the PEM rule each match in the original bytes, so both count as secrets replaced.\",\"outputB\":\"[REDACTED]\\n and redactions.txt N=2\",\"quote\":\"replacing each one with [REDACTED] ... replaced as a whole ... N = secrets replaced\",\"dimension\":\"token inside a PEM block counts toward N\"}]}","cutOff":false} +{"id":"organize-files#0","secs":894,"ok":true,"text":"{\"probes\":[{\"input\":\"inbox/Photo.JPG\",\"readingA\":\"Category folders hold only lowercased basenames. Path order puts 2023/photo.jpg first, so it keeps photo.jpg and Photo.JPG gets a numeric suffix.\",\"outputA\":\"sorted/images/photo-1.jpg\",\"readingB\":\"Each file keeps its inbox-relative directory under the category folder. Photo.JPG lands in sorted/images/photo.jpg, and 2023/photo.jpg lands in sorted/images/2023/photo.jpg, so the names do not collide.\",\"outputB\":\"sorted/images/photo.jpg\",\"quote\":\"into sorted/images/ ... (including subfolders) ... same name in the same folder ... (photo-1.jpg)\",\"dimension\":\"flatten basenames vs keep subpaths\"},{\"input\":\"inbox/notes.txt\",\"readingA\":\"The whole basename is lowercased, so this file, 2023/notes.TXT, and misc/Notes.TXT all become notes.txt. Plain path order makes this the third, so it gets -2.\",\"outputA\":\"sorted/docs/notes-2.txt\",\"readingB\":\"Only the stem is lowercased and the extension text is kept, so the .TXT files become notes.TXT and notes-1.TXT while this file stays notes.txt.\",\"outputB\":\"sorted/docs/notes.txt\",\"quote\":\"rename each file to its name in lowercase ... inserted before the extension\",\"dimension\":\"lowercase whole filename vs stem only\"}]}","cutOff":false} diff --git a/bench/local/ambiguity-results/run1-tools-narration-INVALID.jsonl b/bench/local/ambiguity-results/run1-tools-narration-INVALID.jsonl new file mode 100644 index 0000000..b55662b --- /dev/null +++ b/bench/local/ambiguity-results/run1-tools-narration-INVALID.jsonl @@ -0,0 +1,20 @@ +{"id":"fix-git#0","secs":20,"ok":true,"text":"The task is ambiguous until I see the repo state and the exact wording that two readings could apply differently. I'll inspect the files and git history first.MCP tools aren't available, so I'll inspect the workspace and git state directly to see which readings of the task actually diverge.","cutOff":false} +{"id":"log-summary#0","secs":24,"ok":true,"text":"The task text can be read in more than one way. I'll inspect the log files for concrete lines where those readings would count differently.Molt isn't available, so I'll read the logs directly and look for lines that two readings would count differently.","cutOff":false} +{"id":"duration-bug#0","secs":15,"ok":true,"text":"I'll inspect the task files and look for readings that would change the same input's result.Molt tools aren't available here, so I'll read the task files directly.","cutOff":false} +{"id":"http-json-server#0","secs":9,"ok":true,"text":"I'll inspect the task files and the spec wording so I can list only the inputs where two fair readings would return different results.","cutOff":false} +{"id":"sql-top-customers#0","secs":16,"ok":true,"text":"I'll inspect the database schema and sample rows so any ambiguity probes are grounded in data that could actually change the CSV.MCP tools aren't available, so I'll inspect `shop.db` directly to see which readings would actually change the CSV.","cutOff":false} +{"id":"regex#0","secs":148,"ok":true,"text":"I'll inspect the task files and the wording so I can pinpoint inputs where two fair readings of the regex spec would disagree.The sample log is part of the spec. I'll locate it and check which edge cases it actually contains.","cutOff":false} +{"id":"refactor-pricing#0","secs":13,"ok":true,"text":"I'll inspect the pricing module and the task wording for places where two readings would change the result on the same input.Molt tools aren't connected, so I'll read `pricing.py` directly to compare the coupon rules with the task wording.","cutOff":false} +{"id":"wc-tool#0","secs":420,"ok":false,"error":"TimeoutError: Grok Build did not answer within 420s"} +{"id":"csv-clean#0","secs":420,"ok":false,"error":"TimeoutError: Grok Build did not answer within 420s"} +{"id":"perf-pairs#0","secs":39,"ok":true,"text":"The task says the new `find_pairs` must match today's results exactly. I'll read `pairs.py` first so I can pin down where two faithful readings would still disagree on the same input.MCP tools are not connected in this session, so I’ll read `pairs.py` directly and use that as the oracle for where the task can still be read two ways.","cutOff":false} +{"id":"git-revert-one#0","secs":16,"ok":true,"text":"The task is a history-preserving undo of one commit. I'll inspect the repo just enough to see where two readings of that instruction would leave different file contents.Molt isn't available here, so I'll inspect the repo with the local tools and look only for readings that change the resulting tree.","cutOff":false} +{"id":"bash-rotate#0","secs":420,"ok":false,"error":"TimeoutError: Grok Build did not answer within 420s"} +{"id":"node-summarize#0","secs":152,"ok":false,"error":"Grok Build returned nothing"} +{"id":"size-parse-bug#0","secs":14,"ok":false,"error":"Grok Build returned nothing"} +{"id":"config-migrate#0","secs":420,"ok":false,"error":"TimeoutError: Grok Build did not answer within 420s"} +{"id":"access-report#0","secs":420,"ok":false,"error":"TimeoutError: Grok Build did not answer within 420s"} +{"id":"organize-files#0","secs":420,"ok":false,"error":"TimeoutError: Grok Build did not answer within 420s"} +{"id":"makefile-site#0","secs":420,"ok":false,"error":"TimeoutError: Grok Build did not answer within 420s"} +{"id":"cron-next#0","secs":420,"ok":false,"error":"TimeoutError: Grok Build did not answer within 420s"} +{"id":"redact-secrets#0","secs":420,"ok":false,"error":"TimeoutError: Grok Build did not answer within 420s"} diff --git a/bench/local/ambiguity-results/run2-tool-stalls-partial.jsonl b/bench/local/ambiguity-results/run2-tool-stalls-partial.jsonl new file mode 100644 index 0000000..47f9100 --- /dev/null +++ b/bench/local/ambiguity-results/run2-tool-stalls-partial.jsonl @@ -0,0 +1,20 @@ +{"id":"fix-git#0","secs":179,"ok":true,"text":"{\"probes\":[{\"input\":\"reflog: checkout from branch add-bio to master. add-bio about.md is \\\"Bio from add-bio\\\\n\\\" and index.md is \\\"Home\\\\n\\\". Branch rewrite-about has a newer commit with about.md \\\"Bio from rewrite-about\\\\n\\\". master about.md is \\\"Old bio\\\\n\\\" and index.md is \\\"Home\\\\n\\\".\",\"readingA\":\"The changes lost by checking out master are the previous branch, so merge add-bio.\",\"outputA\":\"about.md is \\\"Bio from add-bio\\\\n\\\"; index.md stays \\\"Home\\\\n\\\"\",\"readingB\":\"Search every unmerged about-page edit and take the newest commit, rewrite-about.\",\"outputB\":\"about.md is \\\"Bio from rewrite-about\\\\n\\\"\",\"quote\":\"then checked out master, and now I can't find those changes. Find them\",\"dimension\":\"previous branch vs newest about edit\"},{\"input\":\"Only copy of the work is branch feature, whose one commit sets about.md to \\\"New about\\\\n\\\" and index.md to \\\"New home\\\\n\\\". master has about.md \\\"Old about\\\\n\\\" and index.md \\\"Old home\\\\n\\\".\",\"readingA\":\"Merge the branch that holds the work, so every change on that branch lands on master.\",\"outputA\":\"about.md is \\\"New about\\\\n\\\" and index.md is \\\"New home\\\\n\\\"\",\"readingB\":\"Bring only the about-page edits onto master.\",\"outputB\":\"about.md is \\\"New about\\\\n\\\" and index.md stays \\\"Old home\\\\n\\\"\",\"quote\":\"changes to my site's about page\",\"dimension\":\"whole branch vs about.md only\"},{\"input\":\"stash@{0} sets about.md to \\\"Stashed about\\\\n\\\". The branch left at checkout, feature, sets about.md to \\\"Branch about\\\\n\\\". master about.md is \\\"Master about\\\\n\\\".\",\"readingA\":\"Merge into master means git merge of the branch that was current before checkout.\",\"outputA\":\"about.md is \\\"Branch about\\\\n\\\"\",\"readingB\":\"The checkout hid uncommitted work in the stash, so apply that stash onto master.\",\"outputB\":\"about.md is \\\"Stashed about\\\\n\\\"\",\"quote\":\"checked out master, and now I can't find those changes\",\"dimension\":\"stash vs previous branch\"},{\"input\":\"Merge-base about.md is \\\"# About\\\\nAlpha\\\\nOmega\\\\n\\\". master about.md is \\\"# About\\\\nAlpha master\\\\nOmega\\\\n\\\". Recovered branch about.md is \\\"# About\\\\nAlpha\\\\nOmega feature\\\\n\\\". index.md is identical on both.\",\"readingA\":\"git merge does a 3-way merge and keeps both sides' edits to the about page.\",\"outputA\":\"about.md is \\\"# About\\\\nAlpha master\\\\nOmega feature\\\\n\\\"\",\"readingB\":\"Put the recovered about page on master by taking that file's contents as found.\",\"outputB\":\"about.md is \\\"# About\\\\nAlpha\\\\nOmega feature\\\\n\\\"\",\"quote\":\"merge them into master\",\"dimension\":\"3-way merge vs take found file\"}]}","cutOff":false} +{"id":"log-summary#0","secs":240,"ok":false,"error":"TimeoutError: Grok Build did not answer within 240s"} +{"id":"duration-bug#0","secs":36,"ok":true,"text":"{\"probes\":[]}","cutOff":false} +{"id":"regex#0","secs":240,"ok":false,"error":"TimeoutError: Grok Build did not answer within 240s"} +{"id":"wc-tool#0","secs":240,"ok":false,"error":"TimeoutError: Grok Build did not answer within 240s"} +{"id":"http-json-server#0","secs":243,"ok":false,"error":"TimeoutError: Grok Build did not answer within 240s"} +{"id":"sql-top-customers#0","secs":246,"ok":false,"error":"TimeoutError: Grok Build did not answer within 240s"} +{"id":"refactor-pricing#0","secs":123,"ok":true,"text":"The task text leaves a few output-level choices open. I'll check the current pricing rules so each probe is a real input where those readings diverge.","cutOff":false} +{"id":"csv-clean#0","secs":240,"ok":false,"error":"TimeoutError: Grok Build did not answer within 240s"} +{"id":"bash-rotate#0","secs":240,"ok":false,"error":"TimeoutError: Grok Build did not answer within 240s"} +{"id":"config-migrate#0","secs":240,"ok":false,"error":"TimeoutError: Grok Build did not answer within 240s"} +{"id":"perf-pairs#0","secs":85,"ok":true,"text":"{\"probes\":[]}","cutOff":false} +{"id":"git-revert-one#0","secs":240,"ok":false,"error":"TimeoutError: Grok Build did not answer within 240s"} +{"id":"access-report#0","secs":240,"ok":false,"error":"TimeoutError: Grok Build did not answer within 240s"} +{"id":"size-parse-bug#0","secs":4,"ok":true,"text":"I'll read `sizes.py` and `test_sizes.py` to find docstring rules that two correct implementations could interpret differently.","cutOff":false} +{"id":"node-summarize#0","secs":240,"ok":false,"error":"TimeoutError: Grok Build did not answer within 240s"} +{"id":"organize-files#0","secs":240,"ok":false,"error":"TimeoutError: Grok Build did not answer within 240s"} +{"id":"makefile-site#0","secs":240,"ok":false,"error":"TimeoutError: Grok Build did not answer within 240s"} +{"id":"cron-next#0","secs":240,"ok":false,"error":"TimeoutError: Grok Build did not answer within 240s"} +{"id":"redact-secrets#0","secs":240,"ok":false,"error":"TimeoutError: Grok Build did not answer within 240s"} diff --git a/bench/local/ambiguity-results/traps3.jsonl b/bench/local/ambiguity-results/traps3.jsonl new file mode 100644 index 0000000..5129c8f --- /dev/null +++ b/bench/local/ambiguity-results/traps3.jsonl @@ -0,0 +1,12 @@ +{"id":"regex#1","secs":380,"ok":true,"text":"{\"probes\":[{\"input\":\"256.1.2.3 2024-01-15\",\"readingA\":\"Contain means a valid four-octet sequence may be a substring, so 56.1.2.3 is an IPv4 address and the date is later on the line.\",\"outputA\":\"re.search matches\",\"readingB\":\"The four numbers are one dotted token; that token starts at 256, which is outside 0-255.\",\"outputB\":\"re.search does not match\",\"quote\":\"contain a valid IPv4 address (four numbers 0-255, no leading zeros)\",\"dimension\":\"IP substring vs whole token\"},{\"input\":\"1.2.3.4.5 2024-06-01\",\"readingA\":\"1.2.3.4 is four numbers 0-255 and is followed later on the line by the date; the extra .5 does not continue the last octet.\",\"outputA\":\"re.search matches\",\"readingB\":\"A valid IPv4 address is exactly four dotted numbers, so the token 1.2.3.4.5 is not an IPv4 address.\",\"outputB\":\"re.search does not match\",\"quote\":\"valid IPv4 address (four numbers 0-255\",\"dimension\":\"extra dotted field after IP\"},{\"input\":\"10.0.0.12024-01-01\",\"readingA\":\"Followed later allows the date to start immediately, with the last octet taken as 1 so 10.0.0.1 is followed by 2024-01-01.\",\"outputA\":\"re.search matches\",\"readingB\":\"Each of the four numbers is a maximal digit run, so the last number is 12024, which is outside 0-255.\",\"outputB\":\"re.search does not match\",\"quote\":\"four numbers 0-255, no leading zeros) followed later on the same line\",\"dimension\":\"IP/date digit boundary\"},{\"input\":\"8.8.8.8 2024-13-01\",\"readingA\":\"YYYY-MM-DD form is four digits, a hyphen, two digits, a hyphen, and two digits, so month 13 matches.\",\"outputA\":\"re.search matches\",\"readingB\":\"A date in that form has a month from 01 through 12, so 13 fails.\",\"outputB\":\"re.search does not match\",\"quote\":\"a date in YYYY-MM-DD form\",\"dimension\":\"date shape vs month range\"},{\"input\":\"8.8.8.8 2024-04-31\",\"readingA\":\"The form only needs month 01-12 and day 01-31, so April 31 matches.\",\"outputA\":\"re.search matches\",\"readingB\":\"A date must be a real calendar day, and April has 30 days.\",\"outputB\":\"re.search does not match\",\"quote\":\"a date in YYYY-MM-DD form\",\"dimension\":\"day 01-31 vs calendar day\"},{\"input\":\"8.8.8.8 2024-01-011\",\"readingA\":\"The character sequence 2024-01-01 is a YYYY-MM-DD form on the line, so the extra trailing digit is ignored.\",\"outputA\":\"re.search matches\",\"readingB\":\"DD is exactly two digits at the end of the date token, so day 011 is not YYYY-MM-DD.\",\"outputB\":\"re.search does not match\",\"quote\":\"a date in YYYY-MM-DD form\",\"dimension\":\"date substring vs bounded day\"},{\"input\":\"8.8.8.8 2024-1-9\",\"readingA\":\"YYYY-MM-DD fixes month and day at two digits, so a one-digit month and day fail.\",\"outputA\":\"re.search does not match\",\"readingB\":\"The form only orders year, month, and day with hyphens, so one-digit month and day match.\",\"outputB\":\"re.search matches\",\"quote\":\"a date in YYYY-MM-DD form\",\"dimension\":\"zero-padded month and day\"}]}","cutOff":false} +{"id":"regex#2","secs":386,"ok":true,"text":"{\"probes\":[{\"input\":\"01.2.3.4 2024-05-06\",\"readingA\":\"A valid address may be any span of four 0-255 numbers with no leading zeros, so the span 1.2.3.4 counts.\",\"outputA\":\"re.search matches\",\"readingB\":\"The address is the whole dotted token; 01.2.3.4 has a leading zero, so it is not valid.\",\"outputB\":\"re.search does not match\",\"quote\":\"contain a valid IPv4 address (four numbers 0-255, no leading zeros)\",\"dimension\":\"IP substring vs whole token\"},{\"input\":\"1.2.3.4.5 2024-05-06\",\"readingA\":\"Four numbers 1.2.3.4 are present (a word boundary also falls between 4 and the next dot), then a date.\",\"outputA\":\"re.search matches\",\"readingB\":\"A valid IPv4 is exactly four numbers and cannot be a prefix of a longer dotted sequence.\",\"outputB\":\"re.search does not match\",\"quote\":\"valid IPv4 address (four numbers 0-255\",\"dimension\":\"extra trailing octet\"},{\"input\":\"host10.0.0.1 2024-05-06\",\"readingA\":\"The digit-and-dot run 10.0.0.1 is a valid address even when a letter is glued to its left.\",\"outputA\":\"re.search matches\",\"readingB\":\"The address must be a whole word, and there is no word boundary between the letter t and 10.\",\"outputB\":\"re.search does not match\",\"quote\":\"contain a valid IPv4 address\",\"dimension\":\"word boundary vs digit-dot token\"},{\"input\":\"10.0.0.12024-05-06\",\"readingA\":\"The address 10.0.0.1 is immediately followed on the line by the date 2024-05-06.\",\"outputA\":\"re.search matches\",\"readingB\":\"The date must be a separate later field; the glued token 10.0.0.12024 is not an address and 12024-05-06 is not YYYY-MM-DD.\",\"outputB\":\"re.search does not match\",\"quote\":\"followed later on the same line by a date\",\"dimension\":\"adjacent IP and date\"},{\"input\":\"10.0.0.1 2024-05-069\",\"readingA\":\"YYYY-MM-DD form occurs as the span 2024-05-06, with an extra digit after it.\",\"outputA\":\"re.search matches\",\"readingB\":\"The date token is 2024-05-069, which is not YYYY-MM-DD form.\",\"outputB\":\"re.search does not match\",\"quote\":\"a date in YYYY-MM-DD form\",\"dimension\":\"date substring vs whole token\"},{\"input\":\"10.0.0.1 2024-13-01\",\"readingA\":\"The form is only four digits, a hyphen, two digits, a hyphen, and two digits.\",\"outputA\":\"re.search matches\",\"readingB\":\"MM is a month in 01-12, so 13 is not a date in that form.\",\"outputB\":\"re.search does not match\",\"quote\":\"a date in YYYY-MM-DD form\",\"dimension\":\"month range vs digit widths\"},{\"input\":\"10.0.0.1 2024-04-31\",\"readingA\":\"Month 04 is in 01-12 and day 31 is in 01-31, so the form is satisfied.\",\"outputA\":\"re.search matches\",\"readingB\":\"A date must be a real calendar day, and April has 30 days.\",\"outputB\":\"re.search does not match\",\"quote\":\"a date in YYYY-MM-DD form\",\"dimension\":\"calendar day vs 01-31\"},{\"input\":\"10.0.0.1 2023-02-29\",\"readingA\":\"February is allowed 29 days whenever day is in 01-31 (or month lengths ignore leap years).\",\"outputA\":\"re.search matches\",\"readingB\":\"The calendar date must exist, and 2023 is not a leap year.\",\"outputB\":\"re.search does not match\",\"quote\":\"a date in YYYY-MM-DD form\",\"dimension\":\"leap-day validity\"}]}","cutOff":false} +{"id":"regex#0","secs":396,"ok":true,"text":"{\"probes\":[{\"input\":\"1.2.3.256 2020-01-01\",\"readingA\":\"A valid IPv4 may be a substring: 1.2.3.25 is four numbers in 0-255 with no leading zeros, and the date comes later on the line.\",\"outputA\":\"re.search matches\",\"readingB\":\"The dotted token must be one address; the fourth number is 256, which is outside 0-255, so a shorter prefix does not count.\",\"outputB\":\"re.search does not match\",\"quote\":\"contain a valid IPv4 address (four numbers 0-255, no leading zeros)\",\"dimension\":\"IPv4 substring vs whole token at 255\"},{\"input\":\"01.2.3.4 2020-01-01\",\"readingA\":\"Containment allows the inner address 1.2.3.4, whose octets are in 0-255 and have no leading zeros, followed later by the date.\",\"outputA\":\"re.search matches\",\"readingB\":\"No leading zeros applies to the address as written; the first octet is 01, so the line has no valid IPv4.\",\"outputB\":\"re.search does not match\",\"quote\":\"no leading zeros\",\"dimension\":\"leading zero vs inner unpadded IP\"},{\"input\":\"1.2.3.4.5 2020-01-01\",\"readingA\":\"The line contains four dotted numbers 1.2.3.4 (or 2.3.4.5), each 0-255, and a later YYYY-MM-DD date.\",\"outputA\":\"re.search matches\",\"readingB\":\"A valid IPv4 is exactly four numbers; a five-component dotted sequence is not an IPv4 and does not contain one.\",\"outputB\":\"re.search does not match\",\"quote\":\"valid IPv4 address (four numbers 0-255\",\"dimension\":\"exactly four octets vs longer dotted sequence\"},{\"input\":\"host1.2.3.4x 2020-01-01\",\"readingA\":\"Contain means the character sequence 1.2.3.4 counts even when letters are glued to it, and the date is later on the line.\",\"outputA\":\"re.search matches\",\"readingB\":\"The address must be its own token; host1.2.3.4x is not four numbers 0-255, so the line does not match.\",\"outputB\":\"re.search does not match\",\"quote\":\"contain a valid IPv4 address\",\"dimension\":\"IP embedded in letters vs standalone token\"},{\"input\":\"1.2.3.4 2020-13-01\",\"readingA\":\"YYYY-MM-DD form means digit widths 4-2-2 only, so month 13 still matches after the IP.\",\"outputA\":\"re.search matches\",\"readingB\":\"A date requires month 01-12, so 2020-13-01 is not a date and the line does not match.\",\"outputB\":\"re.search does not match\",\"quote\":\"a date in YYYY-MM-DD form\",\"dimension\":\"month shape vs month 01-12\"},{\"input\":\"1.2.3.4 2020-04-31\",\"readingA\":\"YYYY-MM-DD with month 01-12 and day 01-31 accepts day 31 in every month, so the line matches.\",\"outputA\":\"re.search matches\",\"readingB\":\"It must be a real calendar date; April has 30 days, so 2020-04-31 does not match.\",\"outputB\":\"re.search does not match\",\"quote\":\"a date in YYYY-MM-DD form\",\"dimension\":\"day 01-31 vs real calendar date\"},{\"input\":\"1.2.3.4 12020-01-01\",\"readingA\":\"The line contains the form 2020-01-01 after a valid IP, so a longer digit run still matches.\",\"outputA\":\"re.search matches\",\"readingB\":\"YYYY is exactly four digits; 12020-01-01 is not that form, so there is no date after the IP.\",\"outputB\":\"re.search does not match\",\"quote\":\"a date in YYYY-MM-DD form\",\"dimension\":\"date substring vs exact year width\"},{\"input\":\"1.2.3.4 2020-01-01T12:00:00\",\"readingA\":\"YYYY-MM-DD form is present after the IP; a trailing time does not stop the match.\",\"outputA\":\"re.search matches\",\"readingB\":\"The date must be the whole field; 2020-01-01T12:00:00 is a datetime, not a YYYY-MM-DD date.\",\"outputB\":\"re.search does not match\",\"quote\":\"followed later on the same line by a date in YYYY-MM-DD form\",\"dimension\":\"date form inside datetime vs standalone date\"}]}","cutOff":false} +{"id":"access-report#0","secs":488,"ok":true,"text":"{\"probes\":[{\"input\":\"127.0.0.1 - - [10/Oct/2000:13:55:36 -0700] \\\"GET /foo HTTP/1.1\\\" 200 10\",\"readingA\":\"Split the quoted request on every U+0020, keeping the empty token, so the parts are GET, (empty), /foo, HTTP/1.1. That is not three parts, so the line is skipped.\",\"outputA\":\"no /foo line; 2xx: 0\",\"readingB\":\"Split on whitespace and collapse repeated spaces, so the parts are GET, /foo, HTTP/1.1. Count the path and the status.\",\"outputB\":\"\\\" 1 /foo\\\" and 2xx: 1\",\"quote\":\"not exactly three space-separated parts\",\"dimension\":\"empty tokens from consecutive spaces\"},{\"input\":\"127.0.0.1 - - [10/Oct/2000:13:55:36 -0700] \\\"GET\\t/foo\\tHTTP/1.1\\\" 200 10\",\"readingA\":\"\\\"Space-separated\\\" means the character space only. Tabs are not separators, so the quoted request is one part and the line is skipped.\",\"outputA\":\"no /foo line; 2xx: 0\",\"readingB\":\"Any whitespace separates parts, so GET, /foo, and HTTP/1.1 are three parts. Count /foo and status 200.\",\"outputB\":\"\\\" 1 /foo\\\" and 2xx: 1\",\"quote\":\"exactly three space-separated parts\",\"dimension\":\"space versus any whitespace\"},{\"input\":\"127.0.0.1 - - [10/Oct/2000:13:55:36 -0700] \\\"GET /foo\\\" 404 20\",\"readingA\":\"Skip applies to the whole line, so a two-part quoted request is excluded from path ranking and from status classes.\",\"outputA\":\"no path line; 4xx: 0\",\"readingB\":\"The three-part rule only drops the line from path ranking; STATUS is still a request in a class.\",\"outputB\":\"no path line; 4xx: 1\",\"quote\":\"Skip any line whose quoted request is not exactly three space-separated parts\",\"dimension\":\"skipped lines versus status totals\"},{\"input\":\"127.0.0.1 - - [10/Oct/2000:13:55:36 -0700] \\\"GET /a HTTP/1.1\\\" 200 5 \\\"http://ref.example/\\\" \\\"Moz ua\\\"\",\"readingA\":\"The quoted request is the first quoted field, GET /a HTTP/1.1 (three parts). Fields after BYTES are ignored. Count /a and 200.\",\"outputA\":\"\\\" 1 /a\\\" and 2xx: 1\",\"readingB\":\"The stated record ends at BYTES, so a line with further quoted fields is not used. The span from the first quote through the last quote is also not three parts.\",\"outputB\":\"no /a line; 2xx: 0\",\"quote\":\"\\\"METHOD PATH PROTOCOL\\\" STATUS BYTES\",\"dimension\":\"extra fields and which quotes bound the request\"},{\"input\":\"127.0.0.1 alice bob [10/Oct/2000:13:55:36 -0700] \\\"GET /a HTTP/1.1\\\" 200 5\",\"readingA\":\"Named Common Log Format; the shown \\\"- -\\\" is just absent ident and authuser, so alice and bob are valid. Count /a and 200.\",\"outputA\":\"\\\" 1 /a\\\" and 2xx: 1\",\"readingB\":\"The backtick pattern requires those two fields to be literal hyphens, so this line is not a match and is skipped.\",\"outputB\":\"no /a line; 2xx: 0\",\"quote\":\"Common Log Format: `HOST - - [DATE]`\",\"dimension\":\"literal hyphens versus any ident fields\"},{\"input\":\"127.0.0.1 - - [10/Oct/2000:13:55:36 -0700] \\\"GET /a HTTP/1.1\\\" 0200 5\",\"readingA\":\"Status is the integer 200, which lies in 200-299, so it increments 2xx. The path is still /a.\",\"outputA\":\"\\\" 1 /a\\\" and 2xx: 1\",\"readingB\":\"Each class is a three-digit token (2xx means 2 followed by two digits). The token 0200 matches none of the four classes. The path is still /a.\",\"outputB\":\"\\\" 1 /a\\\" and 2xx: 0\",\"quote\":\"`2xx: N`, `3xx: N`, `4xx: N`, `5xx: N`\",\"dimension\":\"integer class versus exact 3-digit token\"},{\"input\":\"127.0.0.1 - - [10/Oct/2000:13:55:36 -0700] \\\"GET /B HTTP/1.1\\\" 200 1\\n127.0.0.1 - - [10/Oct/2000:13:55:36 -0700] \\\"GET /a HTTP/1.1\\\" 200 1\",\"readingA\":\"Path ascending is raw code-point order, so /B (byte 0x42) comes before /a (byte 0x61).\",\"outputA\":\"\\\" 1 /B\\\" then \\\" 1 /a\\\"\",\"readingB\":\"Path ascending is case-insensitive alphabetical order, so /a comes before /B. Counts stay separate.\",\"outputB\":\"\\\" 1 /a\\\" then \\\" 1 /B\\\"\",\"quote\":\"sorted by count descending then path ascending\",\"dimension\":\"case-sensitive versus case-folded path order\"},{\"input\":\"127.0.0.1 - - [10/Oct/2000:13:55:36 -0700] \\\"GET /a HTTP/1.1\\\" 200 1\",\"readingA\":\"A 6-character right-aligned field is space-filled.\",\"outputA\":\"\\\" 1 /a\\\"\",\"readingB\":\"A 6-character right-aligned numeric field is zero-filled.\",\"outputB\":\"\\\"000001 /a\\\"\",\"quote\":\"the count right-aligned in a 6-character field, one space, then the path\",\"dimension\":\"space fill versus zero fill\"}]}","cutOff":false} +{"id":"access-report#2","secs":601,"ok":true,"text":"{\"probes\":[{\"input\":\"10.0.0.1 - - [01/Jan/2020:00:00:00 +0000] \\\"GET /ok HTTP/1.1\\\" 200 5\\n10.0.0.1 - - [01/Jan/2020:00:00:00 +0000] \\\"GET /x HTTP/1.1 extra\\\" 404 10\",\"readingA\":\"Skip drops the whole line from every total, so the 4-part quoted request contributes to neither paths nor status classes.\",\"outputA\":\" 1 /ok\\n\\n2xx: 1\\n3xx: 0\\n4xx: 0\\n5xx: 0\",\"readingB\":\"The three-part test only gates path ranking; the status token on that same line still increments its class.\",\"outputB\":\" 1 /ok\\n\\n2xx: 1\\n3xx: 0\\n4xx: 1\\n5xx: 0\",\"quote\":\"Skip any line whose quoted request is not exactly three space-separated parts\",\"dimension\":\"skip scope vs status totals\"},{\"input\":\"10.0.0.1 - - [01/Jan/2020:00:00:00 +0000] \\\"GET /x HTTP/1.1\\\" 200 10\",\"readingA\":\"Space-separated parts are the non-empty tokens after collapsing repeated spaces, so GET, /x, HTTP/1.1 is exactly three and the line counts.\",\"outputA\":\" 1 /x\\n\\n2xx: 1\\n3xx: 0\\n4xx: 0\\n5xx: 0\",\"readingB\":\"Splitting on each single space keeps the empty field, giving GET, empty, /x, HTTP/1.1 (four parts), so the line is skipped.\",\"outputB\":\"no path line; 2xx: 0\\n3xx: 0\\n4xx: 0\\n5xx: 0\",\"quote\":\"exactly three space-separated parts\",\"dimension\":\"repeated spaces inside the request\"},{\"input\":\"10.0.0.1 - - [01/Jan/2020:00:00:00 +0000] \\\"GET\\\\t/tab\\\\tHTTP/1.1\\\" 200 8\",\"readingA\":\"Only U+0020 is a space, so the tab-separated quoted request is one part and the line is skipped.\",\"outputA\":\"no path line; 2xx: 0\\n3xx: 0\\n4xx: 0\\n5xx: 0\",\"readingB\":\"Any whitespace separates parts, so GET, /tab, HTTP/1.1 is exactly three and the line counts.\",\"outputB\":\" 1 /tab\\n\\n2xx: 1\\n3xx: 0\\n4xx: 0\\n5xx: 0\",\"quote\":\"exactly three space-separated parts\",\"dimension\":\"tab versus space separator\"},{\"input\":\"10.0.0.1 alice bob [01/Jan/2020:00:00:00 +0000] \\\"GET /secret HTTP/1.1\\\" 200 50\",\"readingA\":\"The two hyphens in the format pattern are required literal tokens, so ident/authuser alice bob does not match and the line is skipped.\",\"outputA\":\"no path line; 2xx: 0\\n3xx: 0\\n4xx: 0\\n5xx: 0\",\"readingB\":\"HOST, DATE, METHOD, PATH, PROTOCOL, STATUS, and BYTES are the varying fields, and the hyphens are just the usual empty ident and authuser, so any tokens there still count.\",\"outputB\":\" 1 /secret\\n\\n2xx: 1\\n3xx: 0\\n4xx: 0\\n5xx: 0\",\"quote\":\"Common Log Format: `HOST - - [DATE] \\\"METHOD PATH PROTOCOL\\\" STATUS BYTES`\",\"dimension\":\"literal hyphens vs ident/authuser\"},{\"input\":\"10.0.0.1 - - [01/Jan/2020:00:00:00 +0000] \\\"GET /index.html HTTP/1.1\\\" 200 2326 \\\"http://example.com/\\\" \\\"Mozilla/5.0\\\"\",\"readingA\":\"The quoted request is the first quote pair, which is three parts, so referer and user-agent after BYTES stay in the report.\",\"outputA\":\" 1 /index.html\\n\\n2xx: 1\\n3xx: 0\\n4xx: 0\\n5xx: 0\",\"readingB\":\"The line must match the stated pattern ending at BYTES; the extra quoted fields make it not that line shape, so it is skipped.\",\"outputB\":\"no path line; 2xx: 0\\n3xx: 0\\n4xx: 0\\n5xx: 0\",\"quote\":\"\\\"METHOD PATH PROTOCOL\\\" STATUS BYTES\",\"dimension\":\"trailing referer and user-agent\"},{\"input\":\"10.0.0.1 - - [01/Jan/2020:00:00:00 +0000] \\\"GET /z HTTP/1.1\\\" 0200 10\",\"readingA\":\"The status is the integer 200, which sits in the 2xx class.\",\"outputA\":\" 1 /z\\n\\n2xx: 1\\n3xx: 0\\n4xx: 0\\n5xx: 0\",\"readingB\":\"A 2xx status is a three-digit token whose first digit is 2, and the token 0200 does not match, so no class increments.\",\"outputB\":\" 1 /z\\n\\n2xx: 0\\n3xx: 0\\n4xx: 0\\n5xx: 0\",\"quote\":\"2xx: N`, `3xx: N`, `4xx: N`, `5xx: N` giving how many requests had a status in each class\",\"dimension\":\"integer status value vs 2xx token shape\"},{\"input\":\"10.0.0.1 - - [01/Jan/2020:00:00:00 +0000] \\\"GET http://ex.com/foo?x=1 HTTP/1.1\\\" 200 10\",\"readingA\":\"The path is the second request field with its ?query removed, so the counted path is http://ex.com/foo.\",\"outputA\":\" 1 http://ex.com/foo\\n\\n2xx: 1\\n3xx: 0\\n4xx: 0\\n5xx: 0\",\"readingB\":\"The path is the URI path component of the request-target, so the counted path is /foo.\",\"outputB\":\" 1 /foo\\n\\n2xx: 1\\n3xx: 0\\n4xx: 0\\n5xx: 0\",\"quote\":\"most requested paths (a path with a ?query string counts as the path without it)\",\"dimension\":\"absolute-form request-target vs URL path\"},{\"input\":\"(empty access.log)\",\"readingA\":\"The report is the path lines, then one empty line, then the four class lines, so zero paths still produce that empty line before 2xx.\",\"outputA\":\"\\n2xx: 0\\n3xx: 0\\n4xx: 0\\n5xx: 0\\n\",\"readingB\":\"With no path lines the separator is absent and the file begins at the class lines.\",\"outputB\":\"2xx: 0\\n3xx: 0\\n4xx: 0\\n5xx: 0\\n\",\"quote\":\"first the 5 most requested paths ...; then one empty line; then four lines\",\"dimension\":\"blank line when there are no paths\"}]}","cutOff":false} +{"id":"access-report#1","secs":685,"ok":true,"text":"{\"probes\":[{\"input\":\"10.0.0.1 - - [10/Oct/2000:13:55:36 -0700] \\\"GET /my file HTTP/1.1\\\" 404 12\",\"readingA\":\"The skip applies to the whole line, so this request is omitted from the status-class totals as well as from the path list.\",\"outputA\":\"\\n2xx: 0\\n3xx: 0\\n4xx: 0\\n5xx: 0\\n\",\"readingB\":\"The line is left out of the path ranking because the quoted request has four parts, but status 404 still increments the 4xx total.\",\"outputB\":\"\\n2xx: 0\\n3xx: 0\\n4xx: 1\\n5xx: 0\\n\",\"quote\":\"Skip any line whose quoted request is not exactly three space-separated parts\",\"dimension\":\"skip scope vs status totals\"},{\"input\":\"10.0.0.1 - - [10/Oct/2000:13:55:36 -0700] \\\"GET /foo HTTP/1.1\\\" 200 5\",\"readingA\":\"Consecutive spaces are one separator, so the quoted request is the three parts GET, /foo, HTTP/1.1 and the line is kept.\",\"outputA\":\" 1 /foo\\n\\n2xx: 1\\n3xx: 0\\n4xx: 0\\n5xx: 0\\n\",\"readingB\":\"Splitting on each space character yields an empty part (GET, empty, /foo, HTTP/1.1), which is not exactly three parts, so the line is skipped.\",\"outputB\":\"\\n2xx: 0\\n3xx: 0\\n4xx: 0\\n5xx: 0\\n\",\"quote\":\"exactly three space-separated parts\",\"dimension\":\"consecutive spaces as empty parts\"},{\"input\":\"10.0.0.1 - - [10/Oct/2000:13:55:36 -0700] \\\"GET\\t/foo\\tHTTP/1.1\\\" 200 5\",\"readingA\":\"Space-separated means U+0020 only. Tab characters do not split the quoted request, so the line is skipped.\",\"outputA\":\"\\n2xx: 0\\n3xx: 0\\n4xx: 0\\n5xx: 0\\n\",\"readingB\":\"The three tokens are separated by whitespace, so the request counts as GET, /foo, HTTP/1.1.\",\"outputB\":\" 1 /foo\\n\\n2xx: 1\\n3xx: 0\\n4xx: 0\\n5xx: 0\\n\",\"quote\":\"exactly three space-separated parts\",\"dimension\":\"tab vs space separator\"},{\"input\":\"10.0.0.1 alice bob [10/Oct/2000:13:55:36 -0700] \\\"GET /secret HTTP/1.1\\\" 201 5\",\"readingA\":\"The format template hard-codes the two dash fields, so a line with ident alice and authuser bob is left out of the report.\",\"outputA\":\"\\n2xx: 0\\n3xx: 0\\n4xx: 0\\n5xx: 0\\n\",\"readingB\":\"Common Log Format allows any ident and authuser tokens; the quoted request has three parts, so /secret is counted and 201 is 2xx.\",\"outputB\":\" 1 /secret\\n\\n2xx: 1\\n3xx: 0\\n4xx: 0\\n5xx: 0\\n\",\"quote\":\"Common Log Format: `HOST - - [DATE]\",\"dimension\":\"literal dashes vs any ident/user\"},{\"input\":\"10.0.0.1 - - [10/Oct/2000:13:55:36 -0700] \\\"GET /foo HTTP/1.1\\\" 304 -\",\"readingA\":\"In Common Log Format a hyphen is the byte count when no bytes were sent, so /foo is listed and 304 counts as 3xx.\",\"outputA\":\" 1 /foo\\n\\n2xx: 0\\n3xx: 1\\n4xx: 0\\n5xx: 0\\n\",\"readingB\":\"BYTES in the template is a numeric count, so a trailing hyphen fails the line and the request is omitted.\",\"outputB\":\"\\n2xx: 0\\n3xx: 0\\n4xx: 0\\n5xx: 0\\n\",\"quote\":\"`STATUS BYTES`\",\"dimension\":\"hyphen allowed as byte count\"},{\"input\":\"\",\"readingA\":\"The report skeleton always writes the path section, then one empty line, then the four class lines, so an empty log still begins with a blank line.\",\"outputA\":\"\\n2xx: 0\\n3xx: 0\\n4xx: 0\\n5xx: 0\\n\",\"readingB\":\"The empty line is only a separator after path lines. With no paths, the file starts at the status block.\",\"outputB\":\"2xx: 0\\n3xx: 0\\n4xx: 0\\n5xx: 0\\n\",\"quote\":\"then one empty line; then four lines\",\"dimension\":\"blank separator when no paths\"},{\"input\":\"10.0.0.1 - - [10/Oct/2000:13:55:36 -0700] \\\"GET /a HTTP/1.1\\\" 200 1\\n10.0.0.1 - - [10/Oct/2000:13:55:36 -0700] \\\"GET /b HTTP/1.1\\\" 200 1\\n10.0.0.1 - - [10/Oct/2000:13:55:36 -0700] \\\"GET /c HTTP/1.1\\\" 200 1\\n10.0.0.1 - - [10/Oct/2000:13:55:36 -0700] \\\"GET /d HTTP/1.1\\\" 200 1\\n10.0.0.1 - - [10/Oct/2000:13:55:36 -0700] \\\"GET /e HTTP/1.1\\\" 200 1\\n10.0.0.1 - - [10/Oct/2000:13:55:36 -0700] \\\"GET /f HTTP/1.1\\\" 200 1\",\"readingA\":\"Sort by count descending then path ascending and emit exactly five lines, so the tied paths are /a through /e.\",\"outputA\":\" 1 /a\\n 1 /b\\n 1 /c\\n 1 /d\\n 1 /e\\n\\n2xx: 6\\n3xx: 0\\n4xx: 0\\n5xx: 0\\n\",\"readingB\":\"Path ascending only orders rows; every path tied on the cutoff count is kept, so all six count-1 paths are emitted.\",\"outputB\":\" 1 /a\\n 1 /b\\n 1 /c\\n 1 /d\\n 1 /e\\n 1 /f\\n\\n2xx: 6\\n3xx: 0\\n4xx: 0\\n5xx: 0\\n\",\"quote\":\"the 5 most requested paths ... sorted by count descending then path ascending\",\"dimension\":\"top-5 cutoff when counts tie\"}]}","cutOff":false} +{"id":"node-summarize#0","secs":335,"ok":true,"text":"{\"probes\":[{\"input\":\"[{\\\"user\\\":\\\"alice\\\",\\\"amount\\\":\\\"10.00\\\",\\\"type\\\":\\\"refund\\\"}]\",\"readingA\":\"Every user string that appears is a key; ignored types add nothing, so the balance is zero.\",\"outputA\":\"{\\\"alice\\\":\\\"0.00\\\"}\",\"readingB\":\"Non-credit/debit transactions are dropped before users are collected, so alice is omitted.\",\"outputB\":\"{}\",\"quote\":\"keys are every user that appears in the file; transactions of any other type are ignored\",\"dimension\":\"users only on ignored types\"},{\"input\":\"[{\\\"user\\\":\\\"Bob\\\",\\\"amount\\\":\\\"1.00\\\",\\\"type\\\":\\\"credit\\\"},{\\\"user\\\":\\\"alice\\\",\\\"amount\\\":\\\"2.00\\\",\\\"type\\\":\\\"credit\\\"}]\",\"readingA\":\"Sort keys by UTF-16 code unit, so uppercase Bob comes before alice.\",\"outputA\":\"{\\\"Bob\\\":\\\"1.00\\\",\\\"alice\\\":\\\"2.00\\\"}\",\"readingB\":\"Sort keys case-insensitively in alphabetical order, so alice comes before Bob.\",\"outputB\":\"{\\\"alice\\\":\\\"2.00\\\",\\\"Bob\\\":\\\"1.00\\\"}\",\"quote\":\"in sorted order\",\"dimension\":\"case-sensitive vs case-insensitive sort\"},{\"input\":\"[{\\\"user\\\":\\\"10\\\",\\\"amount\\\":\\\"1.00\\\",\\\"type\\\":\\\"credit\\\"},{\\\"user\\\":\\\"2\\\",\\\"amount\\\":\\\"3.00\\\",\\\"type\\\":\\\"debit\\\"}]\",\"readingA\":\"Lexicographic sort of user strings, emitted in that order.\",\"outputA\":\"{\\\"10\\\":\\\"1.00\\\",\\\"2\\\":\\\"-3.00\\\"}\",\"readingB\":\"Numeric/integer-index order (ordinary JSON object key order for \\\"2\\\" and \\\"10\\\").\",\"outputB\":\"{\\\"2\\\":\\\"-3.00\\\",\\\"10\\\":\\\"1.00\\\"}\",\"quote\":\"in sorted order\",\"dimension\":\"lexicographic vs numeric key order\"},{\"input\":\"[{\\\"user\\\":\\\"a\\\",\\\"amount\\\":\\\"12.5\\\",\\\"type\\\":\\\"credit\\\"}]\",\"readingA\":\"Any decimal is accepted; 12.5 currency units prints with two decimals.\",\"outputA\":\"{\\\"a\\\":\\\"12.50\\\"}\",\"readingB\":\"Amount must look like \\\"12.50\\\" (exactly two fractional digits); \\\"12.5\\\" is ignored and the user is omitted.\",\"outputB\":\"{}\",\"quote\":\"amount is a decimal string like \\\"12.50\\\"\",\"dimension\":\"amount scale must be exactly two decimals\"},{\"input\":\"[{\\\"user\\\":\\\"a\\\",\\\"amount\\\":\\\"0.014\\\",\\\"type\\\":\\\"credit\\\"},{\\\"user\\\":\\\"a\\\",\\\"amount\\\":\\\"0.014\\\",\\\"type\\\":\\\"credit\\\"}]\",\"readingA\":\"Add exact values, then round half up once to two decimals (0.028 → 0.03).\",\"outputA\":\"{\\\"a\\\":\\\"0.03\\\"}\",\"readingB\":\"Round each amount half up to cents first (0.014 → 0.01), then add.\",\"outputB\":\"{\\\"a\\\":\\\"0.02\\\"}\",\"quote\":\"balance (credits minus debits) as a string with exactly two decimals\",\"dimension\":\"round final sum vs round each amount\"},{\"input\":\"[{\\\"user\\\":\\\"a\\\",\\\"amount\\\":\\\"0.005\\\",\\\"type\\\":\\\"debit\\\"}]\",\"readingA\":\"Round half away from zero to two decimals.\",\"outputA\":\"{\\\"a\\\":\\\"-0.01\\\"}\",\"readingB\":\"Round half to even (or toward zero) to two decimals.\",\"outputB\":\"{\\\"a\\\":\\\"0.00\\\"}\",\"quote\":\"with exactly two decimals\",\"dimension\":\"half-cent rounding mode\"},{\"input\":\"[{\\\"user\\\":\\\"a\\\",\\\"amount\\\":\\\"-5.00\\\",\\\"type\\\":\\\"debit\\\"}]\",\"readingA\":\"Amount is a signed decimal; a debit subtracts -5.00, which increases the balance.\",\"outputA\":\"{\\\"a\\\":\\\"5.00\\\"}\",\"readingB\":\"Amount is an unsigned magnitude like \\\"12.50\\\" and type alone sets the sign, so a debit of 5.00 decreases the balance.\",\"outputB\":\"{\\\"a\\\":\\\"-5.00\\\"}\",\"quote\":\"decimal string like \\\"12.50\\\"; credits minus debits\",\"dimension\":\"negative amount vs type-only sign\"},{\"input\":\"{\\\"user\\\":\\\"alice\\\",\\\"amount\\\":\\\"5.00\\\",\\\"type\\\":\\\"credit\\\"}\",\"readingA\":\"Input must be a JSON array; any other JSON value is an error.\",\"outputA\":\"exit 1, error on stderr, no JSON on stdout\",\"readingB\":\"It is valid JSON but not an array, so there are no transactions to sum.\",\"outputB\":\"{}\",\"quote\":\"reads a JSON array of transactions; isn't valid JSON\",\"dimension\":\"valid JSON that is not an array\"}]}","cutOff":false} +{"id":"node-summarize#1","secs":315,"ok":true,"text":"{\"probes\":[{\"input\":\"[{\\\"user\\\":\\\"zoe\\\",\\\"amount\\\":\\\"5.00\\\",\\\"type\\\":\\\"refund\\\"}]\",\"readingA\":\"Every user string that appears on any transaction is a key; a non-credit/debit type adds nothing, so the balance is zero.\",\"outputA\":\"{\\\"zoe\\\":\\\"0.00\\\"}\",\"readingB\":\"A transaction whose type is not credit or debit is dropped before users are recorded, so zoe is not a key.\",\"outputB\":\"{}\",\"quote\":\"keys are every user that appears in the file ... transactions of any other type are ignored\",\"dimension\":\"ignored type vs which users appear\"},{\"input\":\"[{\\\"user\\\":\\\"a\\\",\\\"amount\\\":\\\"1.00\\\",\\\"type\\\":\\\"credit\\\"},{\\\"user\\\":\\\"a\\\",\\\"amount\\\":\\\"1.00\\\",\\\"type\\\":\\\"Credit\\\"}]\",\"readingA\":\"Type match is exact and case-sensitive, so \\\"Credit\\\" is ignored and only the first amount counts.\",\"outputA\":\"{\\\"a\\\":\\\"1.00\\\"}\",\"readingB\":\"\\\"credit\\\" and \\\"debit\\\" are categories matched case-insensitively, so both amounts are credits.\",\"outputB\":\"{\\\"a\\\":\\\"2.00\\\"}\",\"quote\":\"type is \\\"credit\\\" or \\\"debit\\\" (transactions of any other type are ignored)\",\"dimension\":\"type category case match\"},{\"input\":\"[{\\\"user\\\":\\\"a\\\",\\\"amount\\\":\\\"1.005\\\",\\\"type\\\":\\\"credit\\\"}]\",\"readingA\":\"Format the exact decimal to two places with round-half-away-from-zero (half up).\",\"outputA\":\"{\\\"a\\\":\\\"1.01\\\"}\",\"readingB\":\"Format the exact decimal to two places with round-half-to-even; the hundredths digit 0 stays even.\",\"outputB\":\"{\\\"a\\\":\\\"1.00\\\"}\",\"quote\":\"decimal string like \\\"12.50\\\" ... as a string with exactly two decimals ... without floating-point error\",\"dimension\":\"half-cent rounding mode\"},{\"input\":\"[{\\\"user\\\":\\\"a\\\",\\\"amount\\\":\\\"0.014\\\",\\\"type\\\":\\\"credit\\\"},{\\\"user\\\":\\\"a\\\",\\\"amount\\\":\\\"0.014\\\",\\\"type\\\":\\\"credit\\\"}]\",\"readingA\":\"Add the exact amounts (0.028) then round half up to two decimals.\",\"outputA\":\"{\\\"a\\\":\\\"0.03\\\"}\",\"readingB\":\"Round each amount half up to two decimals first (0.01 and 0.01), then add.\",\"outputB\":\"{\\\"a\\\":\\\"0.02\\\"}\",\"quote\":\"balance (credits minus debits) as a string with exactly two decimals\",\"dimension\":\"round each amount vs round the total\"},{\"input\":\"[{\\\"user\\\":\\\"a\\\",\\\"amount\\\":\\\"1.00\\\",\\\"type\\\":\\\"credit\\\"},{\\\"user\\\":\\\"B\\\",\\\"amount\\\":\\\"2.00\\\",\\\"type\\\":\\\"debit\\\"}]\",\"readingA\":\"Sorted order is UTF-16 code-unit order, so \\\"B\\\" comes before \\\"a\\\".\",\"outputA\":\"{\\\"B\\\":\\\"-2.00\\\",\\\"a\\\":\\\"1.00\\\"}\",\"readingB\":\"Sorted order is locale collation (base letter, case secondary), so \\\"a\\\" comes before \\\"B\\\".\",\"outputB\":\"{\\\"a\\\":\\\"1.00\\\",\\\"B\\\":\\\"-2.00\\\"}\",\"quote\":\"whose keys are every user that appears in the file, in sorted order\",\"dimension\":\"case in sort order\"},{\"input\":\"[{\\\"user\\\":\\\"a\\\",\\\"amount\\\":\\\"-5.25\\\",\\\"type\\\":\\\"debit\\\"}]\",\"readingA\":\"Parse the signed decimal, then subtract it because the type is debit: 0-(-5.25).\",\"outputA\":\"{\\\"a\\\":\\\"5.25\\\"}\",\"readingB\":\"The amount is a magnitude; only the type supplies the sign, so a debit decreases the balance.\",\"outputB\":\"{\\\"a\\\":\\\"-5.25\\\"}\",\"quote\":\"amount is a decimal string like \\\"12.50\\\" ... balance (credits minus debits)\",\"dimension\":\"sign on amount vs type\"},{\"input\":\"[{\\\"user\\\":\\\"10\\\",\\\"amount\\\":\\\"1.00\\\",\\\"type\\\":\\\"credit\\\"},{\\\"user\\\":\\\"2\\\",\\\"amount\\\":\\\"1.00\\\",\\\"type\\\":\\\"credit\\\"}]\",\"readingA\":\"Sorted order of user strings is lexicographic, so \\\"10\\\" comes before \\\"2\\\".\",\"outputA\":\"{\\\"10\\\":\\\"1.00\\\",\\\"2\\\":\\\"1.00\\\"}\",\"readingB\":\"Sorted order is numeric/natural order of the user ids, so \\\"2\\\" comes before \\\"10\\\".\",\"outputB\":\"{\\\"2\\\":\\\"1.00\\\",\\\"10\\\":\\\"1.00\\\"}\",\"quote\":\"in sorted order\",\"dimension\":\"lexicographic vs numeric user sort\"}]}","cutOff":false} +{"id":"node-summarize#2","secs":368,"ok":true,"text":"{\"probes\":[{\"input\":\"[{\\\"user\\\":\\\"bob\\\",\\\"amount\\\":\\\"3.00\\\",\\\"type\\\":\\\"refund\\\"}]\",\"readingA\":\"Every user string that occurs in the file is a key. A refund is ignored for the sum, so bob's balance is zero.\",\"outputA\":\"{\\\"bob\\\":\\\"0.00\\\"}\",\"readingB\":\"Ignored transactions are dropped before keys are chosen, so a user who only appears on a refund is absent.\",\"outputB\":\"{}\",\"quote\":\"keys are every user that appears in the file / transactions of any other type are ignored\",\"dimension\":\"users only on ignored types\"},{\"input\":\"[{\\\"user\\\":\\\"alice\\\",\\\"amount\\\":\\\"1.00\\\",\\\"type\\\":\\\"credit\\\"},{\\\"user\\\":\\\"alice\\\",\\\"amount\\\":\\\"12.5\\\",\\\"type\\\":\\\"credit\\\"}]\",\"readingA\":\"A decimal string may have a single fractional digit; 12.5 is 12.50 and the balance is 13.50.\",\"outputA\":\"{\\\"alice\\\":\\\"13.50\\\"}\",\"readingB\":\"An amount must look like 12.50 with two fractional digits, so 12.5 is ignored and only 1.00 counts.\",\"outputB\":\"{\\\"alice\\\":\\\"1.00\\\"}\",\"quote\":\"amount is a decimal string like \\\"12.50\\\" / exactly two decimals\",\"dimension\":\"fewer than two fractional digits\"},{\"input\":\"[{\\\"user\\\":\\\"alice\\\",\\\"amount\\\":\\\"12.545\\\",\\\"type\\\":\\\"credit\\\"}]\",\"readingA\":\"Show the exact decimal with two places using half-away-from-zero rounding.\",\"outputA\":\"{\\\"alice\\\":\\\"12.55\\\"}\",\"readingB\":\"Show the exact decimal with two places using half-to-even rounding.\",\"outputB\":\"{\\\"alice\\\":\\\"12.54\\\"}\",\"quote\":\"as a string with exactly two decimals / computed without floating-point error\",\"dimension\":\"rounding mode past two decimals\"},{\"input\":\"[{\\\"user\\\":\\\"alice\\\",\\\"amount\\\":\\\"-3.50\\\",\\\"type\\\":\\\"credit\\\"}]\",\"readingA\":\"The amount is a signed decimal, so a credit adds -3.50.\",\"outputA\":\"{\\\"alice\\\":\\\"-3.50\\\"}\",\"readingB\":\"The amount is an unsigned magnitude like 12.50, so type credit adds 3.50.\",\"outputB\":\"{\\\"alice\\\":\\\"3.50\\\"}\",\"quote\":\"decimal string like \\\"12.50\\\" / credits minus debits\",\"dimension\":\"sign on amount versus type\"},{\"input\":\"[{\\\"user\\\":\\\"alice\\\",\\\"amount\\\":\\\"1.00\\\",\\\"type\\\":\\\"credit\\\"},{\\\"user\\\":\\\"Bob\\\",\\\"amount\\\":\\\"2.00\\\",\\\"type\\\":\\\"debit\\\"}]\",\"readingA\":\"Sorted order is UTF-16 code-unit order, so Bob comes before alice.\",\"outputA\":\"{\\\"Bob\\\":\\\"-2.00\\\",\\\"alice\\\":\\\"1.00\\\"}\",\"readingB\":\"Sorted order is case-insensitive alphabetical order, so alice comes before Bob.\",\"outputB\":\"{\\\"alice\\\":\\\"1.00\\\",\\\"Bob\\\":\\\"-2.00\\\"}\",\"quote\":\"in sorted order\",\"dimension\":\"case sensitivity of user sort\"},{\"input\":\"[{\\\"user\\\":\\\"alice\\\",\\\"amount\\\":\\\"1.00\\\",\\\"type\\\":\\\"credit\\\"},{\\\"user\\\":\\\"alice\\\",\\\"amount\\\":\\\"4.00\\\",\\\"type\\\":\\\"Credit\\\"}]\",\"readingA\":\"Type must be the exact string credit or debit, so only 1.00 is applied.\",\"outputA\":\"{\\\"alice\\\":\\\"1.00\\\"}\",\"readingB\":\"Credit is the credit category in any letter case, so both amounts apply and the balance is 5.00.\",\"outputB\":\"{\\\"alice\\\":\\\"5.00\\\"}\",\"quote\":\"type is \\\"credit\\\" or \\\"debit\\\" (transactions of any other type are ignored)\",\"dimension\":\"type match case folding\"},{\"input\":\"{\\\"user\\\":\\\"alice\\\",\\\"amount\\\":\\\"5.00\\\",\\\"type\\\":\\\"credit\\\"}\",\"readingA\":\"The file is valid JSON, so the run succeeds. There is no array of transactions, so the summary is an empty object on stdout and the exit status is 0.\",\"outputA\":\"{}\",\"readingB\":\"The input must be a JSON array. A JSON object is a failure: an error on stderr, empty stdout, and exit status 1.\",\"outputB\":\"exit 1, empty stdout, error on stderr\",\"quote\":\"reads a JSON array of transactions / If FILE can't be read or isn't valid JSON, it prints an error to stderr and exits with status 1\",\"dimension\":\"valid JSON that is not an array\"}]}","cutOff":false} +{"id":"cron-next#1","secs":729,"ok":true,"text":"{\"probes\":[{\"input\":\"python3 nextrun.py '0 0 */2 * *' '2026-10-06 00:00' 1\",\"readingA\":\"*/n is min-max/n and day-of-month starts at 1, so days 1,3,5,7,... Next midnight is the 7th.\",\"outputA\":\"2026-10-07 00:00\",\"readingB\":\"*/n keeps values divisible by n, anchored at 0, so days 2,4,6,8,... 2026-10-06 00:00 matches but is not strictly after, so the next is the 8th.\",\"outputB\":\"2026-10-08 00:00\",\"quote\":\"a step */n\",\"dimension\":\"*/n anchor on 1-based fields\"},{\"input\":\"python3 nextrun.py '10-20/3 0 * * *' '2026-10-06 00:00' 1\",\"readingA\":\"a-b/n starts at a and adds n through b: minutes 10,13,16,19. First is 00:10.\",\"outputA\":\"2026-10-06 00:10\",\"readingB\":\"a-b/n keeps values in the range divisible by n: minutes 12,15,18. First is 00:12.\",\"outputB\":\"2026-10-06 00:12\",\"quote\":\"a step */n or a-b/n\",\"dimension\":\"range step phase\"},{\"input\":\"python3 nextrun.py '0 5-6 * * *' '2026-10-06 05:30' 1\",\"readingA\":\"Range a-b includes both ends, so hours 5 and 6. 05:00 has passed; next is 06:00.\",\"outputA\":\"2026-10-06 06:00\",\"readingB\":\"Range a-b is half-open on the right, so only hour 5. 05:00 has passed; next is tomorrow 05:00.\",\"outputB\":\"2026-10-07 05:00\",\"quote\":\"a range a-b\",\"dimension\":\"inclusive vs exclusive range\"},{\"input\":\"python3 nextrun.py '0 0 */1 * 1' '2026-10-06 00:00' 1\",\"readingA\":\"*/1 is not the token *, so both day fields are restricted and either field may match. */1 hits every day-of-month, so every day matches; next midnight is Oct 7.\",\"outputA\":\"2026-10-07 00:00\",\"readingB\":\"*/1 matches every day-of-month, so it counts as * and the either-rule does not apply. Only Monday matches. 2026-10-06 is Tuesday, so next is Oct 12.\",\"outputB\":\"2026-10-12 00:00\",\"quote\":\"when both day-of-month and day-of-week are something other than *\",\"dimension\":\"literal * vs match-all\"},{\"input\":\"python3 nextrun.py '0 12 * * 5-1' '2026-10-06 00:00' 1\",\"readingA\":\"A range a-b only counts upward. 5>1, so no day-of-week matches and nothing is printed.\",\"outputA\":\"no lines\",\"readingB\":\"Day-of-week wraps, so 5-1 is Fri, Sat, Sun, Mon. 2026-10-06 is Tuesday; next 12:00 is Friday Oct 9.\",\"outputB\":\"2026-10-09 12:00\",\"quote\":\"a range a-b\",\"dimension\":\"descending range wrap\"},{\"input\":\"python3 nextrun.py '0 0 * * 0-7/2' '2026-10-06 00:00' 1\",\"readingA\":\"7 is just another numeric Sunday at the top of the range. 0-7/2 expands to 0,2,4,6 (Sun, Tue, Thu, Sat). Tue 00:00 is not strictly after itself, so next is Thursday.\",\"outputA\":\"2026-10-08 00:00\",\"readingB\":\"7 means the same value as Sunday=0, so the endpoint is rewritten to 0 before expanding and 0-0/2 is only Sunday. Next Sunday is Oct 11.\",\"outputB\":\"2026-10-11 00:00\",\"quote\":\"7 also means Sunday\",\"dimension\":\"Sunday alias inside a stepped range\"}]}","cutOff":false} +{"id":"cron-next#0","secs":900,"ok":false,"error":"TimeoutError: Grok Build did not answer within 900s"} +{"id":"cron-next#2","secs":780,"ok":true,"text":"{\"probes\":[{\"input\":\"python3 nextrun.py '0 0 */2 * *' '2024-01-01 00:00' 1\",\"readingA\":\"*/n is min-max stepping by n, and day-of-month starts at 1, so days 1,3,5,... Jan 1 is excluded by strictly after.\",\"outputA\":\"2024-01-03 00:00\",\"readingB\":\"*/n means the field value is divisible by n, so days 2,4,6,...\",\"outputB\":\"2024-01-02 00:00\",\"quote\":\"a step */n\",\"dimension\":\"*/n origin on day-of-month\"},{\"input\":\"python3 nextrun.py '10-20/3 0 1 1 *' '2024-01-01 00:00' 1\",\"readingA\":\"a-b/n is the arithmetic sequence a, a+n, a+2n, ... while <= b, so minutes 10,13,16,19.\",\"outputA\":\"2024-01-01 00:10\",\"readingB\":\"a-b/n keeps values in a-b that are divisible by n, so minutes 12,15,18.\",\"outputB\":\"2024-01-01 00:12\",\"quote\":\"a step */n or a-b/n\",\"dimension\":\"a-b/n sequence vs modulo\"},{\"input\":\"python3 nextrun.py '0 0 */1 * 1' '2024-01-01 00:00' 1\",\"readingA\":\"*/1 is something other than *, so day-of-month OR day-of-week applies; */1 matches every date, so every midnight matches.\",\"outputA\":\"2024-01-02 00:00\",\"readingB\":\"*/1 still counts as * (stepped asterisk), so the either-rule does not apply and only Monday matches.\",\"outputB\":\"2024-01-08 00:00\",\"quote\":\"when both day-of-month and day-of-week are something other than *\",\"dimension\":\"whether */n counts as *\"},{\"input\":\"python3 nextrun.py '0 0 * * 5-1' '2024-01-05 00:00' 1\",\"readingA\":\"A range a-b only runs from a up to b, so 5-1 matches no weekday.\",\"outputA\":\"no lines\",\"readingB\":\"A weekday range wraps, so 5-1 is Fri, Sat, Sun, Mon; Friday at the start is excluded.\",\"outputB\":\"2024-01-06 00:00\",\"quote\":\"a range a-b\",\"dimension\":\"inverted range wrap\"},{\"input\":\"python3 nextrun.py '0 0 1-10/3 * *' '2024-01-07 00:00' 1\",\"readingA\":\"a-b/n includes the endpoint when it lands on a step: days 1,4,7,10. Jan 7 equals the start, so the next is Jan 10.\",\"outputA\":\"2024-01-10 00:00\",\"readingB\":\"The end b is exclusive, so days are 1,4,7 only; the next after Jan 7 is Feb 1.\",\"outputB\":\"2024-02-01 00:00\",\"quote\":\"a-b/n\",\"dimension\":\"whether range end b is included\"}]}","cutOff":false} diff --git a/bench/local/ambiguity_probe.py b/bench/local/ambiguity_probe.py new file mode 100644 index 0000000..31eacb3 --- /dev/null +++ b/bench/local/ambiguity_probe.py @@ -0,0 +1,85 @@ +""" +Ambiguity probe experiment (frontier report, Bet 3). + + python3 ambiguity_probe.py run [task,task|all] [reps] # Grok 4.7, inside the container only + python3 ambiguity_probe.py show + +The model sees ONLY the task prompt and the pristine file listing (names + sizes, no contents, +no grader). `run` builds each task's starting folder on the host (setup only; no model runs on +the host), then runs ambiguity_probe_driver.mjs in a throwaway container named maat-bench-ab3-* +with the owner's Grok login mounted read-only outside HOME (as run-in-container.sh does). +Never run grok on the host: its config auto-approves its own tools. +""" +from __future__ import annotations +import json, os, subprocess, sys, tempfile +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +from tasks import TASKS # noqa: E402 +from tasks2 import TASKS2 # noqa: E402 + +ALL = {T.name: T for T in TASKS + TASKS2} +IMG = os.environ.get("PROBE_IMAGE", "maat-bench-grok:6d91d673") +SYSTEM = (HERE / "ambiguity_probe_prompt.txt").read_text() + + +def listing(d: Path) -> str: + rows = [] + for p in sorted(d.rglob("*")): + rel = p.relative_to(d) + if ".git" in rel.parts or not p.is_file(): + continue + rows.append(f"{rel} ({p.stat().st_size} bytes)") + return "\n".join(rows) or "(empty folder)" + + +def user_prompt(T) -> str: + with tempfile.TemporaryDirectory() as t: + d = Path(t) + T.setup(d) + files = listing(d) + return f"TASK:\n{T.PROMPT}\n\nFILES PRESENT (name and size):\n{files}\n" + + +def run(out: Path, names: list[str], reps: int) -> None: + jobs = [{"id": f"{n}#{r}", "system": SYSTEM, "prompt": SYSTEM + "\n\n----\n\n" + user_prompt(ALL[n]) + "\nReply now with the JSON object only."} for n in names for r in range(reps)] + work = Path.home() / ".cache/maat-bench/container-work/ab3" + work.mkdir(parents=True, exist_ok=True) + (work / "jobs.json").write_text(json.dumps(jobs)) + (work / "out.jsonl").unlink(missing_ok=True) + cmd = ["docker", "run", "--rm", "--name", f"maat-bench-ab3-{os.getpid()}", + "-e", f"PROBE_TIMEOUT_MS={os.environ.get('PROBE_TIMEOUT_MS', '240000')}", "-e", f"PROBE_CONC={os.environ.get('PROBE_CONC', '3')}", "-v", f"{HERE}:/bench:ro", "-v", f"{work}:/work", + "-v", f"{Path.home()}/.grok/auth.json:/grok-cred/auth.json:ro", IMG, "sh", "-c", + 'mkdir -p $HOME/.grok /tmp/empty && install -m 600 /grok-cred/auth.json $HOME/.grok/auth.json && ' + 'cd /tmp/empty && ' + 'node /bench/ambiguity_probe_driver.mjs /work/jobs.json /work/out.jsonl'] + subprocess.run(cmd, check=True) + with out.open("a") as f: + f.write((work / "out.jsonl").read_text()) + + +def parse(text: str) -> list[dict]: + s, e = text.find("{"), text.rfind("}") + try: + return json.loads(text[s:e + 1]).get("probes", []) + except Exception: + return [] + + +def show(out: Path) -> None: + for line in out.read_text().splitlines(): + r = json.loads(line) + ps = parse(r.get("text", "")) if r.get("ok") else [] + print(f"== {r['id']} ok={r.get('ok')} probes={len(ps)} {r.get('error','')[:100]}") + for p in ps: + print(f" - [{p.get('dimension')}] {p.get('input')!r}: A={p.get('outputA')!r} B={p.get('outputB')!r}") + + +if __name__ == "__main__": + cmd = sys.argv[1] + if cmd == "run": + names = sys.argv[3].split(",") if len(sys.argv) > 3 and sys.argv[3] != "all" else list(ALL) + run(Path(sys.argv[2]), names, int(sys.argv[4]) if len(sys.argv) > 4 else 1) + else: + show(Path(sys.argv[2])) diff --git a/bench/local/ambiguity_probe_driver.mjs b/bench/local/ambiguity_probe_driver.mjs new file mode 100644 index 0000000..8e2b1dd --- /dev/null +++ b/bench/local/ambiguity_probe_driver.mjs @@ -0,0 +1,19 @@ +// Runs INSIDE the container: reads jobs.json [{id,system,prompt}], writes one line per ask to out.jsonl. +// Uses the installed Maat's askModel (the same call the criteria drafter makes), no tools. +import { readFileSync, appendFileSync } from "node:fs"; +import { execSync } from "node:child_process"; +const root = execSync("npm root -g").toString().trim(); +const { askModel } = await import(`${root}/@solvyx/molt/dist/ask.js`); +const [jobsFile, outFile] = process.argv.slice(2); +const jobs = JSON.parse(readFileSync(jobsFile, "utf8")); +const conc = Number(process.env.PROBE_CONC || 3); +let i = 0; +async function worker() { + while (i < jobs.length) { + const j = jobs[i++]; + const t0 = Date.now(); + const r = await askModel({ baseUrl: "grok-build://subscription", model: "grok-4.7", system: j.system, prompt: j.prompt, maxTokens: 6000, reasoningEffort: process.env.PROBE_EFFORT || "low", cwd: "/tmp/empty", what: "probing ambiguity", timeoutMs: Number(process.env.PROBE_TIMEOUT_MS || 240000) }); + appendFileSync(outFile, JSON.stringify({ id: j.id, secs: Math.round((Date.now() - t0) / 1000), ...r }) + "\n"); + } +} +await Promise.all(Array.from({ length: conc }, worker)); diff --git a/bench/local/ambiguity_probe_prompt.txt b/bench/local/ambiguity_probe_prompt.txt new file mode 100644 index 0000000..c99d0b3 --- /dev/null +++ b/bench/local/ambiguity_probe_prompt.txt @@ -0,0 +1,14 @@ +The files are NOT available to you (your working folder is empty): do not read, search or run anything. Everything you may use is in the message. Your FIRST and ONLY reply must be the JSON object, with no preamble. + +You review a task given to a coding agent BEFORE any work starts. You see only the task text and the listing of the files that exist. Find the places where two reasonable readings of the task would make a correct-looking deliverable produce DIFFERENT OUTPUT on the same input. + +List up to 8 probes. A probe is one concrete input or situation (a literal value, a line, a date, a number, an edge case), the two readings, and what each reading would output for that input. Prefer boundary cases: off-by-one in what counts, inclusive/exclusive ranges, rounding and formatting, whether a match must be a whole token or may be a substring, how two rules combine (and/or), empty or malformed input, ordering and ties, what counts as a status/category. + +Rules: +- Only real divergence: if both readings give the same output on the input, it is not a probe. +- Do not list generic engineering concerns (error handling, performance, style, "should I add tests"). +- Give literal inputs, not descriptions of inputs. Quote the task words each reading rests on. +- Fewer probes is better than padding. Return an empty list if the task is unambiguous. + +Reply with JSON only: +{"probes":[{"input":"...","readingA":"...","outputA":"...","readingB":"...","outputB":"...","quote":"task words the ambiguity rests on","dimension":"one short phrase"}]} diff --git a/bench/local/checkquality_analysis.py b/bench/local/checkquality_analysis.py new file mode 100644 index 0000000..952529b --- /dev/null +++ b/bench/local/checkquality_analysis.py @@ -0,0 +1,223 @@ +"""Check-quality replay, step 4: run-level recall/precision under candidate filters. + +Inputs (scratchpad): cq-runs.json, cq-checks-lint.json, cq-taxonomy.json, +replay.cmds.json, replay.out.jsonl (container replay of every unique check on +pristine, reference and every saved tree of its task). + +Verdict model (bar level): a run is ACCEPTED when at least one live drafted +check remains and every live one passes on the run's own tree. "live" is what +the filter under test leaves. positive = accepted; precision = accepted runs +that the grader passed / accepted; recall = accepted correct / correct. + + python3 checkquality_analysis.py SCRATCH_DIR +""" +from __future__ import annotations + +import json +import math +import sys +from collections import Counter, defaultdict +from pathlib import Path + +S = Path(sys.argv[1]) +R = json.load(open(S / "cq-runs.json")) +C = json.load(open(S / "cq-checks-lint.json")) +TX = {(x["id"], x["name"]): x["cls"] for x in json.load(open(S / "cq-taxonomy.json"))} +CMDS = json.load(open(S / "replay.cmds.json")) +RUNS_BY_ID = {r["id"]: r for r in R} +LANES = ["v10-old", "v10-fix", "v11-old", "v11-new", "v12-old", "v12-new"] +NOISY = {"L14-number", "L10-path"} # measured but excluded from the clean lint set + + +def rule(f: str) -> str: + return f.split(":")[0] + + +# ---------------------------------------------------------------- replay results +res: dict[tuple[str, str], dict] = {} +for line in open(S / "replay.out.jsonl"): + j = json.loads(line) + res[(j["id"], j["tree"])] = j +cmd_id = {(v["task"], v["cmd"]): k for k, v in CMDS.items()} + + +def rep(task: str, cmd: str, tree: str) -> dict | None: + return res.get((cmd_id[(task, cmd)], tree)) + + +def ok(j: dict | None) -> bool | None: + if j is None: + return None + return j["exit"] == 0 and not j["timedOut"] + + +# ---------------------------------------------------------------- per-check features +labels = {r["id"]: r["passed"] for r in R} +by_task_runs: dict[str, list[str]] = defaultdict(list) +for r in R: + by_task_runs[r["task"]].append(r["id"]) + +agree = Counter() +for x in C: + j = rep(x["task"], x["cmd"], x["id"]) + x["own"] = ok(j) + x["pristine"] = ok(rep(x["task"], x["cmd"], "pristine")) + x["ref"] = ok(rep(x["task"], x["cmd"], "ref")) + oc = [ok(rep(x["task"], x["cmd"], rid)) for rid in by_task_runs[x["task"]] if rid != x["id"] and labels[rid]] + ow = [ok(rep(x["task"], x["cmd"], rid)) for rid in by_task_runs[x["task"]] if rid != x["id"] and not labels[rid]] + oc = [v for v in oc if v is not None] + ow = [v for v in ow if v is not None] + x["oc_pass"], x["oc_n"] = sum(oc), len(oc) + x["ow_pass"], x["ow_n"] = sum(ow), len(ow) + if x["own"] is not None and x["ok"] is not None: + agree[(x["ok"], x["own"])] += 1 + +print("## replay calibration: recorded final result vs container replay on the same tree") +tot = sum(agree.values()) +print(f"agree pass/pass {agree[(True, True)]}, fail/fail {agree[(False, False)]}, recorded-pass/replay-fail {agree[(True, False)]}, recorded-fail/replay-pass {agree[(False, True)]} (agreement {100*(agree[(True,True)]+agree[(False,False)])/tot:.1f}% of {tot})") +dis = [x for x in C if x["own"] is not None and x["ok"] is not None and x["own"] != x["ok"]] +dc = Counter((x["task"], x["ok"], x["own"]) for x in dis) +print("disagreements by task (recorded, replay):", dict(dc)) + +# ---------------------------------------------------------------- feature tables +rc = [x for x in C if x["passed"] and x["ok"] is False] +cw = [x for x in C if not x["passed"] and x["ok"] is False] +pc = [x for x in C if x["passed"] and x["ok"]] +pw = [x for x in C if not x["passed"] and x["ok"]] + + +def share(xs, pred): + n = sum(1 for x in xs if pred(x)) + return f"{n}/{len(xs)} ({100*n/len(xs):.0f}%)" if xs else "0/0" + + +print("\n## per-check features by group (final bar on own tree x grader)") +print("group | fails on REF | passes on PRISTINE | fails >=1 other correct tree | fails >=2 | fails >=50% | passes >=80% of other WRONG trees | lint(clean) | lint(any)") +for name, xs in (("refused-correct", rc), ("caught-wrong", cw), ("pass-on-correct", pc), ("pass-on-wrong", pw)): + print(f"{name:16} | {share(xs, lambda x: x['ref'] is False):14} | {share(xs, lambda x: x['pristine'] is True):18} | " + f"{share(xs, lambda x: x['oc_n'] and x['oc_pass'] < x['oc_n']):28} | {share(xs, lambda x: x['oc_n'] - x['oc_pass'] >= 2):9} | " + f"{share(xs, lambda x: x['oc_n'] and (x['oc_n'] - x['oc_pass']) * 2 >= x['oc_n']):11} | " + f"{share(xs, lambda x: x['ow_n'] and x['ow_pass'] * 5 >= x['ow_n'] * 4):33} | " + f"{share(xs, lambda x: any(rule(f) not in NOISY for f in x['flags'])):11} | {share(xs, lambda x: bool(x['flags']))}") + +print("\nrefused-correct by taxonomy class: fails on ref / fails >=2 other correct / lint(clean)") +byc = defaultdict(list) +for x in rc: + byc[TX[(x["id"], x["name"])]].append(x) +for k, xs in sorted(byc.items(), key=lambda kv: -len(kv[1])): + print(f" {k}: n={len(xs)} ref-fail {sum(1 for x in xs if x['ref'] is False)} cross>=2 {sum(1 for x in xs if x['oc_n']-x['oc_pass']>=2)} lint {sum(1 for x in xs if any(rule(f) not in NOISY for f in x['flags']))} union {sum(1 for x in xs if x['ref'] is False or x['oc_n']-x['oc_pass']>=2 or any(rule(f) not in NOISY for f in x['flags']))}") + + +# ---------------------------------------------------------------- run-level filters +def live_filter(name: str): + def f(x: dict) -> bool: # True = keep the check + clean = not any(rule(fl) not in NOISY for fl in x["flags"]) + anylint = not x["flags"] + if name == "F0 current": + return True + if name == "F1 ref-filter": + return x["ref"] is not False + if name == "F2 pristine-pass advisory": + return x["pristine"] is not True + if name == "F3 lint clean": + return clean + if name == "F3b lint all rules": + return anylint + if name == "F4 cross>=2 (LOO)": + return not (x["oc_n"] - x["oc_pass"] >= 2) + if name == "F4b cross>=1 (LOO)": + return not (x["oc_n"] and x["oc_pass"] < x["oc_n"]) + if name == "F4c cross>=50% (LOO)": + return not (x["oc_n"] and (x["oc_n"] - x["oc_pass"]) * 2 >= x["oc_n"]) + if name == "F5 lint clean + pristine": + return clean and x["pristine"] is not True + if name == "F6 lint clean + cross>=2": + return clean and not (x["oc_n"] - x["oc_pass"] >= 2) + if name == "F7 lint clean + ref": + return clean and x["ref"] is not False + if name == "F8 lint+pristine+cross>=2": + return clean and x["pristine"] is not True and not (x["oc_n"] - x["oc_pass"] >= 2) + if name == "F9 ref + pristine": + return x["ref"] is not False and x["pristine"] is not True + raise KeyError(name) + return f + + +checks_by_run: dict[str, list[dict]] = defaultdict(list) +for x in C: + checks_by_run[x["id"]].append(x) + + +def verdict(xs: list[dict], keep, tolerate: int = 0, use_replay: bool = True) -> bool | None: + live = [x for x in xs if keep(x)] + if not live: + return None + fails = 0 + for x in live: + v = x["own"] if (use_replay and x["own"] is not None) else x["ok"] + if v is False: + fails += 1 + if tolerate and len(live) >= 3 and fails <= tolerate: + return True + return fails == 0 + + +def wilson(k: int, n: int) -> tuple[float, float]: + if n == 0: + return (0.0, 0.0) + z = 1.96 + p = k / n + d = 1 + z * z / n + c = (p + z * z / (2 * n)) / d + h = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / d + return (max(0, c - h), min(1, c + h)) + + +def table(title: str, filters: list[str], tolerate: int = 0, use_replay: bool = True) -> None: + print(f"\n## {title}") + print("filter | lane | correct | accepted-correct (recall) | accepted-WRONG | precision | no live check (correct runs) ") + for name in filters: + keep = live_filter(name) + for lane in LANES + ["v12 both", "v10+v11", "ALL"]: + rs = [r for r in R if (r["lane"] == lane if lane in LANES else r["lane"].startswith("v12") if lane == "v12 both" else not r["lane"].startswith("v12") if lane == "v10+v11" else True)] + corr = [r for r in rs if r["passed"]] + acc_c = acc_w = none_c = 0 + for r in rs: + v = verdict(checks_by_run.get(r["id"], []), keep, tolerate, use_replay) + if v is None and r["passed"]: + none_c += 1 + if v: + if r["passed"]: + acc_c += 1 + else: + acc_w += 1 + prec = acc_c / (acc_c + acc_w) if acc_c + acc_w else float("nan") + lo, hi = wilson(acc_c, len(corr)) + if lane in ("v12-new", "v12 both", "v10+v11", "ALL"): + print(f"{name:29} | {lane:8} | {len(corr):7} | {acc_c:3} ({100*acc_c/len(corr):.0f}%, CI {100*lo:.0f}-{100*hi:.0f}) | {acc_w:3} | {prec:.2f} | {none_c}") + + +ALL_F = ["F0 current", "F1 ref-filter", "F2 pristine-pass advisory", "F3 lint clean", "F3b lint all rules", "F4 cross>=2 (LOO)", "F4b cross>=1 (LOO)", "F4c cross>=50% (LOO)", + "F5 lint clean + pristine", "F6 lint clean + cross>=2", "F7 lint clean + ref", "F8 lint+pristine+cross>=2", "F9 ref + pristine"] +table("bar-level verdict with each filter (replayed results on own tree; all live must pass)", ALL_F) +table("same, tolerating 1 failing check when >=3 live", ["F0 current", "F3 lint clean", "F6 lint clean + cross>=2", "F8 lint+pristine+cross>=2"], tolerate=1) +table("same as the first table but on RECORDED results (no replay), sanity", ["F0 current", "F3 lint clean"], use_replay=False) + +# which wrong runs are accepted under F8, and why +print("\n## wrong runs accepted under F8 (false verifieds the filters would leave), v12 lanes") +keep = live_filter("F8 lint+pristine+cross>=2") +for r in R: + if r["lane"].startswith("v12") and not r["passed"] and verdict(checks_by_run.get(r["id"], []), keep): + live = [x for x in checks_by_run[r["id"]] if keep(x)] + print(f" {r['id']:28} why={r['why'][:60]!r} live={[ (x['name'].replace('task:',''), x['tags']) for x in live]}") +print("\n## correct runs still refused under F8, v12-new, with the refusing check's class") +for r in R: + if r["lane"] == "v12-new" and r["passed"]: + xs = checks_by_run.get(r["id"], []) + v = verdict(xs, keep) + if v is False: + bad = [(x["name"].replace("task:", ""), TX.get((x["id"], x["name"]), "?"), x["cmd"][:70]) for x in xs if keep(x) and (x["own"] if x["own"] is not None else x["ok"]) is False] + print(f" {r['id']:28} {bad}") + elif v is None: + print(f" {r['id']:28} no live check (sealed {r['sealed_n']}, cut {r['cut']}, checks {len(xs)})") +json.dump(C, open(S / "cq-checks-final.json", "w")) diff --git a/bench/local/checkquality_lint.py b/bench/local/checkquality_lint.py new file mode 100644 index 0000000..55fff82 --- /dev/null +++ b/bench/local/checkquality_lint.py @@ -0,0 +1,202 @@ +"""Check-quality replay: structural lint rules a drafter could apply at seal time, +before any work exists. Every rule reads only the command, the task text, the +pristine tree and what is installed. No model. + + from checkquality_lint import flags + flags(task_name, cmd, pristine_dir, task_text) -> [rule ids] +""" +from __future__ import annotations + +import ast +import re +from functools import lru_cache +from pathlib import Path + +# what the bench image has (Dockerfile): python3 git sqlite3 jq make curl node; no pytest, no python, no pip +INSTALLED = {"python3", "git", "sqlite3", "jq", "make", "curl", "node", "npm", "npx", "sh", "bash", "awk", "sed", "grep", "find", "xargs", "sort", "uniq", "wc", "head", "tail", "cut", "tr", "cat", "diff", "cmp", "test", "ls", "cp", "mv", "rm", "mkdir", "touch", "tee", "echo", "printf", "true", "false", "seq", "date", "stat", "chmod", "basename", "dirname", "env", "sleep", "kill", "pkill", "timeout", "tac", "rev", "od", "xxd", "file", "md5sum", "sha256sum", "ps", "pgrep", "nproc", "cd", "exit", "read", "set", "export", "eval", "exec", "command", "type", "which", "mktemp", "realpath", "readlink", "ln", "du", "df", "yes", "comm", "join", "paste", "split", "expr", "bc", "nc", "ss", "wait", "trap", "shift", "local", "return", "if", "then", "else", "fi", "for", "do", "done", "while", "case", "esac", "in", "python3.11", "sqlite3"} +INSTALLED = set(INSTALLED) - {"bc", "nc", "ss"} +PY_MODULES_ABSENT = {"pytest", "numpy", "pandas", "requests", "yaml", "hypothesis", "flask"} + + +@lru_cache(maxsize=None) +def pristine_defs(pristine: str) -> dict[str, tuple[int, int]]: + """python function name -> (required positional count, total positional count) across pristine .py files.""" + out: dict[str, tuple[int, int]] = {} + for p in Path(pristine).rglob("*.py"): + try: + tree = ast.parse(p.read_text()) + except Exception: + continue + for node in ast.walk(tree): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + a = node.args + pos = a.posonlyargs + a.args + req = len(pos) - len(a.defaults) + out[node.name] = (req, len(pos) if a.vararg is None else 99) + return out + + +@lru_cache(maxsize=None) +def pristine_files(pristine: str) -> set[str]: + out = set() + for p in Path(pristine).rglob("*"): + if ".git" in p.parts: + continue + rel = p.relative_to(pristine).as_posix() + out.add(rel) + out.add(p.name) + return out + + +@lru_cache(maxsize=None) +def pristine_text(pristine: str) -> str: + buf = [] + for p in Path(pristine).rglob("*"): + if ".git" in p.parts or not p.is_file() or p.stat().st_size > 500_000: + continue + try: + buf.append(p.read_text()) + except Exception: + pass + return "\n".join(buf) + + +def _count_args(call: str) -> int: + """positional args in the text between the call's parentheses (top level commas).""" + depth, n, seen = 0, 0, False + i = 0 + quote = None + while i < len(call): + ch = call[i] + if quote: + if ch == "\\": + i += 2 + continue + if ch == quote: + quote = None + seen = True + elif ch in "\"'": + quote = ch + elif ch in "([{": + depth += 1 + elif ch in ")]}": + if depth == 0: + break + depth -= 1 + elif ch == "," and depth == 0: + n += 1 + elif not ch.isspace(): + seen = True + i += 1 + return (n + 1) if seen else 0 + + +EXT = r"(?:py|js|sh|csv|json|txt|md|log|ini|db|ya?ml|conf|cfg|html|xml|toml|sql|jsonl|gz|tsv|jpe?g|png|pdf|zip|docx|gif)" +PATH_TOK = re.compile(r"(?>?\s*|touch\s+|tee\s+(?:-a\s+)?|mkdir\s+(?:-p\s+)?|\bcp\s+\S+\s+|\bmv\s+\S+\s+|open\(\s*['\"]|-o\s+|-name\s+['\"]?|\bcd\s+)([\w./*-]+)") + + +def flags(task: str, cmd: str, pristine: str, text: str) -> list[str]: + f: list[str] = [] + low = text.lower() + files = pristine_files(pristine) + ptext = pristine_text(pristine) + has_git_dir = (Path(pristine) / ".git").is_dir() + + # L1 tool/module not installed + if re.search(r"python3?\s+-m\s+pytest|\bpytest\b", cmd): + f.append("L1-pytest") + for m in re.finditer(r"(?:^|[;&|(]\s*)(?:time\s+)?([A-Za-z_][\w.-]*)(?=\s|$)", cmd): + w = m.group(1) + if w not in INSTALLED and not w.startswith(("./", "$")) and w not in files and not re.match(r"^[A-Z_]+=", w) and w not in ("p", "PORT", "PID"): + if w in ("python", "pip", "pip3", "pytest", "rg", "fd", "bat", "tree", "ss", "lsof", "netstat", "nc", "bc", "column", "perl", "ruby"): + f.append(f"L1-{w}") + # L2 git in a project with no repository (and the task does not ask for git) + if re.search(r"(?:^|[;&|(]\s*|\s)git\s", cmd) and not has_git_dir and not re.search(r"\bgit\b|\brepo\b|\bcommit\b", low): + f.append("L2-git-norepo") + # L3 sh has no job control: kill %1 / fg / jobs ; pkill -f with a string that is in the command itself + if re.search(r"\bkill\s+%|\bfg\b|\bjobs\b", cmd): + f.append("L3-jobcontrol") + m = re.search(r"pkill\s+-f\s+(['\"]?)([^'\"]+)\1", cmd) + if m and m.group(2).strip() and m.group(2).strip() in cmd.replace(m.group(0), ""): + f.append("L3-pkill-self") + # L4 single-quoted command substitution / variable (never expands) + if re.search(r"'[^']*\$[({][^']*'", cmd) and not re.search(r"awk|sed|perl|python3? -c '", cmd): + f.append("L4-quoted-subst") + # L5 grep with three or more bare words after the pattern (unquoted multi-word pattern) + if re.search(r"grep(?:\s+-\S+)*\s+[A-Za-z]+(?:\s+[a-z]+){2,}\s*(?:$|[|;&)])", cmd): + f.append("L5-grep-unquoted") + # L6 test -z/-n on a double-quoted literal command + if re.search(r"test\s+-[zn]\s+\"[a-z]+ [^$\"]*\"", cmd): + f.append("L6-test-literal") + # L7 mawk / BRE regex dialect: \d anywhere in awk/grep -E/sed, {n} intervals in awk, `+` or {n} in basic grep + if re.search(r"\bawk\b[^|;]*\\d", cmd) or re.search(r"\bgrep\s+(?:-[a-zA-Z]*\s+)*(?:-e\s+)?['\"][^'\"]*\\d", cmd) or re.search(r"\bgrep\s+-[a-zA-Z]*E[a-zA-Z]*[^|;]*\\d", cmd): + f.append("L7-backslash-d") + if re.search(r"\bawk\b[^|;]*\{\d+(?:,\d*)?\}", cmd): + f.append("L7-mawk-interval") + for gm in re.finditer(r"\bgrep\s+((?:-\S+\s+)*)(['\"])(.*?)\2", cmd): + fl, pat = gm.group(1), gm.group(3) + if not re.search(r"-\w*[EP]", fl) and re.search(r"(?<=[\w\])*.])(?()*?\[\]{}$,=\\]*)", cmd): + p = am.group(1) + if p.endswith("/") or re.match(r"^/(usr|bin|sbin|opt|lib|etc|dev|proc|sys|tmp|var|private)(/|$)", p) or p in text: + continue + if re.search(r"nonexist|no[-_]such|missing", p): + continue + f.append(f"L11-abs:{p}") + break + # L12 calls a pristine python function with the wrong number of positional args + defs = pristine_defs(pristine) + for name, (req, tot) in defs.items(): + for cm in re.finditer(r"(? tot: + f.append(f"L12-arity:{name}({n})") + break + # L13 string literal passed to a pristine function that appears nowhere in task text or pristine tree + for name in defs: + if re.search(r"Error|raise|except|-q 'Traceback", cmd): + break + for cm in re.finditer(r"(?= 2 compared/grepped for that is in neither the task text nor the pristine inputs + nums = re.findall(r"(?:-eq|==|-ne|!=|grep -q[a-z]*\s+'\^?|grep -qx\s+'?)\s*(\d{1,6})(?![\w.])", cmd) + nums += re.findall(r"len\([^)]*\)\s*==\s*(\d+)", cmd) + for n in nums: + if int(n) >= 2 and not re.search(r"(? IMAGE.has(n), + hasPyModule: () => false, + awkIntervals: () => false, +}; +const shell = process.argv[2] === "bash" ? "bash" : "sh"; +const items = JSON.parse(readFileSync(0, "utf8")); +const trees = new Map(); +const out = items.map((it) => { + if (!trees.has(it.pristine)) trees.set(it.pristine, readTree(it.pristine)); + return lintAll(it.cmd, { cwd: it.pristine, tree: trees.get(it.pristine), task: it.text, shell, probes }).map((h) => `${h.rule.includes(":") ? h.rule : h.rule}`); +}); +process.stdout.write(JSON.stringify(out)); diff --git a/bench/local/checkquality_lintts.py b/bench/local/checkquality_lintts.py new file mode 100644 index 0000000..abaca3c --- /dev/null +++ b/bench/local/checkquality_lintts.py @@ -0,0 +1,70 @@ +"""Check-quality replay with the SHIPPED lint (src/checklint.ts) instead of checkquality_lint.py. + + npm run build # dist/checklint.js + python3 checkquality_lintts.py SCRATCH_DIR [sh|bash] + +Writes SCRATCH_DIR/ts-/ (a copy of the inputs with cq-checks-lint.json's `flags` +replaced by the TypeScript lint's) and prints per-rule counts by group; then run + python3 checkquality_analysis.py SCRATCH_DIR/ts- +for the run-level recall/precision table. +""" +from __future__ import annotations + +import json +import shutil +import subprocess +import sys +from collections import Counter +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent)) +from tasks import TASKS # noqa: E402 +from tasks2 import TASKS2 # noqa: E402 + +S = Path(sys.argv[1]) +SHELL = sys.argv[2] if len(sys.argv) > 2 else "sh" +TREES = Path.home() / ".cache/maat-bench/cq-trees" +HERE = Path(__file__).parent +PROMPT = {T.name: T.PROMPT for T in [*TASKS, *TASKS2]} + +checks = json.load(open(S / "cq-checks-lint.json")) +tax = {(x["id"], x["name"]): x["cls"] for x in json.load(open(S / "cq-taxonomy.json"))} +items = [{"task": c["task"], "cmd": c["cmd"], "pristine": str(TREES / c["task"] / "pristine"), "text": PROMPT[c["task"]]} for c in checks] +r = subprocess.run(["node", str(HERE / "checkquality_lintts.mjs"), SHELL], input=json.dumps(items), capture_output=True, text=True, check=True) +flags = json.loads(r.stdout) + +out = S / f"ts-{SHELL}" +out.mkdir(exist_ok=True) +for f in S.iterdir(): + if f.is_file() and f.name not in ("cq-checks-lint.json",): + dst = out / f.name + if not dst.exists(): + try: + dst.symlink_to(f) + except OSError: + shutil.copy(f, dst) +for c, fl in zip(checks, flags): + c["flags_py"] = c["flags"] + c["flags"] = fl +json.dump(checks, open(out / "cq-checks-lint.json", "w")) + + +def group(c): + if c["ok"]: + return "pass-on-correct" if c["passed"] else "pass-on-wrong" + return "refused-correct" if c["passed"] else "caught-wrong" + + +rules = Counter() +for c in checks: + g = group(c) + for rule in {f.split(":")[0] for f in c["flags"]}: + rules[(rule, g)] += 1 +groups = ["refused-correct", "caught-wrong", "pass-on-correct", "pass-on-wrong"] +print(f"shell={SHELL} rule | " + " | ".join(groups)) +for rule in sorted({k[0] for k in rules}): + print(f"{rule:22} | " + " | ".join(str(rules[(rule, g)]) for g in groups)) +for label, key in (("python (report)", "flags_py"), ("typescript (shipped)", "flags")): + n = {g: sum(1 for c in checks if group(c) == g and c[key]) for g in groups} + tot = {g: sum(1 for c in checks if group(c) == g) for g in groups} + print(f"{label:22} flagged: " + ", ".join(f"{g} {n[g]}/{tot[g]}" for g in groups)) diff --git a/bench/local/checkquality_load.py b/bench/local/checkquality_load.py new file mode 100644 index 0000000..9a3b643 --- /dev/null +++ b/bench/local/checkquality_load.py @@ -0,0 +1,136 @@ +"""Check-quality replay, step 1: one record per run across the v10/v11/v12 lanes. + +Reads container-results/results-.jsonl (grader labels), each run's +-molt-.log (engine events: seal, drafting cut, proof results with +output, retirements) and the workspace's .maat/log journal (bar_run events, +timestamps). No model, no network. + + python3 checkquality_load.py OUT.json +""" +from __future__ import annotations + +import json +import re +import sys +from datetime import datetime +from pathlib import Path + +CACHE = Path.home() / ".cache/maat-bench" +RESULTS = CACHE / "container-results" +CW = CACHE / "container-work" +LANES = ["v10-old", "v10-fix", "v11-old", "v11-new", "v12-old", "v12-new"] + + +def events(path: Path) -> list[dict]: + out = [] + for line in path.read_text(errors="replace").splitlines(): + try: + out.append(json.loads(line)) + except Exception: + pass + return out + + +def journal(run: Path) -> list[dict]: + ev: list[dict] = [] + logd = run / ".maat/log" + for f in sorted(logd.glob("*.jsonl")) if logd.is_dir() else []: + ev += events(f) + ev.sort(key=lambda e: (e.get("iso", ""), e.get("seq", 0))) + return ev + + +def iso(s: str) -> float: + return datetime.fromisoformat(s.replace("Z", "+00:00")).timestamp() + + +def load_lane(lane: str) -> list[dict]: + rows = {} + for line in (RESULTS / f"results-{lane}.jsonl").read_text().splitlines(): + r = json.loads(line) + rows[(r["task"], r["rep"])] = r + recs = [] + for shard in sorted(CW.glob(f"{lane}-s*")): + for log in sorted(shard.glob("*-molt-*.log")): + m = re.match(r"^(.+)-molt-(\d+)\.log$", log.name) + task, rep = m[1], int(m[2]) + row = rows.get((task, rep)) + if not row: + continue + ws = shard / f"{task}-molt-{rep}" + ev = events(log) + rec = { + "lane": lane, "task": task, "rep": rep, "id": f"{lane}/{task}#{rep}", "ws": str(ws), + "passed": bool(row["passed"]), "why": row.get("why", ""), "claim": row.get("claim"), + "checks_disagree": row.get("checks_disagree") or [], "tier": row.get("tier"), + "secs": row.get("secs"), "timed_out": row.get("timed_out"), "review": row.get("review"), + "said_done": row.get("said_done"), "steps": row.get("turns"), + "sealed_n": None, "notes_n": None, "cut": None, "late_join": 0, "late_join_names": [], + "seal_s": None, "proofs": [], "retired": [], "infos": [], "outcome": None, "job_end": None, + "preflight_broken": [], "passed_before_work": [], + } + for e in ev: + k = e.get("kind") + if k == "info": + t = e.get("text", "") + m2 = re.match(r"^(\d+) task check\(s\) and (\d+) note\(s\) sealed", t) + if m2: + rec["sealed_n"] = int(m2[1]); rec["notes_n"] = int(m2[2]) + if "checks were still being drafted after" in t: + rec["cut"] = "none-ready" if "none was ready" in t else "partial" + m3 = re.match(r"^(\d+) drafted check\(s\) joined this turn", t) + if m3: + rec["late_join"] += int(m3[1]) + m4 = re.match(r"^check (\S+) (?:timed out|failed in its own code, not on the work \((.*?)\))", t) + if m4: + rec["retired"].append({"name": m4[1], "why": m4[2] or "timeout"}) + if t.startswith("criterion `"): + rec["preflight_broken"] += re.findall(r"criterion `([^`]+)` did not run", t) + rec["infos"].append(t[:160]) + elif k in ("proof_refused", "proof_exhausted", "proof_passed", "proof_ok", "proof_result"): + res = e.get("result") or {} + rec["proofs"].append({ + "kind": k, "attempt": e.get("attempt") or e.get("attempts"), "ok": res.get("ok"), + "results": [ + {"name": r.get("name"), "detail": r.get("detail"), "ok": r.get("ok"), "exitCode": r.get("exitCode"), + "output": (r.get("output") or "")[-1500:], "tags": r.get("tags") or [], "hidden": r.get("hidden"), + "timedOut": r.get("timedOut"), "advisory": r.get("advisory"), "skipped": r.get("skipped"), "kind": r.get("kind")} + for r in res.get("results") or [] + ], + }) + elif k == "job_end": + rec["outcome"] = e.get("outcome") + rec["job_end"] = {kk: e.get(kk) for kk in ("steps", "durationMs", "outcome", "selfChecked", "checksDisagree", "tier", "turnEndedBy")} + jl = journal(ws) + start = next((e["iso"] for e in jl if e.get("kind") == "session_start"), None) + seal = next((e for e in jl if e.get("kind") == "note" and str(e.get("data", {}).get("text", "")).startswith("task criteria sealed")), None) + if start and seal: + rec["seal_s"] = round(iso(seal["iso"]) - iso(start), 1) + rec["sealed_names"] = seal["data"].get("checks", []) + late = [e for e in jl if e.get("kind") == "note" and e.get("data", {}).get("kind") == "late-checks"] + for e in late: + rec["late_join_names"] += e["data"].get("checks") or [] + bars = [e for e in jl if e.get("kind") == "bar_run"] + rec["bar_runs"] = [ + {"iso": e["iso"], "ok": e["data"].get("ok"), "checks": [{"name": c["name"], "detail": c.get("detail"), "ok": c.get("ok"), "exitCode": c.get("exitCode")} for c in e["data"].get("checks", [])]} + for e in bars + ] + pbw = [e for e in jl if e.get("kind") == "note" and "already pass" in str(e.get("data", {}).get("text", ""))] + rec["passed_before_work"] = [str(e["data"].get("text"))[:200] for e in pbw] + rec["journal_start"] = start + rec["journal_end"] = next((e["iso"] for e in reversed(jl) if e.get("kind") == "session_end"), None) + recs.append(rec) + return recs + + +def main() -> None: + out = [] + for lane in LANES: + recs = load_lane(lane) + print(lane, len(recs), "runs;", sum(r["passed"] for r in recs), "passed", file=sys.stderr) + out += recs + Path(sys.argv[1]).write_text(json.dumps(out)) + + +if __name__ == "__main__": + main() diff --git a/bench/local/checkquality_run.py b/bench/local/checkquality_run.py new file mode 100644 index 0000000..dbebfd5 --- /dev/null +++ b/bench/local/checkquality_run.py @@ -0,0 +1,100 @@ +"""Check-quality replay, step 3: run (command, tree) jobs inside the bench image. + +Every job copies its tree to a scratch dir inside the container and runs the +command with `sh -c`, as the bar does (node spawn shell:true -> /bin/sh), with +a 120 s timeout. Tasks run in parallel; jobs within one task run one at a time +(server checks bind fixed ports). Output: one JSON line per job. + + python3 checkquality_run.py JOBS.jsonl OUT.jsonl [--image maat-bench:9ba6b27e] [--workers 6] + +A job: {"id": ..., "task": ..., "cmd": ..., "tree": ""}. +Trees may be under ~/.cache/maat-bench (container-work, cq-trees); that folder +is mounted read-only at /mnt. +""" +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from pathlib import Path + +CACHE = Path.home() / ".cache/maat-bench" + +DRIVER = r''' +import json, os, shutil, subprocess, sys, tempfile, time +from concurrent.futures import ThreadPoolExecutor +jobs = [json.loads(l) for l in open("/jobs.jsonl") if l.strip()] +bytask = {} +for j in jobs: + bytask.setdefault(j["task"], []).append(j) +out = open("/out.jsonl", "a") +import threading +lock = threading.Lock() +def run_one(j): + tmp = tempfile.mkdtemp(prefix="cq-", dir="/scratch") + w = os.path.join(tmp, "w") + src = j["tree"] + try: + subprocess.run(["cp", "-a", src, w], check=True) + t0 = time.time() + outf = open(os.path.join(tmp, "out.txt"), "w+", errors="replace") + p = subprocess.Popen(["/bin/sh", "-c", j["cmd"]], cwd=w, stdout=outf, stderr=subprocess.STDOUT, + stdin=subprocess.DEVNULL, preexec_fn=os.setsid, + env={"PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", "HOME": "/root", "LANG": "C.UTF-8", "TMPDIR": tmp}) + try: + code = p.wait(timeout=30); to = False + except subprocess.TimeoutExpired: + code, to = 124, True + time.sleep(0.2) + try: os.killpg(p.pid, 9) + except Exception: pass + outf.flush(); outf.seek(0); o = outf.read(); outf.close() + res = {"id": j["id"], "tree": j.get("treeid", src), "exit": code, "out": o[-2000:], "timedOut": to, "ms": int((time.time() - t0) * 1000)} + except Exception as e: + res = {"id": j["id"], "tree": j.get("treeid", src), "exit": -1, "out": "harness: " + str(e)[:300], "timedOut": False, "ms": 0} + finally: + shutil.rmtree(tmp, ignore_errors=True) + with lock: + out.write(json.dumps(res) + "\n"); out.flush() +def run_task(t): + for j in bytask[t]: + run_one(j) +with ThreadPoolExecutor(max_workers=int(sys.argv[1])) as ex: + list(ex.map(run_task, sorted(bytask, key=lambda t: -len(bytask[t])))) +''' + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("jobs") + ap.add_argument("out") + ap.add_argument("--image", default="maat-bench:9ba6b27e") + ap.add_argument("--workers", type=int, default=6) + a = ap.parse_args() + jobs = [json.loads(l) for l in open(a.jobs) if l.strip()] + # rewrite Mac paths to the mount + mapped = [] + for j in jobs: + t = j["tree"] + if not t.startswith(str(CACHE)): + sys.exit(f"tree outside the cache mount: {t}") + mapped.append({**j, "treeid": j.get("treeid", t), "tree": "/mnt" + t[len(str(CACHE)):]}) + tmp = Path(a.out).with_suffix(".jobs.tmp") + tmp.write_text("".join(json.dumps(j) + "\n" for j in mapped)) + drv = Path(a.out).with_suffix(".driver.py") + drv.write_text(DRIVER) + Path(a.out).touch() + cmd = [ + "docker", "run", "--rm", "--network", "none", + "-v", f"{CACHE}:/mnt:ro", "-v", f"{tmp.resolve()}:/jobs.jsonl:ro", "-v", f"{drv.resolve()}:/driver.py:ro", + "-v", f"{Path(a.out).resolve()}:/out.jsonl", "--tmpfs", "/scratch:exec,size=2g", + a.image, "python3", "/driver.py", str(a.workers), + ] + print(" ".join(cmd), file=sys.stderr) + r = subprocess.run(cmd) + print("docker exit", r.returncode, file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/bench/local/checkquality_taxonomy.py b/bench/local/checkquality_taxonomy.py new file mode 100644 index 0000000..ccc5cce --- /dev/null +++ b/bench/local/checkquality_taxonomy.py @@ -0,0 +1,126 @@ +"""Check-quality replay: hand classification of every drafted check that failed +at the final bar on grader-passed work (v10-old/fix, v11-old/new, v12-old/new). + +Each (run id, check name) was read with its command and recorded output +(scratchpad refused-correct.txt). Classes: + + GV guessed value: a hand-computed number or an invented literal/case/message + GI guessed interface: an invented path, file, argument, signature, input or module + SB shell/tool bug in the check's own code (dash job control, pkill -f self-kill, + quoting, mawk/BRE dialect, precedence, chained greps on consumed stdin) + WL wrong logic: tests a state correct work cannot produce, or misreads the spec + OS over-strict: a format stricter than the task states + ET environment: tool missing (pytest) + EA environment assumption (git HEAD in a non-repo, .maat/ counted, `time` in sh) + PL product lint: the check PASSED (exit 0) but evidence.ts refused its shape + + python3 checkquality_taxonomy.py CQ_CHECKS.json +""" +from __future__ import annotations + +import json +import re +import sys +from collections import Counter, defaultdict + +# rules in order; first match wins. (task regex, name regex, cmd regex) -> class +RULES: list[tuple[str, str, str, str]] = [ + # product-lint refusals of passing checks + (".", ".", r"\|\| exit 0\s*$", "PL"), + ("duration-bug", "run-tests", r"unittest.*\| grep -q", "PL"), + # environment + (".", ".", r"-m pytest", "ET"), + (".", ".", r"git diff --quiet HEAD --", "EA"), + ("organize-files", "sorted-only", r"ls -A", "EA"), + ("perf-pairs", ".", r"^time python3", "EA"), + # shell / tool bugs + ("http-json-server", ".", r"kill %1|pkill -f|^p=python3", "SB"), + ("makefile-site", ".", r"grep -q nothing to be done|\$\{f#src/\}", "SB"), + ("git-revert-one", ".", r"test -z \"git status|'\^\$\(git rev-parse", "SB"), + ("sql-top-customers", ".", r"awk", "SB"), + ("node-summarize", ".", r"grep -vE '\^-\?\\d", "SB"), + ("csv-clean", ".", r"grep -q '\^\[\^@\]\+@|or exit\(1\)", "SB"), + ("cron-next", "run-cron-2", r".", "SB"), + ("access-report", "validate-top-5", r".", "SB"), + ("config-migrate", "migrate_v1_configs", r".", "SB"), + # wrong logic + ("fix-git", ".", r"git merge --no-commit|git diff master\.\.HEAD|git diff HEAD -- about\.md", "WL"), + ("git-revert-one", ".", r"git diff --quiet HEAD~1 HEAD", "WL"), + ("sql-top-customers", ".", r"grep -qv", "WL"), + ("config-migrate", "migrate-v1-files|verify_port_default|validate-v2-schema|port-defaults-to-5432", r".", "WL"), + ("csv-clean", "validate-age-rules", r".", "WL"), + ("access-report", "path-count-format|check-report-format", r".", "WL"), + ("log-summary", "period-order", r".", "WL"), + # over-strict format + ("access-report", "verify-format", r".", "OS"), + # guessed interface + ("perf-pairs", ".", r".", "GI"), + ("refactor-pricing", ".", r".", "GI"), + ("config-migrate", ".", r"sample_v1|test\.json|example\.json|cd /app|migrate\.py", "GI"), + ("http-json-server", ".", r"/app/|/workspace/", "GI"), + ("fix-git", ".", r"about\.html|origin/master", "GI"), + ("makefile-site", ".", r"build/(NAME|example|README|test)\.html", "GI"), + ("access-report", "generate-report", r"task\.py", "GI"), + ("git-revert-one", ".", r"git revert \"Add", "GI"), + # guessed value + ("wc-tool", ".", r".", "GV"), + ("bash-rotate", ".", r"grep -q 'usage'", "GV"), + ("fix-git", ".", r"grep -q '(modified|changed|changes|merge)'", "GV"), + ("git-revert-one", ".", r"wc -l \| grep -q '\^[13]\$'|grep -q 'revert'", "GV"), + ("makefile-site", ".", r"nothing to be done|Nothing to do|\[0-9\]\+ files", "GV"), + ("size-parse-bug", ".", r"== 1024", "GV"), + ("csv-clean", ".", r"len\(rows\)==24|len\(d\)==12|local@domain\.com", "GV"), + ("log-summary", ".", r".", "GV"), + ("organize-files", ".", r"-eq 5|grep -qx 1", "GV"), + ("cron-next", ".", r".", "GV"), +] + + +def classify(task: str, name: str, cmd: str) -> str: + n = name.replace("task:", "") + for t, nm, c, cls in RULES: + if re.search(t, task) and re.search(nm, n) and re.search(c, cmd, re.S): + return cls + return "??" + + +def main() -> None: + rows = json.load(open(sys.argv[1])) + rc = [x for x in rows if x["passed"] and x["ok"] is False] + cnt: Counter = Counter() + per_task: dict = defaultdict(Counter) + per_lane: dict = defaultdict(Counter) + unk = [] + for x in rc: + cls = classify(x["task"], x["name"], x["cmd"]) + x["cls"] = cls + cnt[cls] += 1 + per_task[x["task"]][cls] += 1 + per_lane[x["lane"]][cls] += 1 + if cls == "??": + unk.append(x) + print(f"refused-correct checks: {len(rc)} in {len(set(x['id'] for x in rc))} runs") + for k, v in cnt.most_common(): + print(f" {k}: {v} ({100*v/len(rc):.0f}%)") + print("\nby lane:") + for lane in ["v10-old", "v10-fix", "v11-old", "v11-new", "v12-old", "v12-new"]: + c = per_lane[lane] + print(f" {lane:8} n={sum(c.values()):3} " + " ".join(f"{k}={c[k]}" for k in ["GV", "GI", "SB", "WL", "OS", "ET", "EA", "PL"])) + print("\nby task:") + for t, c in sorted(per_task.items(), key=lambda kv: -sum(kv[1].values())): + print(f" {t:18} n={sum(c.values()):3} " + " ".join(f"{k}={v}" for k, v in c.most_common())) + # run-level: the class of the check(s) that refused each correct run + runs: dict = defaultdict(set) + for x in rc: + runs[x["id"]].add(x["cls"]) + print("\nruns refused only by one class:") + rc_cnt = Counter("+".join(sorted(s)) for s in runs.values()) + for k, v in rc_cnt.most_common(): + print(f" {k}: {v}") + for x in unk: + print("UNCLASSIFIED", x["id"], x["name"], x["cmd"][:120]) + json.dump([{k: x[k] for k in ("id", "lane", "task", "name", "cmd", "cls")} for x in rc], open(sys.argv[1].replace("cq-checks", "cq-taxonomy"), "w"), indent=1) + + +if __name__ == "__main__": + main() diff --git a/bench/local/checkquality_tiers.py b/bench/local/checkquality_tiers.py new file mode 100644 index 0000000..f9c7cf2 --- /dev/null +++ b/bench/local/checkquality_tiers.py @@ -0,0 +1,128 @@ +"""Check-quality replay, step 5: what positive evidence separates right from wrong +once buggy checks are retired; the review gate; the drafting budget. + + python3 checkquality_tiers.py SCRATCH_DIR +""" +from __future__ import annotations + +import json +import sys +from collections import Counter, defaultdict +from pathlib import Path + +S = Path(sys.argv[1]) +R = json.load(open(S / "cq-runs.json")) +C = json.load(open(S / "cq-checks-final.json")) +LANES = ["v10-old", "v10-fix", "v11-old", "v11-new", "v12-old", "v12-new"] +NOISY = {"L14-number", "L10-path"} +by_run = defaultdict(list) +for x in C: + by_run[x["id"]].append(x) + + +def rule(f): + return f.split(":")[0] + + +def clean(x): + return not any(rule(f) not in NOISY for f in x["flags"]) + + +def own(x): + return x["own"] if x["own"] is not None else x["ok"] + + +def keep_f8(x): + return clean(x) and x["pristine"] is not True and not (x["oc_n"] - x["oc_pass"] >= 2) + + +def keep_f3p(x): # product-implementable today: lint + pristine (no cross-tree) + return clean(x) and x["pristine"] is not True + + +def keep_f0(x): + return True + + +def discriminating(x): # oracle feature: fails on >=50% of the OTHER wrong trees (not available to the product) + return x["ow_n"] >= 3 and (x["ow_n"] - x["ow_pass"]) * 2 >= x["ow_n"] + + +def strongest(live_pass): + if any("value" in x["tags"] and "surface" not in x["tags"] for x in live_pass): + return "runs+value" + if any("surface" not in x["tags"] for x in live_pass): + return "runs" + return "surface" + + +def run_table(title, keep, lanes): + print(f"\n## {title}") + print("evidence of strongest passing live check | lane | accepted correct | accepted WRONG | precision") + for lane in lanes: + rows = defaultdict(lambda: [0, 0]) + for r in R: + if lane != "ALL" and not r["lane"].startswith(lane): + continue + xs = [x for x in by_run.get(r["id"], []) if keep(x)] + if not xs or any(own(x) is False for x in xs): + continue + ev = strongest(xs) + rows[ev][0 if r["passed"] else 1] += 1 + # extra: value AND failed on pristine AND (oracle) discriminating + if any("value" in x["tags"] and "surface" not in x["tags"] and x["pristine"] is False for x in xs): + rows["runs+value & failed-on-pristine"][0 if r["passed"] else 1] += 1 + if any(discriminating(x) for x in xs): + rows["ORACLE: >=1 check that fails most wrong trees"][0 if r["passed"] else 1] += 1 + if any(discriminating(x) and "value" in x["tags"] for x in xs): + rows["ORACLE: discriminating & value"][0 if r["passed"] else 1] += 1 + for ev in ["runs+value", "runs", "surface", "runs+value & failed-on-pristine", "ORACLE: >=1 check that fails most wrong trees", "ORACLE: discriminating & value"]: + c, w = rows[ev] + print(f" {ev:46} | {lane:8} | {c:3} | {w:3} | {c/(c+w) if c+w else float('nan'):.2f}") + + +run_table("F0 current bar (no retirement): precision by evidence tier of accepted runs", keep_f0, ["v12-new", "v12", "ALL"]) +run_table("F3+pristine (lint at seal + pristine-pass advisory; implementable today)", keep_f3p, ["v12-new", "v12", "ALL"]) +run_table("F8 (lint + pristine + cross-tree>=2, needs a second independent work)", keep_f8, ["v12-new", "v12", "ALL"]) + +print("\n## checks that PASS on their own tree: does the 'value' tag or pristine-failure separate correct from wrong work? (per check)") +for name, pred in (("value tag", lambda x: "value" in x["tags"]), ("failed on pristine", lambda x: x["pristine"] is False), ("value & failed on pristine", lambda x: "value" in x["tags"] and x["pristine"] is False), + ("ORACLE discriminating", discriminating), ("value & ORACLE discriminating", lambda x: "value" in x["tags"] and discriminating(x))): + pc = [x for x in C if x["passed"] and own(x)] + pw = [x for x in C if not x["passed"] and own(x)] + print(f" {name:30} pass-on-correct {sum(map(pred, pc))}/{len(pc)} pass-on-wrong {sum(map(pred, pw))}/{len(pw)}") + +print("\n## the review gate, runs whose final bar passed and that were reviewed (recorded): votes x grader") +c = Counter() +for r in R: + if r["proofs"] and r["proofs"][-1]["ok"] and r["review"]: + c[(r["review"]["votes"], r["passed"])] += 1 +for v in ["0/3", "1/3", "2/3", "3/3"]: + print(f" {v}: correct {c[(v, True)]} wrong {c[(v, False)]} precision-if-accepted {c[(v, True)]/(c[(v, True)]+c[(v, False)]) if c[(v, True)]+c[(v, False)] else float('nan'):.2f}") +print(f" gate '0/3 only' keeps {c[('0/3', True)]} correct of {sum(v for (k, p), v in c.items() if p)}, blocks {c[('0/3', False)]} wrong of {sum(v for (k, p), v in c.items() if not p)}") + +print("\n## drafting budget: sealed checks vs the 54 s cut (all six lanes)") +for lane in LANES: + rs = [r for r in R if r["lane"] == lane] + cut = [r for r in rs if r["cut"]] + nocut = [r for r in rs if not r["cut"]] + def mean_sealed(xs): + v = [r["sealed_n"] or 0 for r in xs] + return sum(v) / len(v) if v else 0 + def barpass(xs): + c = [r for r in xs if r["passed"]] + return f"{sum(1 for r in c if r['proofs'] and r['proofs'][-1]['ok'])}/{len(c)}" + print(f" {lane:8} cut fired {len(cut):2}/{len(rs)} mean sealed: cut {mean_sealed(cut):.1f} vs no-cut {mean_sealed(nocut):.1f} correct runs bar-passed: cut {barpass(cut)} no-cut {barpass(nocut)} seal_s>=54 {sum(1 for r in rs if (r['seal_s'] or 0) >= 53.5)}") +print("\n v12-new 'none-ready' runs (0 checks at the cut):") +for r in R: + if r["lane"] == "v12-new" and r["cut"] == "none-ready": + print(f" {r['id']:26} grader={'pass' if r['passed'] else 'FAIL'} late-joined {r['late_join']} claim={r['claim']} seal_s={r['seal_s']}") +print("\n correct runs by number of sealed checks (all lanes): bar-pass rate") +by = defaultdict(lambda: [0, 0]) +for r in R: + if r["passed"]: + n = r["sealed_n"] if r["sealed_n"] is not None else 0 + by[min(n, 4)][1] += 1 + by[min(n, 4)][0] += bool(r["proofs"] and r["proofs"][-1]["ok"]) +for n in sorted(by): + print(f" sealed {n}: bar-passed {by[n][0]}/{by[n][1]}") diff --git a/bench/local/checkquality_trees.py b/bench/local/checkquality_trees.py new file mode 100644 index 0000000..8d5ea5b --- /dev/null +++ b/bench/local/checkquality_trees.py @@ -0,0 +1,121 @@ +"""Check-quality replay, step 2: pristine and reference trees for all 20 tasks. + +refs2.py has the 15 tasks2 solvers; the five tasks.py solvers are here. Every +reference tree is graded before it is kept (it must pass; pristine must fail). + + python3 checkquality_trees.py OUT_DIR +""" +from __future__ import annotations + +import shutil +import subprocess +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import refs2 # noqa: E402 +from tasks import TASKS # noqa: E402 +from tasks2 import TASKS2 # noqa: E402 + + +def sh(cmd: str, cwd: Path) -> str: + return subprocess.run(cmd, shell=True, cwd=cwd, capture_output=True, text=True).stdout + + +def solve_fix_git(d: Path) -> None: + lost = sh("git log -g --format=%H --grep='Move to Stanford' HEAD | head -1", d).strip() + assert lost + sh(f"git checkout -q master && git merge -q --no-edit {lost}", d) + + +def solve_log_summary(d: Path) -> None: + import csv + import datetime as dt + import re + day0 = dt.date(2025, 8, 12) + counts: dict = {} + for f in sorted((d / "logs").glob("*.log")): + day = dt.date.fromisoformat(f.stem) + for line in f.read_text().splitlines(): + m = re.search(r"\[(ERROR|WARNING|INFO)\]", line) + if not m: + continue + for period, ok in (("today", day == day0), ("last_7_days", day0 - dt.timedelta(days=6) <= day <= day0), ("total", True)): + if ok: + counts[(period, m.group(1))] = counts.get((period, m.group(1)), 0) + 1 + with open(d / "summary.csv", "w", newline="") as fh: + w = csv.writer(fh) + w.writerow(["period", "severity", "count"]) + for p in ("today", "last_7_days", "total"): + for s in ("ERROR", "WARNING", "INFO"): + w.writerow([p, s, counts.get((p, s), 0)]) + + +def solve_regex(d: Path) -> None: + (d / "regex.txt").write_text(r"(? None: + (d / "dur.py").write_text( + 'import re\n\n\ndef parse_duration(s):\n' + ' """Seconds in a duration string: "90s", "5m", "1h30m", "2h", "1h5m10s",\n' + ' and a bare number means seconds ("45"). Whitespace around it is ignored."""\n' + ' s = s.strip()\n' + ' if s.isdigit():\n return int(s)\n' + ' total = 0\n pos = 0\n' + ' for m in re.finditer(r"(\\d+)([hms])", s):\n' + ' if m.start() != pos:\n raise ValueError(s)\n' + ' pos = m.end()\n' + ' total += int(m.group(1)) * {"h": 3600, "m": 60, "s": 1}[m.group(2)]\n' + ' if pos != len(s) or pos == 0:\n raise ValueError(s)\n' + ' return total\n' + ) + + +def solve_wc_tool(d: Path) -> None: + (d / "wc.py").write_text( + 'import sys\n\n\ndef counts(data: bytes):\n' + ' return data.count(b"\\n"), len(data.split()), len(data)\n\n\n' + 'def main(argv):\n' + ' files = argv[1:]\n' + ' if not files:\n' + ' l, w, b = counts(sys.stdin.buffer.read())\n' + ' print(f"{l} {w} {b}")\n return\n' + ' tl = tw = tb = 0\n' + ' for f in files:\n' + ' with open(f, "rb") as fh:\n l, w, b = counts(fh.read())\n' + ' tl += l; tw += w; tb += b\n' + ' print(f"{l} {w} {b} {f}")\n' + ' if len(files) > 1:\n print(f"{tl} {tw} {tb} total")\n\n\n' + 'if __name__ == "__main__":\n main(sys.argv)\n' + ) + + +LOCAL = {"fix-git": solve_fix_git, "log-summary": solve_log_summary, "regex": solve_regex, + "duration-bug": solve_duration_bug, "wc-tool": solve_wc_tool} + + +def main() -> None: + out = Path(sys.argv[1]) + bad = 0 + for T in TASKS + TASKS2: + for kind in ("pristine", "ref"): + d = out / T.name / kind + if d.exists(): + shutil.rmtree(d) + d.mkdir(parents=True) + T.setup(d) + if kind == "ref": + solver = LOCAL.get(T.name) or getattr(refs2, "solve_" + T.name.replace("-", "_")) + solver(d) + ok, why = T.grade(d) + want = kind == "ref" + flag = "ok " if ok == want else "BAD" + bad += ok != want + print(f"{flag} {T.name:20} {kind:8} grader={ok} {why[:60]}") + print("problems:", bad) + + +if __name__ == "__main__": + main() diff --git a/bench/local/container/Dockerfile b/bench/local/container/Dockerfile new file mode 100644 index 0000000..2816c8f --- /dev/null +++ b/bench/local/container/Dockerfile @@ -0,0 +1,17 @@ +# The local benchmark in a Linux container: Maat runs model-written commands +# unattended (--yes), so the machine it may damage is this one, not the Mac, +# and the tools are GNU ones, as on Terminal-Bench and most servers. +FROM node:22-bookworm-slim +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3 git sqlite3 jq make curl ca-certificates procps file xxd \ + && rm -rf /var/lib/apt/lists/* \ + && git config --system user.name "bench" && git config --system user.email "bench@localhost" \ + && git config --system init.defaultBranch master +# The agent under test runs as this user; run-in-container.sh keeps the graders and the +# reference solutions in root-only places it cannot read. +RUN useradd -m -u 1001 -s /bin/bash agent && chmod 700 /root +ENV BENCH_AGENT_USER=agent +WORKDIR /opt +# Maat itself is installed per build by run-in-container.sh (FROM maat-bench + npm i -g the packed tarball). + +ENV BENCH_WORK=/work RESULTS_DIR=/results diff --git a/bench/local/container/Dockerfile.grok b/bench/local/container/Dockerfile.grok new file mode 100644 index 0000000..8f7726e --- /dev/null +++ b/bench/local/container/Dockerfile.grok @@ -0,0 +1,9 @@ +# maat-bench plus the Grok Build CLI (Linux arm64/x86_64), for the grok-build://subscription lane. +# Installed with the official script (https://x.ai/cli/install.sh), pinned to the version the +# owner's Mac runs so ACP behaviour matches. No credentials are baked in: run-in-container.sh +# copies them in at start (SUBSCRIPTION=grok). +FROM maat-bench:agentu +ARG GROK_VERSION=1.0.46 +RUN curl -fsSL https://x.ai/cli/install.sh | HOME=/opt/grok-install GROK_BIN_DIR=/opt/grok-install/.grok/bin bash -s ${GROK_VERSION} \ + && cp -L "$(readlink -f /opt/grok-install/.grok/bin/grok)" /tmp/grok.bin && rm -f /usr/local/bin/grok /usr/local/bin/agent && mv /tmp/grok.bin /usr/local/bin/grok && rm -rf /opt/grok-install \ + && grok --version diff --git a/bench/local/container/Dockerfile.opencode b/bench/local/container/Dockerfile.opencode new file mode 100644 index 0000000..f6cfbfd --- /dev/null +++ b/bench/local/container/Dockerfile.opencode @@ -0,0 +1,9 @@ +# Any maat-bench image plus the OpenCode CLI (npm `opencode-ai`, linux arm64/x86_64), for the +# opencode://subscription judge (MAAT_JUDGE_URL). Built on top of the plain or the grok image +# (run-in-container.sh passes BASE), so OPENCODE=1 combines with SUBSCRIPTION=grok. Pinned to the +# version the owner's Mac runs. No credentials are baked in: run-in-container.sh copies +# auth.json in at start. +ARG BASE=maat-bench:agentu +FROM ${BASE} +ARG OPENCODE_VERSION=1.18.33 +RUN npm install -g --no-fund --no-audit opencode-ai@${OPENCODE_VERSION} && opencode --version diff --git a/bench/local/container/run-in-container.sh b/bench/local/container/run-in-container.sh new file mode 100755 index 0000000..8ad1b1c --- /dev/null +++ b/bench/local/container/run-in-container.sh @@ -0,0 +1,83 @@ +#!/bin/zsh +# Run bench/local/run.py inside a throwaway Linux container, against a packed Maat. +# container/run-in-container.sh -- +# Make the tarball with: npm run pack:cli && npm pack ./out-cli +# The image is maat-bench:, built once and reused, so every run +# records exactly which Maat it tested. Env passed through: REFERENCE BENCH_MODEL +# BENCH_URL BENCH_LIMIT RESULTS (file name). Results land in +# ~/.cache/maat-bench/container-results/; each task's folder and Maat's log are kept +# in ~/.cache/maat-bench/container-work// (the only host path it can write). +# SUBSCRIPTION=grok: use the owner's Grok Build login (BENCH_URL=grok-build://subscription +# BENCH_MODEL=grok-4.7). Image is maat-bench-grok: (Dockerfile.grok: official grok +# installer, linux, pinned). ~/.grok/auth.json is mounted read-only OUTSIDE the container's +# HOME and copied to $HOME/.grok/auth.json at start (mode 600), so the Mac's file is never +# modified; nothing else from ~/.grok is used. GROK_OWN_TOOLS=1 adds a generated +# permission_mode=always-approve config (see below) so Grok works with its own tools. No OpenRouter key is passed in this mode. +# OPENCODE=1 (combines with SUBSCRIPTION=grok): add the OpenCode CLI (Dockerfile.opencode, +# image tag suffix -oc) and ~/.local/share/opencode/auth.json, mounted read-only outside HOME and +# copied to /home/agent/.local/share/opencode/auth.json (mode 600). It is for the judge only: +# ARMS="oc:MAAT_JUDGE_MODEL=opencode/big-pickle,MAAT_JUDGE_URL=opencode://subscription" +# The key is never printed. +set -e +name=$1; tgz=${2:A}; shift 2; [ "$1" = "--" ] && shift +here=${0:A:h}; local_dir=${here:h} +# Image tags carry ":agentu": images with the unprivileged `agent` user (older ones lack it). +sha=$(shasum -a 256 "$tgz" | cut -c1-8); base=maat-bench:agentu; img=maat-bench:$sha-u +docker image inspect $base >/dev/null 2>&1 || docker build -q -t $base "$here" >/dev/null +if [ "$SUBSCRIPTION" = grok ]; then + base=maat-bench-grok:agentu; img=maat-bench-grok:$sha-u + docker image inspect $base >/dev/null 2>&1 || docker build -q -t $base -f "$here/Dockerfile.grok" "$here" >/dev/null + [ -f ~/.grok/auth.json ] || { echo "no ~/.grok/auth.json: run 'grok login' on the Mac first" >&2; exit 1; } + # The token is short-lived (~6 h) and a refresh inside the container may rotate the refresh + # token the Mac holds; start with a fresh one so no refresh is needed during the run. + left=$(python3 -c "import json,os,datetime as d;v=list(json.load(open(os.path.expanduser('~/.grok/auth.json'))).values())[0];print(int((d.datetime.fromisoformat(v['expires_at'][:26]+'+00:00')-d.datetime.now(d.timezone.utc)).total_seconds()))" 2>/dev/null || echo 0) + [ "$left" -gt 1800 ] || echo "warning: grok token expires in ${left}s; run any 'grok' command on the Mac to refresh it first" >&2 +fi +if [ "$OPENCODE" = 1 ]; then + [ -f ~/.local/share/opencode/auth.json ] || { echo "no ~/.local/share/opencode/auth.json: run 'opencode auth login' on the Mac first" >&2; exit 1; } + ocbase=${base%:*}-oc:agentu + docker image inspect $ocbase >/dev/null 2>&1 || docker build -q -t $ocbase -f "$here/Dockerfile.opencode" --build-arg BASE=$base "$here" >/dev/null + base=$ocbase; img=${img%:*}-oc:${img##*:} +fi +if ! docker image inspect "$img" >/dev/null 2>&1; then + ctx=$(mktemp -d); cp "$tgz" "$ctx/maat.tgz" + printf 'FROM %s\nCOPY maat.tgz /tmp/maat.tgz\nRUN npm install -g --no-fund --no-audit /tmp/maat.tgz && rm /tmp/maat.tgz\n' "$base" > "$ctx/Dockerfile" + docker build -q -t "$img" "$ctx" >/dev/null; rm -rf "$ctx" +fi +out=~/.cache/maat-bench/container-results; mkdir -p $out +work=~/.cache/maat-bench/container-work/$name; mkdir -p $work +key= +# A Grok lane gets the OpenRouter key only when a judge model needs it (JUDGE_KEY=1). +[ "$SUBSCRIPTION" = grok ] && [ "${JUDGE_KEY:-0}" != 1 ] || key=$(node -e "import('$HOME/Documents/molt-desktop/dist/providers.js').then(m=>process.stdout.write(m.readAuth().openrouter||''))") +url=${BENCH_URL:-https://openrouter.ai/api/v1} +# The NUC tunnel is the Mac's localhost; from the container it is host.docker.internal. +url=${url//127.0.0.1/host.docker.internal}; url=${url//localhost/host.docker.internal} +# The key goes in through a file only this user can read, removed when the +# run ends: on the command line (-e KEY=...) it showed in every `ps` listing. +envf=$(mktemp); chmod 600 "$envf"; trap 'rm -f "$envf"' EXIT +printf 'OPENROUTER_API_KEY=%s\n' "$key" > "$envf" +# The graders and reference solutions are mounted where only root can reach (/root is 700), +# copied to a root-only /opt/bench, and run.py runs from there as root; it runs the agent +# as the unprivileged `agent` user (BENCH_AGENT_USER) and grades as root afterwards. +credmount=; startcmd='export MOLT_DIST_ABS=$(npm root -g)/@solvyx/molt/dist; mkdir -p /work && chmod 755 /work && rm -rf /opt/bench && cp -a /root/bench-src /opt/bench && chmod -R go-rwx /opt/bench && cd /opt/bench && python3 run.py "$@"' +if [ "$SUBSCRIPTION" = grok ]; then + credmount="-v $HOME/.grok/auth.json:/root/grok-cred/auth.json:ro" + # grok runs as `agent`, so the credential copy lives in ITS home, owned by it, mode 600. + startcmd='install -d -o agent -g agent -m 700 /home/agent/.grok && install -o agent -g agent -m 600 /root/grok-cred/auth.json /home/agent/.grok/auth.json && '$startcmd + # GROK_OWN_TOOLS=1: let Grok Build use its own shell/edit tools (inside the container only). + # Needed because Grok hides Maat's MCP tools behind its search_tool/use_tool, so by default + # the model asks for run_terminal_command, Maat refuses, and the turn ends with no work done. + # Writes a fresh config.toml in the container; the Mac's is never read or copied. + [ -z "$GROK_OWN_TOOLS" ] || startcmd='install -d -o agent -g agent -m 700 /home/agent/.grok && printf "[ui]\npermission_mode = \"always-approve\"\n" > /home/agent/.grok/config.toml && chown agent:agent /home/agent/.grok/config.toml && '$startcmd +fi +if [ "$OPENCODE" = 1 ]; then + credmount="$credmount -v $HOME/.local/share/opencode/auth.json:/root/oc-cred/auth.json:ro" + startcmd='install -d -o agent -g agent -m 700 /home/agent/.local /home/agent/.local/share /home/agent/.local/share/opencode && install -o agent -g agent -m 600 /root/oc-cred/auth.json /home/agent/.local/share/opencode/auth.json && '$startcmd +fi +echo "maat $img → $out/${RESULTS:-results.jsonl}" +docker run --rm --name "maat-bench-$name" \ + -v "$local_dir":/root/bench-src:ro -v "$out":/results -v "$work":/work ${=credmount} \ + --env-file "$envf" -e REFERENCE="${REFERENCE:-0}" -e BENCH_MODEL="${BENCH_MODEL:-}" \ + -e ARMS="${ARMS:-}" -e BENCH_TASKS="${BENCH_TASKS:-}" -e BENCH_URL="$url" -e BENCH_LIMIT="${BENCH_LIMIT:-600}" -e BENCH_REASONING="${BENCH_REASONING:-}" -e BENCH_GATE="${BENCH_GATE:-}" -e RESULTS="${RESULTS:-results.jsonl}" \ + -e MAAT_BUILD="$sha" -e PYTHONDONTWRITEBYTECODE=1 \ + "$img" sh -c "$startcmd" run "$@" diff --git a/bench/local/crosscheck.py b/bench/local/crosscheck.py new file mode 100644 index 0000000..b2b5f5f --- /dev/null +++ b/bench/local/crosscheck.py @@ -0,0 +1,476 @@ +""" +Bet 1 offline experiment (reports/frontier-2026-10-06.md): cross-check every run's +sealed drafted checks against every other run's final tree, and simulate the +"retire a drafted check on independent evidence" rule. No model calls, no network. + + python3 crosscheck.py inventory # list runs/trees found (writes the cache index) + python3 crosscheck.py execute [-j 6] # one container per tree: grade + every distinct check of its task + python3 crosscheck.py analyze [--md out] # matrices, retire rule, precision/recall + +Trees are never modified: each is bind-mounted read-only into a throwaway +`maat-bench` container (network off, memory/pids capped) which copies it to +/work/ (the path the run had) and runs the hidden grader and each check on +a fresh copy, so one check's side effects (git checkout, rm) cannot leak into the +next. Results are cached per tree in ~/.cache/maat-bench/crosscheck/ (resumable). + +Retire rule (per run R, per drafted check c in R's sealed set): + witnesses = trees of OTHER runs of the same task that fail c AND pass every other + check of R's set; + c is retired when >=2 witnesses fail with the same output signature + (variant "strict": the witnesses must also differ in content from each other and + from R's own tree). R's verdict is recomputed with retired checks removed. +""" + +from __future__ import annotations + +import argparse +import collections +import concurrent.futures as cf +import hashlib +import json +import os +import re +import shutil +import signal +import subprocess +import sys +import tempfile +from pathlib import Path + +HERE = Path(__file__).resolve().parent +CACHE = Path.home() / ".cache/maat-bench" +RES = CACHE / "container-results" +WORK = CACHE / "container-work" +OUT = CACHE / "crosscheck" +IMAGE = "maat-bench" +CHECK_TIMEOUT = 20 + +# results file stem -> where its trees live +SETS = [ + "fail5", "grok-fix", "grok-smoke", "grok-smoke2", "grok-smoke3", "grok1", "m8b", "m8b2", + "q2-base", "q3-base", "q4-fix", "v9-new", "v9-old", "v10-fix", "v10-old", "v11-new", "v11-old", + "v12-new", "v12-old", "gem1", "sub-gemini", "sub-grok", +] + + +# ------------------------------------------------------------------ inside the container +def inside() -> None: + sys.path.insert(0, str(HERE)) + job = json.load(open("/job.json")) + from tasks import TASKS # noqa: PLC0415 + from tasks2 import TASKS2 # noqa: PLC0415 + + by = {T.name: T for T in TASKS + TASKS2} + os.environ["GIT_CONFIG_COUNT"] = "1" + os.environ["GIT_CONFIG_KEY_0"] = "safe.directory" + os.environ["GIT_CONFIG_VALUE_0"] = "*" + dst = Path("/work") / job["name"] + + def fresh() -> Path: + shutil.rmtree(dst, ignore_errors=True) + dst.parent.mkdir(parents=True, exist_ok=True) + subprocess.run(["cp", "-R", "--preserve=mode,timestamps", "/src", str(dst)], check=True) + return dst + + res: dict = {"name": job["name"], "checks": {}} + try: + d = fresh() + ok, why = by[job["task"]].grade(d) + res["grade"] = [bool(ok), str(why)[:300]] + except Exception as e: # noqa: BLE001 + res["grade"] = [None, f"grader crashed: {e}"[:300]] + for cmd in job["checks"]: + d = fresh() + # a check names the folder of the run that drafted it (/work/-molt-); in another + # run's tree that path must mean "this tree", or every foreign tree fails it spuriously + run_cmd = re.sub(r"/work/[\w.-]+-molt-\d+", str(dst), cmd) + p = subprocess.Popen(run_cmd, shell=True, cwd=d, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + stdin=subprocess.DEVNULL, start_new_session=True) + try: + out, _ = p.communicate(timeout=CHECK_TIMEOUT) + rc = p.returncode + except subprocess.TimeoutExpired: + try: + os.killpg(p.pid, signal.SIGKILL) + except Exception: # noqa: BLE001 + pass + out, _ = p.communicate() + rc = 124 + try: + os.killpg(p.pid, signal.SIGKILL) # leave no server behind + except Exception: # noqa: BLE001 + pass + res["checks"][cmd] = {"rc": rc, "out": out.decode("utf8", "replace")[:600]} + print("@@RESULT@@" + json.dumps(res)) + + +# ------------------------------------------------------------------ inventory (host) +def tree_hash(d: Path) -> str: + h = hashlib.sha256() + for p in sorted(d.rglob("*")): + rel = p.relative_to(d) + if rel.parts[0] in (".maat", ".git") or "__pycache__" in rel.parts or "node_modules" in rel.parts: + continue + if p.is_file() and not p.is_symlink(): + h.update(str(rel).encode()) + try: + h.update(p.read_bytes()) + except OSError: + pass + return h.hexdigest()[:12] + + +def journal_info(d: Path) -> dict: + """Sealed drafted checks (name -> command) from bar_run events, last bar's failures, tokens.""" + checks: dict[str, str] = {} + last = None + tokens = 0 + nbar = 0 + for p in sorted((d / ".maat/log").glob("*.jsonl")) if (d / ".maat/log").is_dir() else []: + for line in p.read_text(errors="replace").splitlines(): + try: + e = json.loads(line) + except ValueError: + continue + k = e.get("kind") + if k == "response": + tokens += e["data"].get("promptTokens", 0) or 0 + elif k == "bar_run": + nbar += 1 + last = e["data"] + for c in e["data"]["checks"]: + checks[c["name"]] = c["detail"] + info = {"checks": checks, "nbar": nbar, "tokens": tokens} + if last: + info["last_ok"] = bool(last["ok"]) + info["last_failed_names"] = [c["name"] for c in last["checks"] if not c["ok"]] + info["first_ok"] = None + return info + + +def find_tree(stem: str, row: dict) -> Path | None: + name = f"{row['task']}-{row.get('agent', 'molt')}-{row['rep']}" + if stem in ("gem1", "sub-gemini", "sub-grok"): + p = CACHE / f"work-{stem}" / name + return p if p.is_dir() else None + cands = [WORK / d / name for d in sorted(os.listdir(WORK)) if (d == stem or d.startswith(stem + "-")) and (WORK / d / name).is_dir()] + if len(cands) <= 1: + return cands[0] if cands else None + for c in cands: # two shards ran the same task/rep: the one whose journal matches the row + if journal_info(c)["tokens"] == row.get("tokens_in"): + return c + shard = [c for c in cands if re.search(r"-s\d+$", c.parent.name)] + return (shard or cands)[0] + + +def build_inventory() -> list[dict]: + runs = [] + for stem in SETS: + f = RES / f"results-{stem}.jsonl" + if not f.exists(): + continue + for line in f.read_text().splitlines(): + if not line.strip(): + continue + r = json.loads(line) + if r.get("agent", "molt") != "molt": + continue + d = find_tree(stem, r) + if d is None: + continue + ji = journal_info(d) + runs.append({ + "id": f"{stem}/{r['task']}-{r['rep']}", "set": stem, "task": r["task"], "rep": r["rep"], + "dir": str(d), "grader_row": r["passed"], "claim": r.get("claim"), "tier": r.get("tier"), + "review": r.get("review"), "checks_disagree": r.get("checks_disagree"), + "hash": tree_hash(d), **ji, + }) + return runs + + +def load_inventory() -> list[dict]: + OUT.mkdir(parents=True, exist_ok=True) + idx = OUT / "inventory.json" + if idx.exists(): + return json.loads(idx.read_text()) + runs = build_inventory() + idx.write_text(json.dumps(runs)) + return runs + + +def safe(i: str) -> str: + return i.replace("/", "__") + + +# ------------------------------------------------------------------ execute (host) +def exec_tree(run: dict, cmds: list[str]) -> str: + out = OUT / "trees" / (safe(run["id"]) + ".json") + old = None + if out.exists(): + old = json.loads(out.read_text()) + if old.get("pathfix"): + if set(cmds) <= set(old["checks"]): + return "cached" + else: # results from before the /work/ rewrite: redo only commands that name such a path + cmds = [c for c in cmds if "/work/" in c] + if not cmds: + old["pathfix"] = True + out.write_text(json.dumps(old)) + return "cached" + with tempfile.TemporaryDirectory() as td: + job = Path(td) / "job.json" + job.write_text(json.dumps({"name": Path(run["dir"]).name, "task": run["task"], "checks": cmds})) + try: + p = subprocess.run( + ["docker", "run", "--rm", "--network", "none", "--memory", "1g", "--pids-limit", "512", + "-v", f"{HERE}:/bench:ro", "-v", f"{run['dir']}:/src:ro", "-v", f"{job}:/job.json:ro", + IMAGE, "python3", "/bench/crosscheck.py", "--inside"], + capture_output=True, text=True, timeout=60 + CHECK_TIMEOUT * (len(cmds) + 2) * 2) + txt = p.stdout + except subprocess.TimeoutExpired: + return "container-timeout" + m = re.search(r"@@RESULT@@(.*)", txt) + if not m: + return "no-result: " + (p.stderr or txt)[-200:] + new = json.loads(m.group(1)) + if old is not None: # merge: keep earlier results for commands not re-run + old["checks"].update(new["checks"]) + new = old + new["pathfix"] = True + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(new)) + return "ok" + + +def execute(jobs: int) -> None: + runs = load_inventory() + cmds_by_task: dict[str, set[str]] = collections.defaultdict(set) + for r in runs: + cmds_by_task[r["task"]].update(r["checks"].values()) + todo = [(r, sorted(cmds_by_task[r["task"]])) for r in runs] + print(f"{len(runs)} trees, {sum(len(v) for v in cmds_by_task.values())} distinct (task,check) commands", flush=True) + done = 0 + with cf.ThreadPoolExecutor(jobs) as ex: + futs = {ex.submit(exec_tree, r, c): r for r, c in todo} + for f in cf.as_completed(futs): + done += 1 + s = f.result() + if s not in ("ok", "cached"): + print(" !", futs[f]["id"], s, flush=True) + if done % 25 == 0: + print(f" {done}/{len(todo)}", flush=True) + + +# ------------------------------------------------------------------ analysis +def sig(rc: int, out: str, run_name: str) -> str: + o = re.sub(r"/work/[\w.-]+", "/work/T", out) + o = re.sub(r"0x[0-9a-f]+|\b[0-9a-f]{7,40}\b", "H", o) + o = re.sub(r"\d{4,}", "N", o) + o = re.sub(r"\s+", " ", o).strip()[:300] + return f"{rc}|{o}" + + +def analyze(md_out: str | None) -> None: + runs = load_inventory() + res: dict[str, dict] = {} + for r in runs: + p = OUT / "trees" / (safe(r["id"]) + ".json") + if p.exists(): + res[r["id"]] = json.loads(p.read_text()) + runs = [r for r in runs if r["id"] in res] + for r in runs: + g = res[r["id"]]["grade"][0] + r["grader"] = r["grader_row"] if g is None else g + r["grade_rerun"] = g + r["verified"] = bool(r["claim"] and r["claim"].startswith("verified")) + L: list[str] = [] + out = L.append + + gdis = [r for r in runs if r["grade_rerun"] is not None and r["grade_rerun"] != r["grader_row"]] + out(f"Trees analysed: {len(runs)}; grader re-run disagrees with the recorded verdict on {len(gdis)}" + + (": " + ", ".join(r["id"] for r in gdis[:12]) if gdis else "") + ".") + + # matrix: task -> cmd -> run id -> (ok, sig) + M: dict[str, dict[str, dict[str, tuple[bool, str]]]] = collections.defaultdict(lambda: collections.defaultdict(dict)) + for r in runs: + for cmd, v in res[r["id"]]["checks"].items(): + M[r["task"]][cmd][r["id"]] = (v["rc"] == 0, sig(v["rc"], v["out"], r["id"])) + by_task: dict[str, list[dict]] = collections.defaultdict(list) + for r in runs: + by_task[r["task"]].append(r) + + # per-run simulation + insts = [] # one per (run, drafted check) + for r in runs: + if not r["checks"]: + continue + t = r["task"] + cmds = sorted(set(r["checks"].values())) + name_of = {v: k for k, v in r["checks"].items()} + for c in cmds: + own = M[t][c].get(r["id"]) + others = [o for o in by_task[t] if o["id"] != r["id"] and o["id"] in M[t][c]] + wit = [] + for o in others: + ok, sg = M[t][c][o["id"]] + if ok: + continue + if all(M[t][c2].get(o["id"], (False,))[0] for c2 in cmds if c2 != c): + wit.append((o, sg)) + groups = collections.defaultdict(list) + for o, sg in wit: + groups[sg].append(o) + best = max(groups.values(), key=len) if groups else [] + retired = len(best) >= 2 + strict_w = {o["hash"] for o in best if o["hash"] != r["hash"]} + retired_strict = len(strict_w) >= 2 + ngp = sum(1 for o in best if o["grader"]) + insts.append({ + "run": r["id"], "task": t, "name": name_of[c], "cmd": c, "own_fail": own is not None and not own[0], + "own_grader": r["grader"], "retired": retired, "retired_strict": retired_strict, + "n_witness": len(best), "n_witness_pass": ngp, "n_other_trees": len(others), + }) + + # global check label: fails >=1 grader-passing tree = wrong ; fails only grader-failing = right + glabel = {} + for t, cm in M.items(): + gp = {r["id"] for r in by_task[t] if r["grader"]} + gf = {r["id"] for r in by_task[t] if not r["grader"]} + for c, row in cm.items(): + F = {i for i, (ok, _) in row.items() if not ok} + if not F: + glabel[(t, c)] = "never-fails" + elif F & gp: + glabel[(t, c)] = "wrong" + else: + glabel[(t, c)] = "right-exact" if F == gf & set(row) else "right-partial" + + # ---- headline: refusals by own tree + ref = [i for i in insts if i["own_fail"]] + wrong_ref = [i for i in ref if i["own_grader"]] + right_ref = [i for i in ref if not i["own_grader"]] + + def rate(xs, key): + return f"{sum(1 for i in xs if i[key])}/{len(xs)} ({100 * sum(1 for i in xs if i[key]) / max(1, len(xs)):.0f}%)" + + out("\n## Retire rule on refusing checks (check fails its own run's final tree)\n") + out("| class | instances | retired (rule) | retired (strict: distinct trees) |") + out("|---|---|---|---|") + out(f"| wrong refusal (own tree grader-PASS) | {len(wrong_ref)} | {rate(wrong_ref, 'retired')} | {rate(wrong_ref, 'retired_strict')} |") + out(f"| right refusal (own tree grader-FAIL) | {len(right_ref)} | {rate(right_ref, 'retired')} | {rate(right_ref, 'retired_strict')} |") + # global labelling + gl = collections.Counter(glabel[(i["task"], i["cmd"])] for i in ref) + out(f"\nRefusing instances by global check label (check fails some grader-passing tree = wrong): {dict(gl)}") + for lab in ("wrong", "right-exact", "right-partial"): + xs = [i for i in ref if glabel[(i["task"], i["cmd"])] == lab] + out(f"- global {lab}: {len(xs)} instances, retired {rate(xs, 'retired')}, strict {rate(xs, 'retired_strict')}") + + # ---- verdict recompute + out("\n## Verdicts before / after\n") + inst_by_run = collections.defaultdict(dict) + for i in insts: + inst_by_run[i["run"]][i["name"]] = i + for variant, key in (("rule", "retired"), ("strict", "retired_strict")): + tp = fp = fn = tn = 0 + rescued_good = [] + rescued_bad = [] + for r in runs: + v = r["verified"] + after = v + if not v and r.get("nbar") and r.get("last_ok") is False and r["last_failed_names"]: + ii = inst_by_run[r["id"]] + if all(n in ii and ii[n][key] for n in r["last_failed_names"]): + after = True + (rescued_good if r["grader"] else rescued_bad).append(r) + tp += after and r["grader"]; fp += after and not r["grader"] + fn += (not after) and r["grader"]; tn += (not after) and not r["grader"] + if variant == "rule": + r["_after"] = after + # before + btp = sum(1 for r in runs if r["verified"] and r["grader"]); bfp = sum(1 for r in runs if r["verified"] and not r["grader"]) + P = sum(1 for r in runs if r["grader"]) + out(f"**{variant}**: runs {len(runs)}, grader-pass {P}. Before: verified {btp + bfp} (TP {btp}, FP {bfp}) precision {100 * btp / max(1, btp + bfp):.0f}% recall {100 * btp / max(1, P):.0f}%." + f" After: verified {tp + fp} (TP {tp}, FP {fp}) precision {100 * tp / max(1, tp + fp):.0f}% recall {100 * tp / max(1, P):.0f}%." + f" Rescued grader-pass: {len(rescued_good)}; false-done introduced (grader-FAIL now verified): {len(rescued_bad)}.") + out(" rescued-good: " + ", ".join(sorted(r["id"] for r in rescued_good))) + out(" false-done: " + ", ".join(sorted(r["id"] for r in rescued_bad))) + if variant == "rule": + refused_pass = [r for r in runs if r["grader"] and not r["verified"] and r.get("last_ok") is False] + out(f" grader-pass runs whose last bar refused them: {len(refused_pass)}; of these rescued {len(rescued_good)}") + rr = [r for r in rescued_good + rescued_bad if (r.get("review") or {}).get("confirmed") and (r["review"].get("votes") or "0/3")[0] != "0"] + out(f" rescued runs whose independent review contradicted the task: {len(rr)}") + + # ---- per task + out("\n## Per task\n") + out("| task | trees | grader pass | distinct trees | runs w/ checks | distinct checks | refusing inst. | wrong-ref retired | right-ref retired | rescued good | false-done |") + out("|---|---|---|---|---|---|---|---|---|---|---|") + for t in sorted(by_task): + rs = by_task[t] + ii = [i for i in insts if i["task"] == t] + rf = [i for i in ii if i["own_fail"]] + w = [i for i in rf if i["own_grader"]] + rt = [i for i in rf if not i["own_grader"]] + rg = rb = 0 + for r in rs: + if r["verified"] or r.get("last_ok") is not False or not r["last_failed_names"]: + continue + m = inst_by_run[r["id"]] + if all(n in m and m[n]["retired"] for n in r["last_failed_names"]): + rg += r["grader"]; rb += not r["grader"] + out(f"| {t} | {len(rs)} | {sum(r['grader'] for r in rs)} | {len({r['hash'] for r in rs})} | {sum(1 for r in rs if r['checks'])} | " + f"{len({c for c in M[t]})} | {len(rf)} | {sum(i['retired'] for i in w)}/{len(w)} | {sum(i['retired'] for i in rt)}/{len(rt)} | {rg} | {rb} |") + + # ---- retired checks list + out("\n## Retired checks (distinct task+command, refusing instances only)\n") + out("| task | check | instances | retired | global label | own-grader pass/fail | fails grader-passing trees | fails grader-failing trees |") + out("|---|---|---|---|---|---|---|---|") + agg = collections.defaultdict(list) + for i in ref: + agg[(i["task"], i["cmd"])].append(i) + for (t, c), xs in sorted(agg.items()): + if not any(i["retired"] for i in xs): + continue + row = M[t][c] + gp = {r["id"] for r in by_task[t] if r["grader"]} + F = {k for k, (ok, _) in row.items() if not ok} + out(f"| {t} | `{xs[0]['name']}`: `{c[:90].replace('|', '/')}` | {len(xs)} | {sum(i['retired'] for i in xs)} | {glabel[(t, c)]} | " + f"{sum(i['own_grader'] for i in xs)}/{sum(not i['own_grader'] for i in xs)} | {len(F & gp)}/{len(gp & set(row))} | {len(F - gp)}/{len(set(row) - gp)} |") + + # right checks that got retired (the cost) + out("\n## Right-refusal instances that the rule retires\n") + for i in right_ref: + if i["retired"]: + out(f"- {i['run']} `{i['name']}`: {i['cmd'][:100]} (witnesses {i['n_witness']}, of which grader-pass {i['n_witness_pass']}, label {glabel[(i['task'], i['cmd'])]})") + + text = "\n".join(L) + print(text) + if md_out: + Path(md_out).write_text(text + "\n") + (OUT / "insts.json").write_text(json.dumps(insts)) + (OUT / "runs-analysed.json").write_text(json.dumps([{k: v for k, v in r.items() if k != "checks"} for r in runs])) + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("cmd", nargs="?", default="analyze", choices=["inventory", "execute", "analyze"]) + ap.add_argument("-j", type=int, default=6) + ap.add_argument("--md") + ap.add_argument("--inside", action="store_true") + a = ap.parse_args() + if a.inside: + return inside() + if a.cmd == "inventory": + (OUT / "inventory.json").unlink(missing_ok=True) + runs = load_inventory() + c = collections.Counter(r["task"] for r in runs) + print(len(runs), "trees;", sum(1 for r in runs if r["checks"]), "with sealed checks") + for t, n in sorted(c.items()): + print(f" {t}: {n}") + elif a.cmd == "execute": + execute(a.j) + else: + analyze(a.md) + + +if __name__ == "__main__": + main() diff --git a/bench/local/denials.py b/bench/local/denials.py new file mode 100644 index 0000000..393daea --- /dev/null +++ b/bench/local/denials.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Denied tool calls in bench journals, by the gate's reason. + +Headless with --yes, every gated "ask" is answered "User denied this action.", +so a `permission` event with allowed:false is a call the model could not run. +This reads `**/.maat/log/*.jsonl` under a folder (or several) and prints a +table: gate reason -> denied count, then the commands behind each reason, +grouped by the rule in autonomy.ts that most likely fired. + + python3 bench/local/denials.py ~/.cache/maat-bench/container-work/v9-new-s* + python3 bench/local/denials.py DIR [DIR...] [--top N] [--width W] + +Journal event shape: {"kind":"permission","data":{"name","detail","allowed", +"asked","autonomy","why"}}. `detail` is the command for bash, the path for a +file tool, and the raw arguments for act. +""" +import argparse +import collections +import glob +import json +import os +import re +import sys + +# The IRREVERSIBLE rules of src/autonomy.ts, first match names the row. +RULES = [ + ("rm/rmdir/unlink/shred", r"\b(rm|rmdir|unlink|shred)\b"), + ("find -delete/-exec", r"-delete\b|-exec\b"), + ("tee", r"\btee\b"), + ("heredoc/redirect write (>)", r"(?)>(?![>&])"), + ("pkill/killall", r"\b(pkill|killall)\b"), + ("git history/state", r"\bgit\s+(push|reset|clean|checkout|restore|branch|rebase|stash)\b"), + ("inline program (-c/-e)", r"\b(python[\d.]*|node|ruby|perl|php|deno|bun)\b[^|;&]*\s-(c|e|eval|pi|pe)\b"), + ("sh -c / bash -c", r"\b(sh|bash|zsh)\b[^|;&]*\s-c\b"), + ("pipe into interpreter", r"\|\s*(sh|bash|zsh|python|node)\b"), + ("sudo/chmod/dd/mkfs", r"\b(sudo|chmod|dd|mkfs)\b"), +] + + +def rule_of(name, detail, why): + if why and "cannot be undone" not in why: + return why + if name != "bash": + return f"{name}: {why or 'denied'}" + for label, pat in RULES: + if re.search(pat, detail): + return f"cannot be undone: {label}" + return "cannot be undone: (other)" + + +def journals(paths): + for p in paths: + if os.path.isfile(p): + yield p + continue + yield from sorted(glob.glob(os.path.join(p, "**", ".maat", "log", "*.jsonl"), recursive=True)) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("folders", nargs="+") + ap.add_argument("--top", type=int, default=3, help="example commands shown per rule") + ap.add_argument("--width", type=int, default=110) + a = ap.parse_args() + + paths = [] + for f in a.folders: + paths.extend(glob.glob(os.path.expanduser(f)) or [f]) + files = list(journals(paths)) + allowed = denied = 0 + by_rule = collections.defaultdict(collections.Counter) + runs = set() + for jf in files: + with open(jf, errors="replace") as fh: + for line in fh: + if '"kind":"permission"' not in line: + continue + try: + d = json.loads(line)["data"] + except (ValueError, KeyError): + continue + if d.get("allowed") is False: + denied += 1 + runs.add(jf) + cmd = str(d.get("detail", "")).replace("\n", " ") + by_rule[rule_of(d.get("name", ""), cmd, d.get("why", ""))][cmd] += 1 + else: + allowed += 1 + print(f"{len(files)} journals, {allowed} allowed, {denied} denied, {len(runs)} journals with a denial") + print(f"\n{'denied':>6} rule") + for rule, c in sorted(by_rule.items(), key=lambda kv: -sum(kv[1].values())): + print(f"{sum(c.values()):>6} {rule}") + for rule, c in sorted(by_rule.items(), key=lambda kv: -sum(kv[1].values())): + print(f"\n{rule}") + for cmd, n in c.most_common(a.top): + shown = cmd if len(cmd) <= a.width else cmd[: a.width - 1] + "…" + print(f" {n:>3}x {shown}") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bench/local/env_values.py b/bench/local/env_values.py new file mode 100644 index 0000000..d2551de --- /dev/null +++ b/bench/local/env_values.py @@ -0,0 +1,414 @@ +"""Bet 2 / half 1: where do the expected values in sealed drafted checks come from? + +Offline, no model: reads the sealed checks out of the saved .maat journals +(bar_run events), classifies each by the provenance of its asserted literals, +and measures what making class (e) checks advisory would have done to the +grader-labelled runs. + + (a) every asserted literal is in the task text + (b) ... or in the pristine input files + (c) ... or in the output of running the pristine program(s) + (d) the check asserts no free literal (relation / structure / exit status) + (e) at least one asserted literal comes from none of the above + +A "value literal" is an operand of a comparison (==, !=, -eq, -ne, <, >, <=, +>=, =~), of assertEqual-style helpers, a grep -q/-x pattern, or a line of a +here-doc given to diff/cmp: the same constructs src/tiers.ts assertsValue +looks at, but returning the literals instead of a boolean. Trivial literals +(0, 1, the empty string) are ignored: they are exit codes and indexes. + + python3 env_values.py [--json out.json] [--v11] +""" +from __future__ import annotations + +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +from collections import Counter, defaultdict +from pathlib import Path + +import replay_common as R + +NUMWORDS = {w: i for i, w in enumerate( + "zero one two three four five six seven eight nine ten eleven twelve".split())} + +# ------------------------------------------------------------------ literals +STR = r"""(?:\\?"((?:[^"\\]|\\.)*)\\?"|'([^']*)')""" +NUM = r"-?\d+(?:\.\d+)?" +OPS = re.compile(r"(==|!=|>=|<=|=~|(?<=\s)-eq(?=\s)|(?<=\s)-ne(?=\s)|(?<=\s)-ge(?=\s)|(?<=\s)-le(?=\s)|(?<=\s)-gt(?=\s)|(?<=\s)-lt(?=\s)|(?<=[\w)\]\s])\s>(?=\s*\d)|(?<=\s)=(?=\s))") + + +def _operand(text: str) -> list[str]: + """Literals of the operand that starts text (a string, number, or bracketed list/dict).""" + t = text.lstrip() + out: list[str] = [] + if t[:1] in "[{(": + close = {"[": "]", "{": "}", "(": ")"}[t[0]] + depth, end = 0, len(t) + for i, ch in enumerate(t): + if ch in "[{(": + depth += 1 + elif ch in "]})": + depth -= 1 + if depth == 0: + end = i + 1 + break + t = t[:end] + for m in re.finditer(STR, t): + out.append(m.group(1) if m.group(1) is not None else m.group(2)) + out += re.findall(r"(? list[str]: + t = text.rstrip() + m = re.search(STR + r"$", t) + if m: + s = m.group(1) if m.group(1) is not None else m.group(2) + return [] if "$" in s else [s] + m = re.search(r"(? list[str]: + """Port of the constructs in tiers.ts assertsValue, returning the literals asserted.""" + lits: list[str] = [] + for m in OPS.finditer(run): + at, op = m.start(), m.group(1).strip() + if op == "=": + seg = re.split(r"[;\n]|&&|\|\|", run[:at])[-1] + if not re.search(r"(?:^|\s)(?:\[\[?|test)\s", seg): + continue + lits += _operand(run[m.end(): m.end() + 200]) + lits += _before(run[max(0, at - 80): at]) + for m in re.finditer(r"(?:\bassert_?[Ee]qual|\bassert_eq|\.(?:deep|strict|deepStrict)?[Ee]qual)\w*\s*\(([^\n]*)", run): + lits += [a if a else b for a, b in re.findall(STR, m.group(1))] + lits += re.findall(r"(?>"] + for m in re.finditer(r"\bgrep\b([^\n|;&]*)", run): + words = re.findall(r"\"(?:[^\"\\]|\\.)*\"|'[^']*'|\S+", m.group(1)) + flags = [w for w in words if w.startswith("-")] + if not any(re.fullmatch(r"-[A-Za-z]+", f) and set(f) & set("qx") for f in flags): + continue + pat = next((w for w in words if not w.startswith("-")), None) + if pat: + body = re.sub(r"^([\"'])(.*)\1$", r"\2", pat, flags=re.S) + if body and not re.search(r"\$[\w{(]", body) and not re.search(r"\[[^\]]*\]|\\[dws.]|\.\*|\w\+", body) and (re.search(r"(?>?\s*|touch\s+|tee\s+(?:-a\s+)?|mkdir\s+(?:-p\s+)?|\bcp\s+\S+\s+|\bmv\s+\S+\s+|" + r"open\(\s*['\"])([\w./-]+)") + + +def path_refs(run: str) -> list[str]: + """Paths the check mentions that it does not create itself (inputs / expected outputs).""" + made = {m.group(1) for m in WRITE_CTX.finditer(run)} + out = [] + for m in PATH_TOK.finditer(run): + t = m.group(1) + if t in made or t.startswith(("usr/", "dev/", "tmp/", "bin/", "etc/")) or "mktemp" in t: + continue + if t.endswith((".stdout", ".exit", ".group", ".parameters", ".signature")): + continue + out.append(t) + return list(dict.fromkeys(out)) + + +# ------------------------------------------------------------------ provenance +def norm(s: str) -> str: + return re.sub(r"\s+", " ", s.lower().replace("\\n", "\n")).strip() + + +TIGHT = "--tight-numbers" in sys.argv # small integers are coincidences, never provenance + + +def contains(hay: str, lit: str) -> bool: + h, l = norm(hay), norm(lit) + if not l: + return True + if re.fullmatch(NUM, l): + if TIGHT and re.fullmatch(r"-?\d{1,2}", l): + return False + if re.search(r"(? 1 and all(p in h for p in parts) + + +def read_inputs(d: Path) -> str: + buf = [] + for p in sorted(d.rglob("*")): + if ".git" in p.parts or not p.is_file() or p.stat().st_size > 2_000_000: + continue + buf.append(p.relative_to(d).as_posix()) + try: + buf.append(p.read_text()) + except Exception: + pass + return "\n".join(buf) + + +_run_cache: dict[str, str] = {} + + +def pristine_outputs(task: str) -> str: + """stdout+stderr of every runnable thing in the pristine tree, no arguments and --help.""" + if task in _run_cache: + return _run_cache[task] + src = R.pristine(task) + out: list[str] = [] + tmp = Path(tempfile.mkdtemp(prefix="bet2-run-")) + try: + work = tmp / "w" + shutil.copytree(src, work, symlinks=True) + env = dict(os.environ, HOME=str(tmp), PYTHONDONTWRITEBYTECODE="1") + cmds: list[list[str]] = [] + for p in sorted(work.rglob("*")): + if not p.is_file() or ".git" in p.parts: + continue + rel = p.relative_to(work).as_posix() + runner = {".py": "python3", ".sh": "bash", ".js": "node"}.get(p.suffix) + if runner: + cmds += [[runner, rel], [runner, rel, "--help"]] + if (work / "Makefile").exists(): + cmds.append(["make", "-n"]) + if any(work.rglob("test*.py")): + cmds.append(["python3", "-m", "unittest", "discover", "-v"]) + for c in cmds: + try: + r = subprocess.run(c, cwd=work, env=env, capture_output=True, text=True, + timeout=6, stdin=subprocess.DEVNULL) + out += [r.stdout, r.stderr] + except Exception: + pass + finally: + shutil.rmtree(tmp, ignore_errors=True) + _run_cache[task] = "\n".join(out) + return _run_cache[task] + + +_ctx: dict[str, tuple[str, str]] = {} + + +def invented_paths(task: str, run_cmd: str) -> list[str]: + text, inputs = _ctx[task] + tree = {p.relative_to(R.pristine(task)).as_posix() for p in R.pristine(task).rglob("*") if ".git" not in p.parts} + bad = [] + for t in path_refs(run_cmd): + t = t.lstrip("./") + base = t.rsplit("/", 1)[-1] + if t in tree or contains(text, t) or contains(text, base) or any(f.endswith("/" + base) or f == base for f in tree) \ + or any(f.startswith(t.rstrip("/") + "/") for f in tree): + continue + bad.append(t) + return bad + + +def classify(task: str, run_cmd: str) -> tuple[str, list[str], list[str]]: + """-> (class a-e, literals, the untraced ones).""" + if task not in _ctx: + _ctx[task] = (R.TASK_BY_NAME[task].PROMPT, read_inputs(R.pristine(task))) + text, inputs = _ctx[task] + lits = value_literals(run_cmd) + if not lits: + return "d", [], [] + src = [] + for lit in lits: + if contains(text, lit): + src.append("a") + elif contains(inputs, lit): + src.append("b") + elif contains(pristine_outputs(task), lit): + src.append("c") + else: + src.append("e") + bad = [l for l, s in zip(lits, src) if s == "e"] + if bad: + return "e", lits, bad + return max(src), lits, [] # "a" < "b" < "c": the weakest source present + + +BENDS = [("access-report", 2), ("csv-clean", 2), ("cron-next", 2), ("git-revert-one", 2), + ("organize-files", 0), ("organize-files", 1), ("perf-pairs", 1), ("wc-tool", 0)] + + +# ------------------------------------------------------------------ the replay +def sealed_checks(bars: list[dict]) -> dict[str, dict]: + """name -> {cmd, first_ok, final_ok, ever_failed}, over every bar_run (late-joined included).""" + out: dict[str, dict] = {} + for i, b in enumerate(bars): + for c in b.get("checks", []): + if not R.drafted(c): + continue + d = out.setdefault(c["name"], {"cmd": c.get("detail", ""), "first_ok": c.get("ok"), + "ever_failed": False, "final_ok": None, "kind": c.get("kind")}) + d["final_ok"] = c.get("ok") + d["ever_failed"] |= not c.get("ok") + return out + + +def rate(n: int, d: int) -> str: + return f"{n}/{d} ({100 * n / d:.0f}%)" if d else f"{n}/0" + + +def main() -> None: + only_v11 = "--v11" in sys.argv + rs = R.runs() + seen, uniq = set(), [] + for r in rs: # v9 has re-run trees under one grader row: keep the first, mark the rest ambiguous + if r["id"] in seen: + continue + seen.add(r["id"]) + uniq.append(r) + if only_v11: + uniq = [r for r in uniq if r["arm"].startswith("v11")] + recs = [] + for r in uniq: + bars = R.bar_runs(R.journal_events(r["dir"])) + if not bars: + continue + ch = sealed_checks(bars) + for name, d in ch.items(): + cls, lits, bad = classify(r["task"], d["cmd"]) + cls_strict = cls + ip = invented_paths(r["task"], d["cmd"]) + if cls != "e" and ip: + cls = "e" # broad (e): an expected path/input the task and tree never mention + bad = bad + ["path:" + p for p in ip] + recs.append({"run": r["id"], "arm": r["arm"], "task": r["task"], "passed": r["passed"], + "check": name, "cls": cls, "lits": lits, "bad": bad, "strict_e": any(not b.startswith("path:") for b in bad), "cls_strict": cls_strict, "cmd": d["cmd"], + "first_ok": d["first_ok"], "final_ok": d["final_ok"], "ever_failed": d["ever_failed"], + "bars": len(bars)}) + which = "cls_strict" if "--strict" in sys.argv else "cls" + for x in recs: + x["cls"] = x[which] + print(f"definition of (e): {'strict, untraced value literals only' if which == 'cls_strict' else 'broad, also expected paths/inputs the task and tree never mention'}") + runs_with = {x["run"] for x in recs} + print(f"runs with a sealed check: {len(runs_with)} of {len(uniq)} graded; checks: {len(recs)}") + print("class of all sealed drafted checks:", dict(sorted(Counter(x['cls'] for x in recs).items()))) + + def table(title: str, sel) -> None: + sub = [x for x in recs if sel(x)] + c = Counter(x["cls"] for x in sub) + print(f"\n{title}: n={len(sub)} " + " ".join(f"{k}={c.get(k, 0)}" for k in "abcde") + + f" | (e) share {rate(c.get('e', 0), len(sub))}") + + table("checks passing on all runs", lambda x: x["final_ok"]) + table("REFUSED CORRECT: failed at final bar, grader passed", lambda x: x["passed"] and x["final_ok"] is False) + table("REFUSED CORRECT (ever failed, grader passed)", lambda x: x["passed"] and x["ever_failed"]) + table("CAUGHT WRONG: failed at final bar, grader failed", lambda x: (not x["passed"]) and x["final_ok"] is False) + table("CAUGHT WRONG (ever failed, grader failed)", lambda x: (not x["passed"]) and x["ever_failed"]) + table("passing checks on wrong runs (missed it)", lambda x: (not x["passed"]) and x["final_ok"]) + + # run-level refusal verdicts: positive = work is wrong + def confusion(adv: set[str], mode: str) -> dict: + per: dict[str, list] = defaultdict(list) + for x in recs: + per[x["run"]].append(x) + tp = fp = fn = tn = 0 + for run, xs in per.items(): + fail = [x for x in xs if x["cls"] not in adv and ((x["final_ok"] is False) if mode == "final" else x["ever_failed"])] + refused, wrong = bool(fail), not xs[0]["passed"] + tp += refused and wrong + fp += refused and not wrong + fn += (not refused) and wrong + tn += (not refused) and not wrong + return dict(tp=tp, fp=fp, fn=fn, tn=tn, + precision=tp / (tp + fp) if tp + fp else float("nan"), + recall=tp / (tp + fn) if tp + fn else float("nan"), + fpr=fp / (fp + tn) if fp + tn else float("nan")) + + print("\nrun-level refusal as a wrong-work detector (positive = grader failed)") + for mode in ("final", "ever"): + for label, adv in (("all drafted checks binding", set()), ("(e) advisory-only", {"e"}), + ("only a/b/c binding (d,e advisory)", {"d", "e"})): + c = confusion(adv, mode) + print(f" [{mode:5}] {label:34} TP={c['tp']:3} FP={c['fp']:3} FN={c['fn']:3} TN={c['tn']:3}" + f" precision={c['precision']:.2f} recall={c['recall']:.2f} false-refusal-rate={c['fpr']:.2f}") + + # the 8 v11-new bends named in reports/oracle-2026-10-06b.md section 1 (class A) + bends = [f"v11-new/{t}#{r}" for t, r in BENDS] + print(f"\nthe 8 known v11-new bends: drafted checks that refused the correct work first") + ncheck = ne = nruns_e = 0 + for b in bends: + trig = [x for x in recs if x["run"] == b and x["first_ok"] is False] + ncheck += len(trig) + ne += sum(x["cls"] == "e" for x in trig) + nruns_e += any(x["cls"] == "e" for x in trig) + for x in trig: + print(f" {b:34} {x['check']:36} [{x['cls']}] bad={x['bad'][:2]}") + if not trig: + print(f" {b:34} (no refused sealed check found in journal)") + print(f" triggering checks that are (e): {ne}/{ncheck}; bends with >=1 (e) trigger: {nruns_e}/8") + + # per-task view of class (e) on refused-correct + print("\nrefused-correct checks by task, class (e) first:") + bytask: dict[str, Counter] = defaultdict(Counter) + for x in recs: + if x["passed"] and x["final_ok"] is False: + bytask[x["task"]][x["cls"]] += 1 + for t, c in sorted(bytask.items(), key=lambda kv: -sum(kv[1].values())): + print(f" {t:20} " + " ".join(f"{k}={c[k]}" for k in sorted(c))) + + # examples + print("\nexamples, refused-correct (e):") + shown = set() + for x in recs: + if x["passed"] and x["final_ok"] is False and x["cls"] == "e" and (x["task"], x["check"]) not in shown and len(shown) < 8: + shown.add((x["task"], x["check"])) + print(f" {x['run']} {x['check']} bad={x['bad'][:3]}\n {x['cmd'][:160]!r}") + print("\nexamples, refused-correct (a/b/c/d):") + shown = set() + for x in recs: + if x["passed"] and x["final_ok"] is False and x["cls"] != "e" and (x["task"], x["check"]) not in shown and len(shown) < 8: + shown.add((x["task"], x["check"])) + print(f" [{x['cls']}] {x['run']} {x['check']} lits={x['lits'][:3]}\n {x['cmd'][:160]!r}") + + if "--json" in sys.argv: + Path(sys.argv[sys.argv.index("--json") + 1]).write_text(json.dumps(recs, indent=1)) + R.cleanup() + + +if __name__ == "__main__": + main() diff --git a/bench/local/frame_intact.py b/bench/local/frame_intact.py new file mode 100644 index 0000000..79e55b0 --- /dev/null +++ b/bench/local/frame_intact.py @@ -0,0 +1,221 @@ +"""Bet 2 / half 2: a generic, file-level `frame-intact` replayed on saved trees. + +Offline, no model. For each saved final tree, diff it against the pristine +tree (T.setup) and compare with the paths the task text names. Rules are +task-agnostic; nothing below knows a task by name. + + modified-unnamed an existing file changed and neither its path, its + basename, an ancestor directory, nor its stem word + appears in the task text + deleted-unnamed an existing file vanished and the same holds + deleted-input an existing file vanished though named, and the task + text has no verb that consumes inputs (move, remove, + delete, rename, organize, sort, rotate, archive, clean, + compress, migrate, replace, rewrite, convert, dedupe...) + test-changed an existing test file (test_*, *_test.*, *.test.*, + tests/ spec) was modified or deleted, named or not + history-rewritten a pristine commit is no longer an ancestor of HEAD + (reset/rebase/amend) and the task text does not mention + git history words + extra-commits commits were added and the task text does not mention + commit/merge/revert/branch/rebase/history/git + (soft) new-unnamed new files the task does not name; reported, not a fire + +Fire = any of the hard flags. Measured on: the 8 known v11-new bends; every +grader-failed run whose first bar run was refused; the grader-passed runs. + + python3 frame_intact.py [--list] +""" +from __future__ import annotations + +import filecmp +import re +import subprocess +import sys +from collections import Counter, defaultdict +from pathlib import Path + +import replay_common as R + +IGNORE_PARTS = {".maat", ".git", "__pycache__", ".pytest_cache", ".mypy_cache", "node_modules", ".DS_Store"} +CONSUME = re.compile(r"\b(move|moved|moving|delet\w*|remov\w*|renam\w*|organi[sz]\w*|sort\w*|rotat\w*|archiv\w*|" + r"clean\w*|compress\w*|migrat\w*|replac\w*|rewrit\w*|convert\w*|dedup\w*|drop|purge\w*|" + r"gzip\w*|trim\w*|redact\w*|fix\w*|refactor\w*|revert\w*|merge\w*)\b", re.I) +GITWORDS = re.compile(r"\b(commit\w*|merge\w*|revert\w*|branch\w*|rebase\w*|history|git|checkout|reset)\b", re.I) +TESTFILE = re.compile(r"(^|/)(tests?/|specs?/|test_[^/]*|[^/]*_test\.[^/]*|[^/]*\.test\.[^/]*|[^/]*\.spec\.[^/]*)") +BENDS = [("access-report", 2), ("csv-clean", 2), ("cron-next", 2), ("git-revert-one", 2), + ("organize-files", 0), ("organize-files", 1), ("perf-pairs", 1), ("wc-tool", 0)] + + +def files(root: Path) -> dict[str, Path]: + out = {} + for p in sorted(root.rglob("*")): + rel = p.relative_to(root) + if IGNORE_PARTS & set(rel.parts) or p.suffix == ".pyc": + continue + if p.is_file() and not p.is_symlink(): + out[rel.as_posix()] = p + return out + + +def named(rel: str, text: str) -> bool: + """Is this path named by the task text: full path, basename, an ancestor dir, or the stem as a word.""" + t = text.lower() + parts = rel.lower().split("/") + if rel.lower() in t or parts[-1] in t: + return True + for i in range(1, len(parts)): # ancestor directories: inbox, sorted, logs + d = parts[i - 1] + if re.search(rf"(?= 3 and re.search(rf"(? dict[str, tuple] | None: + """qualified def name -> (positional-or-keyword arg names, vararg, kwonly) for a python file.""" + import ast + try: + tree = ast.parse(path.read_text()) + except Exception: + return None + out: dict[str, tuple] = {} + + def walk(node, prefix=""): + for n in getattr(node, "body", []): + if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)): + a = n.args + out[prefix + n.name] = (tuple(x.arg for x in a.posonlyargs + a.args), a.vararg.arg if a.vararg else None, + tuple(x.arg for x in a.kwonlyargs), a.kwarg.arg if a.kwarg else None) + elif isinstance(n, ast.ClassDef): + walk(n, prefix + n.name + ".") + walk(tree) + return out + + +SCOPE = re.compile(r"\b(every file|all files|each file|under this directory|whole repo|entire repo|every file under)\b", re.I) + + +def git(cmd: str, cwd: Path) -> str: + return subprocess.run(cmd, shell=True, cwd=cwd, capture_output=True, text=True).stdout + + +def git_state(d: Path) -> tuple[list[str], str] | None: + if not (d / ".git").exists(): + return None + revs = git("git rev-list HEAD 2>/dev/null", d).split() + return revs, git("git rev-parse HEAD 2>/dev/null", d).strip() + + +def frame(task: str, final: Path) -> dict: + text = R.TASK_BY_NAME[task].PROMPT + pri = R.pristine(task) + a, b = files(pri), files(final) + flags: dict[str, list[str]] = defaultdict(list) + for rel, p in a.items(): + if rel not in b: + if not named(rel, text): + flags["deleted-unnamed"].append(rel) + elif not CONSUME.search(text): + flags["deleted-input"].append(rel) + if TESTFILE.search(rel): + flags["test-changed"].append(rel + " (deleted)") + elif not filecmp.cmp(p, b[rel], shallow=False): + if not named(rel, text): + flags["modified-unnamed"].append(rel) + if TESTFILE.search(rel): + flags["test-changed"].append(rel + " (modified)") + for rel, p in a.items(): # signature rule: any existing python def whose parameters changed or vanished + if rel.endswith(".py") and rel in b and not TESTFILE.search(rel): + sa, sb = signatures(p), signatures(b[rel]) + if sa is not None and sb is not None: + bad = [n for n, sig in sa.items() if sb.get(n) != sig and not n.rsplit(".", 1)[-1].startswith("_")] + if bad: + flags["signature-changed"].append(f"{rel}:{','.join(bad[:3])}") + soft = [rel for rel in b if rel not in a and not named(rel, text)] + ga, gb = git_state(pri), git_state(final) + if ga and gb and not GITWORDS.search(text): + lost = [r for r in ga[0] if r not in gb[0]] + added = [r for r in gb[0] if r not in ga[0]] + if lost: + flags["history-rewritten"].append(f"{len(lost)} pristine commits unreachable") + if added: + flags["extra-commits"].append(f"{len(added)} new commits") + # variants. V0 = the raw path rules; V1 exempts working-tree edits the task authorises by its nature + # (a git task, or "every file under this directory"); V2 adds stray new files; V3 adds signature changes + exempt = bool(GITWORDS.search(text) or SCOPE.search(text)) + v0 = {k: v for k, v in flags.items() if k != "signature-changed"} + v1 = {k: v for k, v in v0.items() if not (exempt and k in ("modified-unnamed", "deleted-unnamed"))} + v2 = dict(v1, **({"new-unnamed": soft} if soft else {})) + v3 = dict(v2, **({"signature-changed": flags["signature-changed"]} if "signature-changed" in flags else {})) + return {"flags": v0, "variants": {"V0": v0, "V1": v1, "V2": v2, "V3": v3}, "new_unnamed": soft, "fire": bool(v0), + "changed": sum(1 for rel in a if rel not in b or not filecmp.cmp(a[rel], b[rel], shallow=False))} + + +def pct(n: int, d: int) -> str: + return f"{n}/{d} ({100 * n / d:.0f}%)" if d else f"{n}/0" + + +def main() -> None: + rs = R.runs() + seen, uniq = set(), [] + for r in rs: + if r["id"] not in seen: + seen.add(r["id"]) + uniq.append(r) + for r in uniq: + r["frame"] = frame(r["task"], r["dir"]) + bars = R.bar_runs(R.journal_events(r["dir"])) + r["bars"] = bars + r["first_refused"] = bool(bars) and not bars[0]["ok"] + r["never_ok"] = bool(bars) and not any(b["ok"] for b in bars) + r["later_ok"] = r["first_refused"] and any(b["ok"] for b in bars[1:]) + + bend_ids = {f"v11-new/{t}#{k}" for t, k in BENDS} + groups = [ + ("8 known v11-new bends", lambda r: r["id"] in bend_ids), + ("grader-failed, first refused (all arms)", lambda r: not r["passed"] and r["first_refused"]), + (" of which later-ok (bend-shaped)", lambda r: not r["passed"] and r["later_ok"]), + (" of which never-ok", lambda r: not r["passed"] and r["never_ok"]), + ("grader-failed, all", lambda r: not r["passed"]), + ("CORRECT v11-old never-ok (the 23)", lambda r: r["arm"] == "v11-old" and r["passed"] and r["never_ok"]), + ("CORRECT all v11", lambda r: r["arm"].startswith("v11") and r["passed"]), + ("CORRECT first-refused, all arms", lambda r: r["passed"] and r["first_refused"]), + ("CORRECT all arms", lambda r: r["passed"]), + ] + print("fire rate per variant (V0 raw path rules; V1 + exempt git/scope tasks; V2 + stray new files; V3 + def-signature change)") + print(f"{'group':44} {'n':>4} " + " ".join(f"{v:>10}" for v in ("V0", "V1", "V2", "V3"))) + for title, sel in groups: + sub = [r for r in uniq if sel(r)] + cells = [pct(sum(bool(r["frame"]["variants"][v]) for r in sub), len(sub)) for v in ("V0", "V1", "V2", "V3")] + print(f"{title:44} {len(sub):>4} " + " ".join(f"{c:>10}" for c in cells)) + + print("\nthe 8 bends, per tree, V3 flags:") + for r in uniq: + if r["id"] in bend_ids: + f = r["frame"]["variants"]["V3"] + print(f" {r['id']:30} " + ("; ".join(f"{k}: {','.join(v[:3])}" for k, v in f.items()) or "-- nothing --")) + + print("\nfalse fires on grader-passed trees (V3), by flag and file:") + ex = defaultdict(Counter) + for r in uniq: + if r["passed"]: + for k, v in r["frame"]["variants"]["V3"].items(): + for x in v: + ex[k][f"{r['task']}:{x}"] += 1 + for k, c in ex.items(): + print(f" {k}: " + "; ".join(f"{n}x {x}" for x, n in c.most_common(10))) + + for v in ("V0", "V3"): + fired = [r for r in uniq if r["frame"]["variants"][v]] + w = sum(not r["passed"] for r in fired) + print(f"\n{v}: all fires {len(fired)}; on grader-failed {pct(w, len(fired))}; base rate of grader-failed {pct(sum(not r['passed'] for r in uniq), len(uniq))}") + if "--list" in sys.argv: + for r in uniq: + if not r["passed"] and r["later_ok"]: + print(r["id"], r["frame"]["variants"]["V3"]) + R.cleanup() + + +if __name__ == "__main__": + main() diff --git a/bench/local/oracle_prompt.txt b/bench/local/oracle_prompt.txt new file mode 100644 index 0000000..b581702 --- /dev/null +++ b/bench/local/oracle_prompt.txt @@ -0,0 +1,69 @@ +You write an independent reference check for one coding task. Someone else does +the task; you never see their work. Your check runs after they finish, in the +project folder, and decides whether their result is right. + +The danger you exist to remove: the person doing the task can misread it, and +any check they write repeats their misreading. So you work from the task's own +words only, and you write the most literal, obviously-correct version of the +answer you can — brute force, enumerate everything, no clever shortcuts, no +optimisation, every rule in the task applied exactly as written. Slow is fine +(under 30 seconds in total). Clever is how references go wrong. + +FIRST decide whether a reference applies. It applies when the right result can +be COMPUTED from the task text and inputs present in the project: a program or +function with a stated interface, a command whose output is specified, a file +transform, a report or query over files that are here. It does NOT apply when +the answer has to be discovered, searched for, trained, measured on hidden +data, depends on the network, or is a judgement call; or when the task is only +about repository or system state that a plain command checks directly. When +it does not apply, say so — a wrong reference is worse than none. + +When it applies, write ONE self-contained Python 3 file, check.py (standard +library only), that: + 1. holds your reference implementation, written from the task text; + 2. builds test inputs: every example the task gives; every edge case the + task's words imply — boundaries of each stated range, each kind of item + the task lists, with values chosen to expose an off-by-one or a wrong + starting point, + calendar edges (month ends, leap days, year end) if dates are involved, + empty input, duplicates, mixed case, surrounding whitespace, CRLF, if the + task speaks of such data; and a few seeded random inputs (random.seed(1)); + 3. computes EVERY expected result with your reference first, before the + deliverable is run at all; if your reference raises or gives up on any + input, exit 3. Brute force may need to go far (an answer can be years of + minutes away): never cap a search so low that a legal input runs out; + prefer a cap that only guards against an infinite loop; + 4. then runs THE DELIVERABLE exactly as the task says it is used (the command line, + import path or output file the task names) on each input, and your + reference on the same input; + 5. compares only what the task specifies. Strip trailing whitespace at line + ends and the final newline; do not demand formatting, ordering or extra + output the task does not state. If the task says the order does not + matter, compare as sets; + 6. on the first mismatch prints the input, the expected and the actual + result in a few lines and exits 1; when all match, prints a one-line + summary and exits 0. + +Inputs the deliverable reads that the task works on IN PLACE: the original +files, as they were before any work, are in {ORACLE}/before/ (same relative +paths). Read originals from there; read the deliverable's results from the +project folder (the current directory when check.py runs). Write any temporary +files under tempfile.mkdtemp(), never into the project. + +Robustness: if the deliverable is missing, crashes or times out (give each run +a timeout), that is a mismatch — print what happened and exit 1. If YOUR OWN +code fails (an exception outside the deliverable's run), exit 3 so nobody +mistakes your bug for theirs. + +NEVER work out an expected answer by hand: every expected value comes from +running your reference. The one exception is an example whose answer the task +text itself states — copy those verbatim, and before comparing anything run +your reference on them and exit 3 if it disagrees. Hand-computed answers are +the most common way a check goes wrong: a date that "should" be the next +match, a count that "should" be 12. If the task states no answers, there is +no self-test; that is fine. + +Reply with JSON only: +{"applies": true, "reason": "one line", "files": {"check.py": "...full source..."}, "run": "python3 {ORACLE}/check.py"} +or +{"applies": false, "reason": "one line"} diff --git a/bench/local/oracle_replay.py b/bench/local/oracle_replay.py new file mode 100644 index 0000000..4c0e4c1 --- /dev/null +++ b/bench/local/oracle_replay.py @@ -0,0 +1,235 @@ +""" +Offline replay: do drafted checks tell right work from wrong work? + +For every local task, take the saved workspaces (any agent, any rep) and the +hidden grader's verdict on each, plus a pristine copy (no work done). Then: + + A today's drafter + critic (dist/criteria.js draftCriteriaCritiqued) + B an independent reference check (ORACLE_SYSTEM below): a fresh model sees + the task text and the original files only, writes the most literal + reference it can and a harness that compares the deliverable to it. + +Each check set is run in a copy of every workspace. A set "accepts" a +workspace when every check passes. Reported per arm: + wrong work accepted (false verified — the thing to drive to zero) + right work refused (underclaim) + pristine accepted (a check that cannot tell no work from work) + + python3 oracle_replay.py [--arms A,B] [--tasks a,b] [--snap DIR] [--out FILE] +""" +from __future__ import annotations + +import argparse +import json +import os +import shutil +import subprocess +import sys +import tempfile +import time +import urllib.request +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +from tasks import TASKS # noqa: E402 +from tasks2 import TASKS2 # noqa: E402 +from tasks3 import TASKS3 # noqa: E402 +from run import openrouter_key # noqa: E402 + +REPO = HERE.parent.parent +MODEL = os.environ.get("REPLAY_MODEL", "nvidia/nemotron-3-ultra-550b-a55b:free") +SKIP = {".git", ".maat", ".molt", "node_modules", "__pycache__", ".venv"} + +ORACLE_SYSTEM = (REPO / "bench/local/oracle_prompt.txt").read_text() if (REPO / "bench/local/oracle_prompt.txt").exists() else "" + + +def ask(system: str, prompt: str, key: str, max_tokens: int = 16000) -> str: + body = { + "model": MODEL, + "messages": [{"role": "system", "content": system}, {"role": "user", "content": prompt}], + "max_tokens": max_tokens, + "reasoning": {"effort": os.environ.get("REPLAY_EFFORT", "low")}, + } + for attempt in range(7): + try: + req = urllib.request.Request( + "https://openrouter.ai/api/v1/chat/completions", + data=json.dumps(body).encode(), + headers={"Authorization": f"Bearer {key}", "Content-Type": "application/json"}, + ) + with urllib.request.urlopen(req, timeout=300) as r: + j = json.load(r) + text = (j["choices"][0]["message"].get("content") or "").strip() + if text: + return text + except Exception as e: # noqa: BLE001 + print(f" ask failed ({attempt}): {e}", file=sys.stderr) + time.sleep(5 * (attempt + 1)) + return "" + + +def listing(d: Path, limit: int = 40) -> str: + out = [] + for p in sorted(d.rglob("*")): + rel = p.relative_to(d) + if any(part in SKIP for part in rel.parts) or not p.is_file(): + continue + head = "" + try: + t = p.read_text() + head = "\n".join(t.splitlines()[:8]) + except (UnicodeDecodeError, OSError): + head = "(binary)" + out.append(f"--- {rel} ({p.stat().st_size} bytes)\n{head}") + if len(out) >= limit: + break + return "\n".join(out) or "(empty)" + + +def parse_json(text: str): + body = text.strip().removeprefix("```json").removeprefix("```").removesuffix("```") + s, e = body.find("{"), body.rfind("}") + if s < 0 or e < 0: + return None + try: + return json.loads(body[s : e + 1]) + except json.JSONDecodeError: + return None + + +def oracle_for(T, pristine: Path, key: str, store: Path) -> dict: + """Arm B: ask for a reference check; files land in store/, inputs snapshotted to store/before.""" + prompt = f"TASK TEXT:\n{T.PROMPT}\n\nFILES IN THE PROJECT BEFORE ANY WORK:\n{listing(pristine)}" + text = ask(ORACLE_SYSTEM, prompt, key) + j = parse_json(text) or {} + if not j.get("applies"): + return {"applies": False, "reason": j.get("reason", "") or text[:200]} + store.mkdir(parents=True, exist_ok=True) + shutil.copytree(pristine, store / "before", ignore=shutil.ignore_patterns(*SKIP), dirs_exist_ok=True) + files = j.get("files") or {} + for name, content in files.items(): + if "/" in name or name.startswith("."): + continue + (store / name).write_text(content) + run = (j.get("run") or "").replace("{ORACLE}", str(store)) + return {"applies": True, "run": run, "files": list(files)} + + +def product_reference_for(T, pristine: Path, key: str) -> dict: + """Arm C: the shipped src/reference.ts, through the built dist (snapshot, write, try, one retry).""" + js = f""" +import {{ draftReference, snapshotProject }} from '{REPO}/dist/reference.js'; +const snapshot = snapshotProject({json.dumps(str(pristine))}); +const r = snapshot ? await draftReference({{ task: {json.dumps(T.PROMPT)}, snapshot, + baseUrl: 'https://openrouter.ai/api/v1', apiKey: process.env.K, model: {json.dumps(MODEL)}, reasoningEffort: 'low' }}) : {{ ok: false, why: 'no snapshot' }}; +process.stdout.write(JSON.stringify(r.ok ? {{ applies: true, run: r.check.run, reason: r.reason }} : {{ applies: false, reason: r.why }})); +""" + p = subprocess.run(["node", "--input-type=module", "-e", js], capture_output=True, text=True, env=os.environ | {"K": key}, timeout=1200) + try: + return json.loads(p.stdout) + except json.JSONDecodeError: + return {"applies": False, "reason": f"node failed: {p.stderr[-300:]}"} + + +def drafted_for(T, pristine: Path, key: str) -> list[dict]: + """Arm A: today's drafter + critic, through the built dist.""" + js = f""" +import {{ draftCriteriaCritiqued, commandsHere }} from '{REPO}/dist/criteria.js'; +const r = await draftCriteriaCritiqued({{ + task: {json.dumps(T.PROMPT)}, scripts: [], barChecks: [], + baseUrl: 'https://openrouter.ai/api/v1', apiKey: process.env.K, model: {json.dumps(MODEL)}, + reasoningEffort: 'low', commands: commandsHere({json.dumps(str(pristine))}), cwd: {json.dumps(str(pristine))}, +}}); +process.stdout.write(JSON.stringify(r)); +""" + p = subprocess.run(["node", "--input-type=module", "-e", js], capture_output=True, text=True, env=os.environ | {"K": key}, timeout=600) + try: + r = json.loads(p.stdout) + except json.JSONDecodeError: + return [] + return r.get("draft", {}).get("checks", []) if r.get("ok") else [] + + +def real_work(ws: Path) -> bool: + """A workspace whose run died on the provider (e.g. a withdrawn model) holds no work to judge.""" + log = ws.with_name(ws.name + ".log") + return not (log.exists() and "No endpoints found" in log.read_text(errors="replace")) + + +def run_check(run: str, d: Path, timeout: int = 120) -> tuple[int, str]: + try: + p = subprocess.run(run, shell=True, cwd=d, capture_output=True, text=True, timeout=timeout) + return p.returncode, (p.stdout + p.stderr)[-600:] + except subprocess.TimeoutExpired: + return 124, "timeout" + + +def judge(checks: list[str], ws: Path) -> tuple[bool, list]: + tmp = Path(tempfile.mkdtemp(prefix="replay-")) + try: + shutil.copytree(ws, tmp / "w", symlinks=True) + outs = [run_check(c, tmp / "w") for c in checks] + return all(code == 0 for code, _ in outs), outs + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--arms", default="A,B") + ap.add_argument("--tasks") + ap.add_argument("--snap", default=os.environ.get("SNAP", str(Path.home() / ".cache/maat-bench/work"))) + ap.add_argument("--out", default=str(HERE / "oracle-replay.jsonl")) + ap.add_argument("--tag", default="") + a = ap.parse_args() + key = openrouter_key() + snap = Path(a.snap) + arms = a.arms.split(",") + tasks = [T for T in TASKS + TASKS2 + TASKS3 if not a.tasks or T.name in a.tasks.split(",")] + root = Path(tempfile.mkdtemp(prefix="oracle-replay-")) + for T in tasks: + wss = sorted(p for p in snap.glob(f"{T.name}-*-[0-9]") if p.is_dir() and real_work(p)) + if not wss: + continue + pristine = root / T.name / "pristine" + pristine.mkdir(parents=True) + T.setup(pristine) + labels = {ws.name: T.grade(ws)[0] for ws in wss} + for arm in arms: + t0 = time.time() + if arm == "A": + checks = [c["run"] for c in drafted_for(T, pristine, key)] + meta = {"checks": checks} + elif arm == "C": + o = product_reference_for(T, pristine, key) + checks = [o["run"]] if o.get("applies") and o.get("run") else [] + meta = o + else: + o = oracle_for(T, pristine, key, root / T.name / "oracle") + checks = [o["run"]] if o.get("applies") and o.get("run") else [] + meta = o + draft_s = round(time.time() - t0) + ok0, out0 = judge(checks, pristine) if checks else (True, []) + # Preflight: a check that reports its OWN bug (exit 3) on the + # untouched project is dropped before work, as the engine would. + if arm == "B" and checks and out0 and out0[0][0] == 3: + meta["dropped_at_preflight"] = out0[0][1][-300:] + checks = [] + ok0, out0 = True, [] + rows = [] + for ws in wss: + acc, outs = judge(checks, ws) if checks else (True, []) + rows.append({"ws": ws.name, "right": labels[ws.name], "accepted": acc, "out": [o[1][-300:] for o in outs if o[0] != 0][:1]}) + rec = {"tag": a.tag, "task": T.name, "arm": arm, "draft_s": draft_s, "n_checks": len(checks), "pristine_accepted": ok0, "pristine_out": [o[1][-200:] for o in out0][:1], "rows": rows, "meta": meta} + with open(a.out, "a") as f: + f.write(json.dumps(rec) + "\n") + wa = sum(1 for r in rows if not r["right"] and r["accepted"]) + wn = sum(1 for r in rows if not r["right"]) + rr = sum(1 for r in rows if r["right"] and not r["accepted"]) + rn = sum(1 for r in rows if r["right"]) + print(f"{T.name:22} {arm} checks={len(checks)} wrong accepted {wa}/{wn} right refused {rr}/{rn} pristine {'ACCEPTED' if ok0 else 'refused'} ({draft_s}s)", flush=True) + + +if __name__ == "__main__": + main() diff --git a/bench/local/path_rule_count.py b/bench/local/path_rule_count.py new file mode 100644 index 0000000..bdffa30 --- /dev/null +++ b/bench/local/path_rule_count.py @@ -0,0 +1,70 @@ +""" +Count, offline, how many stored drafted checks the seal-time path rule would drop. + +Every receipt under a bench cache (default ~/.cache/maat-bench) lists the +machine-checked criteria its turn sealed, as "- `name: command`" items. This +extracts them, de-duplicates by seal id + name, and runs dist/criteria.js +strayPath on each (cwd = the receipt's workspace, task text unknown so no +task-stated allowance: the count is an upper bound). + + npm run build && python3 bench/local/path_rule_count.py [CACHE_DIR] [--show] +""" +from __future__ import annotations + +import json +import re +import subprocess +import sys +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent.parent +SEAL = re.compile(r"sealed as `([0-9a-f]+)`") +ITEM = re.compile(r"^- `([a-z0-9][\w.-]*): ", re.M) + + +def checks_of(md: str) -> list[tuple[str, str]]: + """(name, command) for each item of the 'Machine-checked' list; commands may span lines.""" + start = md.find("**Machine-checked.**") + if start < 0: + return [] + body = md[start:] + end = re.search(r"\n\*\*[A-Z]", body[20:]) + body = body[: end.start() + 20] if end else body + out = [] + for m in ITEM.finditer(body): + rest = body[m.end():] + close = rest.find("`\n") + close = len(rest) if close < 0 else close + out.append((m.group(1), rest[:close].rstrip("`"))) + return out + + +def main() -> None: + args = [a for a in sys.argv[1:] if not a.startswith("--")] + cache = Path(args[0]).expanduser() if args else Path("~/.cache/maat-bench").expanduser() + seen: dict[tuple[str, str], tuple[Path, str]] = {} + for md in cache.rglob("receipts/*.md"): + text = md.read_text(errors="replace") + seal = SEAL.search(text) + ws = md.parent.parent.parent + for name, run in checks_of(text): + seen.setdefault((seal.group(1) if seal else str(md), name), (ws, run)) + rows = [{"cwd": str(ws), "run": run} for ws, run in seen.values()] + js = ( + "import {strayPath} from './dist/criteria.js';" + "let s='';for await (const c of process.stdin) s+=c;" + "console.log(JSON.stringify(JSON.parse(s).map(r=>strayPath(r.run,{cwd:r.cwd}))));" + ) + p = subprocess.run(["node", "--input-type=module", "-e", js], input=json.dumps(rows), capture_output=True, text=True, cwd=REPO) + if p.returncode != 0: + sys.exit(p.stderr[-600:]) + hits = json.loads(p.stdout) + dropped = [(r, h) for r, h in zip(rows, hits) if h] + print(f"stored drafted checks (unique per seal): {len(rows)}; the path rule would drop {len(dropped)}") + if "--show" in sys.argv: + for r, h in dropped: + print(f" {h} <- {r['run'][:100]!r} ({Path(r['cwd']).name})") + + +if __name__ == "__main__": + main() diff --git a/bench/local/reference_solutions/_wrong/async-pool-node/1/pool.js b/bench/local/reference_solutions/_wrong/async-pool-node/1/pool.js new file mode 100644 index 0000000..310a1f6 --- /dev/null +++ b/bench/local/reference_solutions/_wrong/async-pool-node/1/pool.js @@ -0,0 +1,14 @@ +"use strict"; + +// plausible mistake: process the items in fixed batches of `limit` +async function mapLimit(items, limit, fn) { + if (!Number.isInteger(limit) || limit < 1) throw new RangeError("limit must be an integer >= 1"); + const out = []; + for (let i = 0; i < items.length; i += limit) { + const batch = items.slice(i, i + limit).map((x, k) => fn(x, i + k)); + out.push(...(await Promise.all(batch))); + } + return out; +} + +module.exports = { mapLimit }; diff --git a/bench/local/reference_solutions/_wrong/async-pool-node/2/pool.js b/bench/local/reference_solutions/_wrong/async-pool-node/2/pool.js new file mode 100644 index 0000000..c815cbe --- /dev/null +++ b/bench/local/reference_solutions/_wrong/async-pool-node/2/pool.js @@ -0,0 +1,19 @@ +"use strict"; + +// plausible mistake: worker loops that keep pulling items after a failure +function mapLimit(items, limit, fn) { + if (!Number.isInteger(limit) || limit < 1) return Promise.reject(new RangeError("limit must be an integer >= 1")); + const results = new Array(items.length); + let next = 0; + const worker = async () => { + while (next < items.length) { + const i = next++; + results[i] = await fn(items[i], i); + } + }; + const workers = []; + for (let k = 0; k < Math.min(limit, items.length); k++) workers.push(worker()); + return Promise.all(workers).then(() => results); +} + +module.exports = { mapLimit }; diff --git a/bench/local/reference_solutions/_wrong/bash-backup/1/backup.sh b/bench/local/reference_solutions/_wrong/bash-backup/1/backup.sh new file mode 100644 index 0000000..75235c7 --- /dev/null +++ b/bench/local/reference_solutions/_wrong/bash-backup/1/backup.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# plausible mistake: prunes with ls -t (modification time) instead of the stamp in the name +src=$1 dest=$2 keep=$3 +[ -d "$src" ] || exit 2 +case $keep in ''|*[!0-9]*) exit 2;; esac +[ "$keep" -ge 1 ] || exit 2 +stamp=${BACKUP_NOW:-$(date +%Y%m%d-%H%M%S)} +mkdir -p "$dest" +dest_abs=$(cd "$dest" && pwd -P) +src_abs=$(cd "$src" && pwd -P) +archive="$dest_abs/backup-$stamp.tar.gz" +[ -e "$archive" ] && exit 3 +(cd "$src_abs" && find . \( -type d -name .cache -prune \) -o \( -type f ! -name '*.tmp' ! -path "./${dest_abs#$src_abs/}/*" -print0 \)) | tar -czf "$archive" -C "$src_abs" --null -T - +cd "$dest_abs" && ls -t | grep -E '^backup-[0-9]{8}-[0-9]{6}\.tar\.gz$' | tail -n +"$((keep + 1))" | while read -r f; do rm -f "$f"; done +exit 0 diff --git a/bench/local/reference_solutions/_wrong/bash-backup/2/backup.sh b/bench/local/reference_solutions/_wrong/bash-backup/2/backup.sh new file mode 100644 index 0000000..7c9b662 --- /dev/null +++ b/bench/local/reference_solutions/_wrong/bash-backup/2/backup.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# plausible mistake: --exclude patterns only, so a DEST inside SRC is archived into itself, and DEST is +# created before the arguments are fully validated +src=$1 dest=$2 keep=$3 +mkdir -p "$dest" +[ -d "$src" ] || exit 2 +case $keep in ''|*[!0-9]*) exit 2;; esac +[ "$keep" -ge 1 ] || exit 2 +stamp=${BACKUP_NOW:-$(date +%Y%m%d-%H%M%S)} +dest_abs=$(cd "$dest" && pwd -P) +archive="$dest_abs/backup-$stamp.tar.gz" +[ -e "$archive" ] && exit 3 +tar -czf "$archive" -C "$src" --exclude='*.tmp' --exclude='.cache' . 2>/dev/null +ls "$dest_abs" | grep -E '^backup-[0-9]{8}-[0-9]{6}\.tar\.gz$' | sort -r | tail -n +"$((keep + 1))" | while read -r f; do rm -f "$dest_abs/$f"; done +exit 0 diff --git a/bench/local/reference_solutions/_wrong/csv-to-json-node/1/csv2json.js b/bench/local/reference_solutions/_wrong/csv-to-json-node/1/csv2json.js new file mode 100644 index 0000000..a621a5d --- /dev/null +++ b/bench/local/reference_solutions/_wrong/csv-to-json-node/1/csv2json.js @@ -0,0 +1,29 @@ +"use strict"; +// plausible mistake: split on newlines first, then on commas (quoted commas ok, quoted newlines not), no BOM or duplicate header handling +let s = require("fs").readFileSync(0, "utf8"); +const parse = (line) => { + const out = []; + let f = "", q = false; + for (let i = 0; i < line.length; i++) { + const c = line[i]; + if (q) { + if (c === '"' && line[i + 1] === '"') { f += '"'; i++; } + else if (c === '"') q = false; + else f += c; + } else if (c === '"' && f === "") q = true; + else if (c === ",") { out.push(f); f = ""; } + else f += c; + } + out.push(f); + return out; +}; +const lines = s.split(/\r?\n/).filter((l) => l !== ""); +if (!lines.length) { console.log("[]"); process.exit(0); } +const head = parse(lines[0]); +const rows = lines.slice(1).map((l) => { + const r = parse(l); + const o = {}; + head.forEach((h, i) => (o[h] = r[i] === undefined ? "" : r[i])); + return o; +}); +console.log(JSON.stringify(rows)); diff --git a/bench/local/reference_solutions/_wrong/dedupe-contacts/1/dedupe.py b/bench/local/reference_solutions/_wrong/dedupe-contacts/1/dedupe.py new file mode 100644 index 0000000..909046c --- /dev/null +++ b/bench/local/reference_solutions/_wrong/dedupe-contacts/1/dedupe.py @@ -0,0 +1,24 @@ +import csv +import re +import sys + +# plausible mistake: groups by email first, then by phone, but not transitively across both +with open(sys.argv[1], newline="") as f: + rows = list(csv.DictReader(f)) +rows.sort(key=lambda r: int(r["id"])) +groups = [] +for r in rows: + e = r["email"].strip().lower() + p = re.sub(r"\D", "", r["phone"])[-10:] + for g in groups: + if (e and e == g["e"]) or (len(p) >= 7 and p == g["p"]): + g["rows"].append(r) + break + else: + groups.append({"e": e, "p": p, "rows": [r]}) +with open(sys.argv[2], "w", newline="") as f: + w = csv.writer(f) + w.writerow(["id", "name", "email", "phone"]) + for g in groups: + rs = g["rows"] + w.writerow([rs[0]["id"]] + [next((r[c].strip() for r in rs if r[c].strip()), "") for c in ("name", "email", "phone")]) diff --git a/bench/local/reference_solutions/_wrong/dedupe-contacts/2/dedupe.py b/bench/local/reference_solutions/_wrong/dedupe-contacts/2/dedupe.py new file mode 100644 index 0000000..63dfa48 --- /dev/null +++ b/bench/local/reference_solutions/_wrong/dedupe-contacts/2/dedupe.py @@ -0,0 +1,35 @@ +import csv +import re +import sys + +# plausible mistake: ids sorted as strings, short phones allowed to match +with open(sys.argv[1], newline="") as f: + rows = list(csv.DictReader(f)) +parent = {r["id"]: r["id"] for r in rows} + + +def find(x): + while parent[x] != x: + x = parent[x] + return x + + +seen = {} +for r in sorted(rows, key=lambda r: r["id"]): + for k in (("e", r["email"].strip().lower()), ("p", re.sub(r"\D", "", r["phone"])[-10:])): + if k[1]: + if k in seen: + parent[find(r["id"])] = find(seen[k]) + else: + seen[k] = r["id"] +groups = {} +for r in sorted(rows, key=lambda r: r["id"]): + groups.setdefault(find(r["id"]), []).append(r) +out = [] +for g in groups.values(): + out.append([g[0]["id"]] + [next((r[c].strip() for r in g if r[c].strip()), "") for c in ("name", "email", "phone")]) +out.sort(key=lambda r: r[0]) +with open(sys.argv[2], "w", newline="") as f: + w = csv.writer(f) + w.writerow(["id", "name", "email", "phone"]) + w.writerows(out) diff --git a/bench/local/reference_solutions/_wrong/fix-daily-buckets/1/bucket.py b/bench/local/reference_solutions/_wrong/fix-daily-buckets/1/bucket.py new file mode 100644 index 0000000..591f7a6 --- /dev/null +++ b/bench/local/reference_solutions/_wrong/fix-daily-buckets/1/bucket.py @@ -0,0 +1,16 @@ +from datetime import datetime, timezone + + +def to_dt(ts): + """Event timestamp (epoch seconds, or an ISO-8601 string) -> datetime.""" + if isinstance(ts, str): + return datetime.fromisoformat(ts) # still takes the written wall clock, not the UTC instant + return datetime.fromtimestamp(ts, tz=timezone.utc) + + +def day_of(ts): + return to_dt(ts).strftime("%Y-%m-%d") + + +def hour_of(ts): + return to_dt(ts).hour diff --git a/bench/local/reference_solutions/_wrong/fix-daily-buckets/2/bucket.py b/bench/local/reference_solutions/_wrong/fix-daily-buckets/2/bucket.py new file mode 100644 index 0000000..71afae4 --- /dev/null +++ b/bench/local/reference_solutions/_wrong/fix-daily-buckets/2/bucket.py @@ -0,0 +1,20 @@ +from datetime import datetime, timezone + + +def to_dt(ts): + """Event timestamp (epoch seconds, or an ISO-8601 string) -> aware UTC datetime.""" + if isinstance(ts, str): + d = datetime.fromisoformat(ts) + if d.tzinfo is None: + return d.replace(tzinfo=timezone.utc) + return d.astimezone(timezone.utc) + return datetime.fromtimestamp(ts, tz=timezone.utc) + + +def day_of(ts): + return to_dt(ts).strftime("%Y-%m-%d") + + +def hour_of(ts): + return to_dt(ts).hour +# report.py is left alone: its numeric-timestamp hours are still machine-local diff --git a/bench/local/reference_solutions/_wrong/git-find-culprit/1/solve.sh b/bench/local/reference_solutions/_wrong/git-find-culprit/1/solve.sh new file mode 100644 index 0000000..004136e --- /dev/null +++ b/bench/local/reference_solutions/_wrong/git-find-culprit/1/solve.sh @@ -0,0 +1,7 @@ +# plausible mistake: names the commit whose message says it touched limits.conf last / the parent of the culprit +git bisect start >/dev/null +git bisect bad HEAD >/dev/null +git bisect good "$(git rev-list --max-parents=0 HEAD)" >/dev/null +git bisect run sh check.sh >/dev/null +git rev-parse refs/bisect/bad~1 > culprit.txt +git bisect reset >/dev/null 2>&1 diff --git a/bench/local/reference_solutions/_wrong/git-find-culprit/2/solve.sh b/bench/local/reference_solutions/_wrong/git-find-culprit/2/solve.sh new file mode 100644 index 0000000..ff4e608 --- /dev/null +++ b/bench/local/reference_solutions/_wrong/git-find-culprit/2/solve.sh @@ -0,0 +1,6 @@ +# plausible mistake: right answer, but the bisect is left running and HEAD detached +git bisect start >/dev/null +git bisect bad HEAD >/dev/null +git bisect good "$(git rev-list --max-parents=0 HEAD)" >/dev/null +git bisect run sh check.sh >/dev/null +git rev-parse refs/bisect/bad > culprit.txt diff --git a/bench/local/reference_solutions/_wrong/git-split-lib-history/1/solve.sh b/bench/local/reference_solutions/_wrong/git-split-lib-history/1/solve.sh new file mode 100644 index 0000000..882890e --- /dev/null +++ b/bench/local/reference_solutions/_wrong/git-split-lib-history/1/solve.sh @@ -0,0 +1,6 @@ +# plausible mistake: --subdirectory-filter strips the lib/ prefix, so the paths change +export FILTER_BRANCH_SQUELCH_WARNING=1 +git branch lib-only master +git filter-branch -f --subdirectory-filter lib lib-only >/dev/null 2>&1 +rm -rf .git/refs/original +git checkout -q master diff --git a/bench/local/reference_solutions/_wrong/git-split-lib-history/2/solve.sh b/bench/local/reference_solutions/_wrong/git-split-lib-history/2/solve.sh new file mode 100644 index 0000000..388c8b0 --- /dev/null +++ b/bench/local/reference_solutions/_wrong/git-split-lib-history/2/solve.sh @@ -0,0 +1,8 @@ +# plausible mistake: empty commits are kept, so app-only commits still appear on lib-only +export FILTER_BRANCH_SQUELCH_WARNING=1 +git branch lib-only master +git filter-branch -f --index-filter ' + git ls-files | grep -v "^lib/" | xargs git rm -q --cached --ignore-unmatch +' lib-only >/dev/null 2>&1 +rm -rf .git/refs/original +git checkout -q master diff --git a/bench/local/reference_solutions/_wrong/ini-to-json/1/ini2json.py b/bench/local/reference_solutions/_wrong/ini-to-json/1/ini2json.py new file mode 100644 index 0000000..9ab5883 --- /dev/null +++ b/bench/local/reference_solutions/_wrong/ini-to-json/1/ini2json.py @@ -0,0 +1,37 @@ +import configparser +import json +import sys + +# plausible mistake: lean on configparser (no ${} across sections, inline comments only with the right +# prefixes, no typing of _list, DEFAULT handled its own way) +cp = configparser.ConfigParser(interpolation=configparser.ExtendedInterpolation(), inline_comment_prefixes=(";", "#")) +try: + cp.read(sys.argv[1], encoding="utf-8") +except configparser.Error as e: + print(e, file=sys.stderr) + sys.exit(1) + + +def conv(v): + if v.lower() in ("true", "yes", "on"): + return True + if v.lower() in ("false", "no", "off"): + return False + try: + return int(v) + except ValueError: + pass + try: + return float(v) + except ValueError: + return v + + +out = {} +for s in cp.sections(): + node = out + for part in s.split("."): + node = node.setdefault(part, {}) + for k, v in cp.items(s): + node[k] = conv(v) +json.dump(out, open(sys.argv[2], "w")) diff --git a/bench/local/reference_solutions/_wrong/ini-to-json/2/solve.sh b/bench/local/reference_solutions/_wrong/ini-to-json/2/solve.sh new file mode 100644 index 0000000..68fdd0d --- /dev/null +++ b/bench/local/reference_solutions/_wrong/ini-to-json/2/solve.sh @@ -0,0 +1,8 @@ +# plausible mistake: any ; or # starts an inline comment, not only one preceded by whitespace +cp "$REF/ini2json.py" . +python3 - <<'PY' +s = open('ini2json.py').read() +s = s.replace('re.search(r"\\s[;#]", v)', 're.search(r"[;#]", v)') +open('ini2json.py', 'w').write(s) +PY +grep -q '\[;#\]", v' ini2json.py diff --git a/bench/local/reference_solutions/_wrong/json-diff-cli/1/jdiff.py b/bench/local/reference_solutions/_wrong/json-diff-cli/1/jdiff.py new file mode 100644 index 0000000..371e5dd --- /dev/null +++ b/bench/local/reference_solutions/_wrong/json-diff-cli/1/jdiff.py @@ -0,0 +1,40 @@ +import json +import sys + +# plausible mistake: plain == (so 1 equals true), keys in file order rather than sorted, simple paths only + + +def dump(v): + return json.dumps(v, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + +def walk(path, a, b, out): + if isinstance(a, dict) and isinstance(b, dict): + for k in list(a) + [k for k in b if k not in a]: + if k not in b: + out.append(f"- {path}.{k}: {dump(a[k])}") + elif k not in a: + out.append(f"+ {path}.{k}: {dump(b[k])}") + else: + walk(f"{path}.{k}", a[k], b[k], out) + elif isinstance(a, list) and isinstance(b, list): + for i in range(max(len(a), len(b))): + if i >= len(b): + out.append(f"- {path}[{i}]: {dump(a[i])}") + elif i >= len(a): + out.append(f"+ {path}[{i}]: {dump(b[i])}") + else: + walk(f"{path}[{i}]", a[i], b[i], out) + elif a != b: + out.append(f"~ {path}: {dump(a)} -> {dump(b)}") + + +try: + a = json.load(open(sys.argv[1], encoding="utf-8")) + b = json.load(open(sys.argv[2], encoding="utf-8")) +except (OSError, ValueError): + sys.exit(2) +out = [] +walk("$", a, b, out) +sys.stdout.buffer.write(("".join(l + "\n" for l in out)).encode()) +sys.exit(1 if out else 0) diff --git a/bench/local/reference_solutions/_wrong/ledger-balance/1/ledger.py b/bench/local/reference_solutions/_wrong/ledger-balance/1/ledger.py new file mode 100644 index 0000000..29f3997 --- /dev/null +++ b/bench/local/reference_solutions/_wrong/ledger-balance/1/ledger.py @@ -0,0 +1,37 @@ +import re +import sys +from decimal import Decimal + +# plausible mistake: only leaf accounts are reported (no parent totals), floats avoided but --until ignored for +# validation, and zero-balance accounts are printed +until = sys.argv[sys.argv.index("--until") + 1] if "--until" in sys.argv else None +text = open(sys.argv[1]).read().split("\n") +totals, cur, date = {}, [], None + + +def flush(): + if not cur: + return + known = [(a, v) for a, v in cur if v is not None] + tot = sum(v for _, v in known) + for a, v in cur: + v = -tot if v is None else v + if until is None or date <= until: + totals[a] = totals.get(a, 0) + v + + +for line in text: + if line.startswith(("#", ";")) or not line.strip(): + continue + if line[0] in " \t": + m = re.match(r"\s+(.+?)(?:\s{2,}|\t+)(-?\$-?[\d,.]+)", line) + if m: + cur.append((m.group(1), Decimal(m.group(2).replace("$", "").replace(",", "")))) + else: + cur.append((line.strip().split(";")[0].strip(), None)) + else: + flush() + cur, date = [], line.split()[0] +flush() +for a in sorted(totals): + print(f"{('-$' if totals[a] < 0 else '$') + format(abs(totals[a]), ',.2f'):>12} {a}") diff --git a/bench/local/reference_solutions/_wrong/make-incremental/1/Makefile b/bench/local/reference_solutions/_wrong/make-incremental/1/Makefile new file mode 100644 index 0000000..e75b983 --- /dev/null +++ b/bench/local/reference_solutions/_wrong/make-incremental/1/Makefile @@ -0,0 +1,18 @@ +SRCS := $(sort $(wildcard src/*.md)) +OUTS := $(SRCS:src/%.md=out/%.txt) + +all: out/all.txt + +out/all.txt: $(OUTS) + cat $(OUTS) > $@ + echo "build $@" >> build.log + +out/%.txt: src/%.md + mkdir -p out + (cat common/header.txt; tr a-z A-Z < $<) > $@ + echo "build $@" >> build.log + +clean: + rm -rf out build.log + +.PHONY: all clean diff --git a/bench/local/reference_solutions/_wrong/make-incremental/2/Makefile b/bench/local/reference_solutions/_wrong/make-incremental/2/Makefile new file mode 100644 index 0000000..6ce3c5c --- /dev/null +++ b/bench/local/reference_solutions/_wrong/make-incremental/2/Makefile @@ -0,0 +1,18 @@ +SRCS := $(sort $(wildcard src/*.md)) +OUTS := $(SRCS:src/%.md=out/%.txt) + +all: out/all.txt + +out/all.txt: $(OUTS) + cat $(OUTS) > $@ + echo "build $@" >> build.log + +out/%.txt: src/%.md common/header.txt + mkdir -p out + (cat common/header.txt; tr a-z A-Z < $<) > $@ + echo "build $@" >> build.log + +clean: + rm -rf out + +.PHONY: all clean diff --git a/bench/local/reference_solutions/_wrong/money-split/1/alloc.py b/bench/local/reference_solutions/_wrong/money-split/1/alloc.py new file mode 100644 index 0000000..e227b4d --- /dev/null +++ b/bench/local/reference_solutions/_wrong/money-split/1/alloc.py @@ -0,0 +1,11 @@ +import sys +from decimal import Decimal + +# plausible mistake: round each share independently (the shares may not add up) +total = Decimal(sys.argv[1]) +ws = [int(w) for w in sys.argv[2:]] +s = sum(ws) +if not ws or s == 0: + sys.exit(1) +for w in ws: + print((total * w / s).quantize(Decimal("0.01"))) diff --git a/bench/local/reference_solutions/_wrong/money-split/2/alloc.py b/bench/local/reference_solutions/_wrong/money-split/2/alloc.py new file mode 100644 index 0000000..a8047f8 --- /dev/null +++ b/bench/local/reference_solutions/_wrong/money-split/2/alloc.py @@ -0,0 +1,15 @@ +import re +import sys +from decimal import Decimal + +# plausible mistake: leftover cents go to the first parties, not the largest remainders +total, ws = sys.argv[1], sys.argv[2:] +if not re.fullmatch(r"\d+(\.\d{1,2})?", total) or not ws or not all(re.fullmatch(r"\d+", w) for w in ws) or not any(int(w) for w in ws): + sys.exit(1) +w = [int(x) for x in ws] +cents = int(Decimal(total) * 100) +sh = [cents * x // sum(w) for x in w] +for i in range(cents - sum(sh)): + sh[i] += 1 +for c in sh: + print(f"{c // 100}.{c % 100:02d}") diff --git a/bench/local/reference_solutions/_wrong/perf-asof-join/1/join.py b/bench/local/reference_solutions/_wrong/perf-asof-join/1/join.py new file mode 100644 index 0000000..b64b6f9 --- /dev/null +++ b/bench/local/reference_solutions/_wrong/perf-asof-join/1/join.py @@ -0,0 +1,23 @@ +import bisect +import csv +import sys + +# plausible mistake: dict keyed by effective_ts keeps the FIRST row of a tie, and bisect_left misses an +# exact-match timestamp (strictly-before instead of <=) + + +def main(events_path, rates_path, out_path): + by = {} + with open(rates_path, newline="") as f: + for r in csv.DictReader(f): + by.setdefault(int(r["effective_ts"]), r["rate"]) + keys = sorted(by) + with open(events_path, newline="") as f, open(out_path, "w", newline="") as o: + w = csv.writer(o) + w.writerow(["ts", "id", "rate"]) + for e in csv.DictReader(f): + k = bisect.bisect_right(keys, int(e["ts"])) + w.writerow([e["ts"], e["id"], by[keys[k - 1]] if k else ""]) + + +main(*sys.argv[1:4]) diff --git a/bench/local/reference_solutions/_wrong/refactor-invoice/1/solve.sh b/bench/local/reference_solutions/_wrong/refactor-invoice/1/solve.sh new file mode 100644 index 0000000..2e824dd --- /dev/null +++ b/bench/local/reference_solutions/_wrong/refactor-invoice/1/solve.sh @@ -0,0 +1,8 @@ +# plausible mistake: the region is validated first, so an order with a bad region AND a bad coupon now +# reports a different error than before +cp "$REF"/*.py . +python3 - <<'PY' +s = open('invoice.py').read() +s = s.replace(' region = order["region"]\n', ' region = order["region"]\n rate_bps(region)\n', 1) +open('invoice.py', 'w').write(s) +PY diff --git a/bench/local/reference_solutions/_wrong/refactor-invoice/2/solve.sh b/bench/local/reference_solutions/_wrong/refactor-invoice/2/solve.sh new file mode 100644 index 0000000..802fea9 --- /dev/null +++ b/bench/local/reference_solutions/_wrong/refactor-invoice/2/solve.sh @@ -0,0 +1,7 @@ +# plausible mistake: "simplifying" the half-up cent rounding to round() (banker's rounding) +cp "$REF"/*.py . +python3 - <<'PY' +s = open('tax.py').read() +s = s.replace('(base_cents * rate_bps(region) + 5000) // 10000', 'round(base_cents * rate_bps(region) / 10000)') +open('tax.py', 'w').write(s) +PY diff --git a/bench/local/reference_solutions/_wrong/rename-api-signature/1/solve.sh b/bench/local/reference_solutions/_wrong/rename-api-signature/1/solve.sh new file mode 100644 index 0000000..8628c9a --- /dev/null +++ b/bench/local/reference_solutions/_wrong/rename-api-signature/1/solve.sh @@ -0,0 +1,9 @@ +# plausible mistake: a regex rename that misses functools.partial(get_user, db) (binds uid first now) +# and the alias call; textual references are all updated +cp -R "$REF/shop" "$REF/scripts" "$REF/README.md" "$REF/docs" "$REF/tests" . +python3 - <<'PY' +s = open('shop/views.py').read() +s = s.replace('functools.partial(fetch_user, db=db)', 'functools.partial(fetch_user, db)') +open('shop/views.py', 'w').write(s) +PY +grep -q 'partial(fetch_user, db)' shop/views.py diff --git a/bench/local/reference_solutions/_wrong/rename-api-signature/2/solve.sh b/bench/local/reference_solutions/_wrong/rename-api-signature/2/solve.sh new file mode 100644 index 0000000..69b2cc7 --- /dev/null +++ b/bench/local/reference_solutions/_wrong/rename-api-signature/2/solve.sh @@ -0,0 +1,4 @@ +# plausible mistake: the string in shop/plugins.json is left alone (and the old name kept as an alias) +cp -R "$REF/shop" "$REF/scripts" "$REF/README.md" "$REF/docs" "$REF/tests" . +printf '{"user_loader": "get_user", "name_loader": "get_user_by_name"}\n' > shop/plugins.json +printf '\nget_user = lambda db, uid: fetch_user(uid, db=db)\n' >> shop/db.py diff --git a/bench/local/reference_solutions/_wrong/semver-sort/1/vsort.py b/bench/local/reference_solutions/_wrong/semver-sort/1/vsort.py new file mode 100644 index 0000000..613c5db --- /dev/null +++ b/bench/local/reference_solutions/_wrong/semver-sort/1/vsort.py @@ -0,0 +1,19 @@ +import re +import sys + +# plausible mistake: prerelease identifiers compared as plain strings, build metadata part of the key +RX = re.compile(r"v?(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-([0-9A-Za-z.-]+))?(?:\+([0-9A-Za-z.-]+))?") +rows, bad = [], 0 +for line in sys.stdin.read().splitlines(): + if not line.strip(): + continue + m = RX.fullmatch(line) + if not m: + bad += 1 + continue + pre = m.group(4) + rows.append(((int(m.group(1)), int(m.group(2)), int(m.group(3)), pre is None, pre or "", m.group(5) or ""), line)) +rows.sort(key=lambda r: r[0]) +for _, l in rows: + print(l) +sys.exit(1 if bad else 0) diff --git a/bench/local/reference_solutions/_wrong/sessionize-logs/1/sessions.py b/bench/local/reference_solutions/_wrong/sessionize-logs/1/sessions.py new file mode 100644 index 0000000..f4806d4 --- /dev/null +++ b/bench/local/reference_solutions/_wrong/sessionize-logs/1/sessions.py @@ -0,0 +1,30 @@ +import csv +import re +import sys +from datetime import datetime, timedelta + +# plausible mistake: the timestamp offset is dropped (local wall clock used as if UTC), the file order is +# trusted, and a gap of exactly 1800 s starts a new session +LINE = re.compile(r'(\S+) \S+ \S+ \[([^\]]+) [+-]\d{4}\] "(\S+) (\S+) [^"]*" \d+ \S+ "[^"]*" "([^"]*)"') +sessions = {} +for line in open(sys.argv[1], encoding="utf-8"): + m = LINE.match(line) + if not m or m.group(4).startswith("/static/") or m.group(4).startswith("/favicon.ico"): + continue + t = datetime.strptime(m.group(2), "%d/%b/%Y:%H:%M:%S") + sessions.setdefault((m.group(1), m.group(5)), []).append(t) +rows = [] +for (ip, ua), ts in sessions.items(): + cur = [ts[0], ts[0], 1] + for t in ts[1:]: + if t - cur[1] >= timedelta(minutes=30): + rows.append((ip, ua, *cur)) + cur = [t, t, 1] + else: + cur[1] = t + cur[2] += 1 + rows.append((ip, ua, *cur)) +w = csv.writer(open(sys.argv[2], "w", newline="")) +w.writerow(["ip", "user_agent", "start", "end", "hits", "duration_seconds"]) +for ip, ua, a, b, c in sorted(rows, key=lambda r: (r[2], r[0], r[1])): + w.writerow([ip, ua, a.strftime("%Y-%m-%dT%H:%M:%SZ"), b.strftime("%Y-%m-%dT%H:%M:%SZ"), c, int((b - a).total_seconds())]) diff --git a/bench/local/reference_solutions/_wrong/sessionize-logs/2/solve.sh b/bench/local/reference_solutions/_wrong/sessionize-logs/2/solve.sh new file mode 100644 index 0000000..94479d4 --- /dev/null +++ b/bench/local/reference_solutions/_wrong/sessionize-logs/2/solve.sh @@ -0,0 +1,8 @@ +# plausible mistake: right except the boundary: a gap of exactly 1800 s is treated as a new session +cp "$REF/sessions.py" . +python3 - <<'PY' +s = open('sessions.py').read() +s = s.replace('<= GAP', '< GAP') +open('sessions.py', 'w').write(s) +PY +grep -q '< GAP' sessions.py diff --git a/bench/local/reference_solutions/_wrong/slugify-batch/1/slugify.py b/bench/local/reference_solutions/_wrong/slugify-batch/1/slugify.py new file mode 100644 index 0000000..417bc2c --- /dev/null +++ b/bench/local/reference_solutions/_wrong/slugify-batch/1/slugify.py @@ -0,0 +1,13 @@ +import re +import sys +import unicodedata + +# plausible mistake: ASCII-folding through encode("ascii", "ignore") (drops ß, æ, ø... instead of +# transliterating them) and suffixing duplicates with a per-slug counter that can collide +seen = {} +for line in sys.stdin.read().splitlines(): + s = unicodedata.normalize("NFKD", line.lower()).encode("ascii", "ignore").decode() + s = re.sub(r"[^a-z0-9]+", "-", s).strip("-")[:40].strip("-") or "untitled" + n = seen.get(s, 0) + seen[s] = n + 1 + print(s if n == 0 else f"{s}-{n + 1}") diff --git a/bench/local/reference_solutions/_wrong/sql-longest-gaps/1/solve.sh b/bench/local/reference_solutions/_wrong/sql-longest-gaps/1/solve.sh new file mode 100644 index 0000000..908ad10 --- /dev/null +++ b/bench/local/reference_solutions/_wrong/sql-longest-gaps/1/solve.sh @@ -0,0 +1,12 @@ +# plausible mistake: ordering readings by their text (so offsets and the T/space forms interleave wrongly) +sqlite3 -csv -header sensors.db " +WITH r AS ( + SELECT sensor, ts, LAG(ts) OVER (PARTITION BY sensor ORDER BY ts) AS prev + FROM readings WHERE value IS NOT NULL), +g AS ( + SELECT sensor, prev AS start, + CAST(ROUND((julianday(replace(replace(ts,'T',' '),'Z','')) - julianday(replace(replace(prev,'T',' '),'Z',''))) * 86400) AS INTEGER) AS gap + FROM r WHERE prev IS NOT NULL), +best AS (SELECT sensor, gap, MIN(start) AS start FROM g WHERE gap = (SELECT MAX(gap) FROM g g2 WHERE g2.sensor = g.sensor) GROUP BY sensor) +SELECT sensor, gap AS gap_seconds, replace(start,' ','T') || CASE WHEN start LIKE '%Z' THEN '' ELSE 'Z' END AS gap_start FROM best ORDER BY gap DESC, sensor; +" > longest_gaps.csv diff --git a/bench/local/reference_solutions/_wrong/sql-longest-gaps/2/solve.sh b/bench/local/reference_solutions/_wrong/sql-longest-gaps/2/solve.sh new file mode 100644 index 0000000..e252816 --- /dev/null +++ b/bench/local/reference_solutions/_wrong/sql-longest-gaps/2/solve.sh @@ -0,0 +1,26 @@ +# plausible mistake: faulty NULL readings are not ignored +python3 - <<'PY' +import csv +import sqlite3 +from datetime import datetime, timezone + +db = sqlite3.connect("sensors.db") +by = {} +for sensor, ts in db.execute("SELECT sensor, ts FROM readings"): + t = datetime.fromisoformat(ts.replace("Z", "+00:00")) + if t.tzinfo is None: + t = t.replace(tzinfo=timezone.utc) + by.setdefault(sensor, []).append(int(t.timestamp())) +rows = [] +for sensor, xs in by.items(): + xs.sort() + if len(xs) < 2: + continue + gap, start = max(((b - a), -a) for a, b in zip(xs, xs[1:])) + rows.append((sensor, gap, datetime.fromtimestamp(-start, timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"))) +rows.sort(key=lambda r: (-r[1], r[0])) +with open("longest_gaps.csv", "w", newline="") as f: + w = csv.writer(f) + w.writerow(["sensor", "gap_seconds", "gap_start"]) + w.writerows(rows) +PY diff --git a/bench/local/reference_solutions/_wrong/sqlite-migrate/1/migrate.py b/bench/local/reference_solutions/_wrong/sqlite-migrate/1/migrate.py new file mode 100644 index 0000000..51978fb --- /dev/null +++ b/bench/local/reference_solutions/_wrong/sqlite-migrate/1/migrate.py @@ -0,0 +1,28 @@ +import sqlite3 +import sys + +# plausible mistake: rename the old table out of the way, create the new one, copy, drop the old one -- +# the rename also rewrites order_items' REFERENCES orders(id) to point at the (now dropped) old table +db = sqlite3.connect(sys.argv[1], isolation_level=None) +v = db.execute("PRAGMA user_version").fetchone()[0] +if v == 2: + sys.exit(0) +if v != 1: + sys.exit(1) +db.executescript( + """ +BEGIN; +CREATE TABLE customers(id INTEGER PRIMARY KEY, email TEXT NOT NULL UNIQUE); +INSERT INTO customers(email) SELECT lower(trim(customer)) FROM orders GROUP BY lower(trim(customer)) ORDER BY min(id); +ALTER TABLE orders RENAME TO orders_old; +CREATE TABLE orders(id INTEGER PRIMARY KEY, customer_id INTEGER NOT NULL REFERENCES customers(id), + amount_cents INTEGER NOT NULL, placed_at TEXT NOT NULL, note TEXT); +INSERT INTO orders SELECT o.id, c.id, CAST(ROUND(o.amount * 100) AS INTEGER), o.placed_at, o.note + FROM orders_old o JOIN customers c ON c.email = lower(trim(o.customer)); +DROP TABLE orders_old; +CREATE INDEX idx_orders_placed ON orders(placed_at); +CREATE INDEX idx_orders_customer ON orders(customer_id); +PRAGMA user_version = 2; +COMMIT; +""" +) diff --git a/bench/local/reference_solutions/_wrong/sqlite-migrate/2/solve.sh b/bench/local/reference_solutions/_wrong/sqlite-migrate/2/solve.sh new file mode 100644 index 0000000..84bbbaf --- /dev/null +++ b/bench/local/reference_solutions/_wrong/sqlite-migrate/2/solve.sh @@ -0,0 +1,8 @@ +# plausible mistake: dollars -> cents by truncation (19.99 * 100 = 1998.999...) +cp "$REF/migrate.py" . +python3 - <<'PY' +s = open('migrate.py').read() +s = s.replace('int((Decimal(repr(amount)) * 100).quantize(Decimal(1), rounding=ROUND_HALF_UP))', 'int(amount * 100)') +open('migrate.py', 'w').write(s) +PY +grep -q 'int(amount \* 100)' migrate.py diff --git a/bench/local/reference_solutions/_wrong/text-wrap-cli/1/wrap.py b/bench/local/reference_solutions/_wrong/text-wrap-cli/1/wrap.py new file mode 100644 index 0000000..e25f058 --- /dev/null +++ b/bench/local/reference_solutions/_wrong/text-wrap-cli/1/wrap.py @@ -0,0 +1,11 @@ +import sys +import textwrap + +# plausible mistake: textwrap's defaults (splits long words and breaks at hyphens) +try: + width = int(sys.argv[1]) + assert width > 0 +except Exception: + sys.exit(2) +paras = [p for p in sys.stdin.read().split("\n\n") if p.strip()] +print("\n\n".join(textwrap.fill(" ".join(p.split()), width) for p in paras)) diff --git a/bench/local/reference_solutions/_wrong/text-wrap-cli/2/wrap.py b/bench/local/reference_solutions/_wrong/text-wrap-cli/2/wrap.py new file mode 100644 index 0000000..02df9c4 --- /dev/null +++ b/bench/local/reference_solutions/_wrong/text-wrap-cli/2/wrap.py @@ -0,0 +1,30 @@ +import sys + +# plausible mistake: str.split() treats U+00A0 and other Unicode spaces as separators +try: + width = int(sys.argv[1]) + assert width > 0 +except Exception: + sys.exit(2) +paras, cur = [], [] +for line in sys.stdin.read().split("\n"): + if not line.strip(): + if cur: + paras.append(cur) + cur = [] + else: + cur.extend(line.split()) +if cur: + paras.append(cur) +out = [] +for ws in paras: + lines, line = [], ws[0] + for w in ws[1:]: + if len(line) + 1 + len(w) <= width: + line += " " + w + else: + lines.append(line) + line = w + lines.append(line) + out.append("\n".join(lines)) +sys.stdout.write("\n\n".join(out) + ("\n" if out else "")) diff --git a/bench/local/reference_solutions/_wrong/vendor-units-workaround/1/solve.sh b/bench/local/reference_solutions/_wrong/vendor-units-workaround/1/solve.sh new file mode 100644 index 0000000..b0ff9c9 --- /dev/null +++ b/bench/local/reference_solutions/_wrong/vendor-units-workaround/1/solve.sh @@ -0,0 +1,6 @@ +# plausible mistake: "fix" the vendored table in place +python3 - <<'PY' +s = open('vendor/units.py').read() +s = s.replace('"m": 2592000, "mo"', '"m": 60, "mo"') +open('vendor/units.py', 'w').write(s) +PY diff --git a/bench/local/reference_solutions/_wrong/vendor-units-workaround/2/app/retry.py b/bench/local/reference_solutions/_wrong/vendor-units-workaround/2/app/retry.py new file mode 100644 index 0000000..56dd43c --- /dev/null +++ b/bench/local/reference_solutions/_wrong/vendor-units-workaround/2/app/retry.py @@ -0,0 +1,17 @@ +"""Retry delays.""" +import re + +from vendor import units + +# plausible mistake: findall() quietly ignores whatever it cannot read ("5 ms", "1h30", "-5m", "5M") +_PART = re.compile(r"(\d+(?:\.\d+)?)\s*([smhdw])") + + +def delay_seconds(spec): + parts = _PART.findall(spec) + if not parts: + raise ValueError("bad duration: %r" % (spec,)) + total = 0 + for n, u in parts: + total += units.parse_duration(n + ("min" if u == "m" else u)) if False else float(n) * {"s": 1, "m": 60, "h": 3600, "d": 86400, "w": 604800}[u] + return int(total) if total == int(total) else total diff --git a/bench/local/reference_solutions/async-pool-node/pool.js b/bench/local/reference_solutions/async-pool-node/pool.js new file mode 100644 index 0000000..45eba81 --- /dev/null +++ b/bench/local/reference_solutions/async-pool-node/pool.js @@ -0,0 +1,52 @@ +"use strict"; + +function mapLimit(items, limit, fn) { + return new Promise((resolve, reject) => { + if (!Number.isInteger(limit) || limit < 1) { + reject(new RangeError("limit must be an integer >= 1")); + return; + } + const n = items.length; + if (n === 0) { + resolve([]); + return; + } + const results = new Array(n); + let next = 0; + let inflight = 0; + let done = 0; + let failed = false; + const fail = (e) => { + if (!failed) { + failed = true; + reject(e); + } + }; + const pump = () => { + while (!failed && inflight < limit && next < n) { + const i = next++; + inflight++; + let p; + try { + p = Promise.resolve(fn(items[i], i)); + } catch (e) { + fail(e); + return; + } + p.then( + (v) => { + results[i] = v; + inflight--; + done++; + if (done === n) resolve(results); + else pump(); + }, + (e) => fail(e) + ); + } + }; + pump(); + }); +} + +module.exports = { mapLimit }; diff --git a/bench/local/reference_solutions/bash-backup/backup.sh b/bench/local/reference_solutions/bash-backup/backup.sh new file mode 100644 index 0000000..5b8e142 --- /dev/null +++ b/bench/local/reference_solutions/bash-backup/backup.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# usage: backup.sh SRC DEST KEEP +set -u + +usage() { echo "usage: backup.sh SRC DEST KEEP" >&2; exit 2; } +[ $# -eq 3 ] || usage +src=$1 dest=$2 keep=$3 + +[ -d "$src" ] || { echo "backup.sh: $src is not a directory" >&2; exit 2; } +case $keep in + ''|*[!0-9]*) echo "backup.sh: KEEP must be a positive integer" >&2; exit 2 ;; +esac +[ "$keep" -ge 1 ] || { echo "backup.sh: KEEP must be a positive integer" >&2; exit 2; } + +stamp=${BACKUP_NOW:-$(date +%Y%m%d-%H%M%S)} +src_abs=$(cd "$src" && pwd -P) +mkdir -p "$dest" || exit 1 +dest_abs=$(cd "$dest" && pwd -P) +archive="$dest_abs/backup-$stamp.tar.gz" +if [ -e "$archive" ]; then + echo "backup.sh: $archive already exists" >&2 + exit 3 +fi + +# a DEST inside SRC must not end up in the archive +skip=./.cache-never-matches +case $dest_abs/ in + "$src_abs"/*) skip=./${dest_abs#"$src_abs"/} ;; +esac + +tmp="$archive.partial.$$" +trap 'rm -f "$tmp"' EXIT +( + cd "$src_abs" || exit 1 + find . \( -type d \( -name .cache -o -path "$skip" \) -prune \) -o \( \( -type f -o -type l \) ! -name '*.tmp' -print0 \) +) | tar -czf "$tmp" -C "$src_abs" --null -T - || { echo "backup.sh: tar failed" >&2; exit 1; } +mv "$tmp" "$archive" || exit 1 + +# prune: newest KEEP by the stamp in the name +ls -1 "$dest_abs" | grep -E '^backup-[0-9]{8}-[0-9]{6}\.tar\.gz$' | sort -r | tail -n +"$((keep + 1))" | + while IFS= read -r f; do rm -f -- "$dest_abs/$f"; done +exit 0 diff --git a/bench/local/reference_solutions/csv-to-json-node/csv2json.js b/bench/local/reference_solutions/csv-to-json-node/csv2json.js new file mode 100644 index 0000000..35fbc9d --- /dev/null +++ b/bench/local/reference_solutions/csv-to-json-node/csv2json.js @@ -0,0 +1,78 @@ +"use strict"; +const chunks = []; +process.stdin.on("data", (c) => chunks.push(c)); +process.stdin.on("end", () => { + let s = Buffer.concat(chunks).toString("utf8"); + if (s.charCodeAt(0) === 0xfeff) s = s.slice(1); + const recs = []; + let rec = []; + let field = ""; + let fstart = true; + let inq = false; + let any = false; + for (let i = 0; i < s.length; i++) { + const c = s[i]; + if (inq) { + if (c === '"') { + if (s[i + 1] === '"') { + field += '"'; + i++; + } else inq = false; + } else field += c; + } else if (c === '"' && fstart) { + inq = true; + fstart = false; + any = true; + } else if (c === ",") { + rec.push(field); + field = ""; + fstart = true; + any = true; + } else if (c === "\n" || (c === "\r" && s[i + 1] === "\n")) { + if (c === "\r") i++; + if (any) { + rec.push(field); + recs.push(rec); + } + rec = []; + field = ""; + fstart = true; + any = false; + } else { + field += c; + fstart = false; + any = true; + } + } + const fail = (m) => { + process.stderr.write("csv2json: " + m + "\n"); + process.exit(1); + }; + if (inq) fail("unterminated quoted field"); + if (any) { + rec.push(field); + recs.push(rec); + } + if (recs.length === 0) { + process.stdout.write("[]\n"); + return; + } + const seen = new Map(); + const names = recs[0].map((h, k) => { + if (h === "") h = "col" + (k + 1); + const n = (seen.get(h) || 0) + 1; + seen.set(h, n); + return n === 1 ? h : h + "_" + n; + }); + const out = []; + for (let r = 1; r < recs.length; r++) { + const row = recs[r]; + if (row.length > names.length) fail("row " + r + " has too many fields"); + const o = {}; + names.forEach((n, k) => { + o[n] = k < row.length ? row[k] : ""; + }); + out.push(o); + } + process.stdout.write(JSON.stringify(out, null, 2) + "\n"); +}); diff --git a/bench/local/reference_solutions/dedupe-contacts/dedupe.py b/bench/local/reference_solutions/dedupe-contacts/dedupe.py new file mode 100644 index 0000000..1545beb --- /dev/null +++ b/bench/local/reference_solutions/dedupe-contacts/dedupe.py @@ -0,0 +1,48 @@ +import csv +import re +import sys + + +def main(src, dst): + with open(src, newline="") as f: + rows = list(csv.DictReader(f)) + parent = list(range(len(rows))) + + def find(x): + while parent[x] != x: + parent[x] = parent[parent[x]] + x = parent[x] + return x + + first = {} + for i, r in enumerate(rows): + keys = [] + email = r["email"].strip().lower() + if email: + keys.append(("email", email)) + digits = re.sub(r"\D", "", r["phone"]) + if len(digits) >= 7: + keys.append(("phone", digits[-10:])) + for k in keys: + if k in first: + parent[find(i)] = find(first[k]) + else: + first[k] = i + groups = {} + for i, r in enumerate(rows): + groups.setdefault(find(i), []).append(r) + out = [] + for g in groups.values(): + g.sort(key=lambda r: int(r["id"])) + merged = {"id": g[0]["id"]} + for col in ("name", "email", "phone"): + merged[col] = next((r[col].strip() for r in g if r[col].strip()), "") + out.append(merged) + out.sort(key=lambda r: int(r["id"])) + with open(dst, "w", newline="") as f: + w = csv.DictWriter(f, fieldnames=["id", "name", "email", "phone"]) + w.writeheader() + w.writerows(out) + + +main(sys.argv[1], sys.argv[2]) diff --git a/bench/local/reference_solutions/dedupe-contacts/solve.sh b/bench/local/reference_solutions/dedupe-contacts/solve.sh new file mode 100644 index 0000000..237e831 --- /dev/null +++ b/bench/local/reference_solutions/dedupe-contacts/solve.sh @@ -0,0 +1 @@ +python3 dedupe.py contacts.csv contacts_clean.csv diff --git a/bench/local/reference_solutions/fix-daily-buckets/bucket.py b/bench/local/reference_solutions/fix-daily-buckets/bucket.py new file mode 100644 index 0000000..4120ff4 --- /dev/null +++ b/bench/local/reference_solutions/fix-daily-buckets/bucket.py @@ -0,0 +1,19 @@ +from datetime import datetime, timezone + + +def to_dt(ts): + """Event timestamp (epoch seconds, or an ISO-8601 string) -> aware UTC datetime.""" + if isinstance(ts, str): + d = datetime.fromisoformat(ts) + if d.tzinfo is None: + return d.replace(tzinfo=timezone.utc) + return d.astimezone(timezone.utc) + return datetime.fromtimestamp(ts, tz=timezone.utc) + + +def day_of(ts): + return to_dt(ts).strftime("%Y-%m-%d") + + +def hour_of(ts): + return to_dt(ts).hour diff --git a/bench/local/reference_solutions/fix-daily-buckets/report.py b/bench/local/reference_solutions/fix-daily-buckets/report.py new file mode 100644 index 0000000..2601c3e --- /dev/null +++ b/bench/local/reference_solutions/fix-daily-buckets/report.py @@ -0,0 +1,27 @@ +import json +import sys +from collections import Counter, defaultdict + +import bucket + + +def main(path): + per_day = Counter() + per_hour = defaultdict(Counter) + with open(path) as f: + for line in f: + line = line.strip() + if not line: + continue + ts = json.loads(line)["ts"] + day = bucket.day_of(ts) + per_day[day] += 1 + per_hour[day][bucket.hour_of(ts)] += 1 + for day in sorted(per_day): + counts = per_hour[day] + peak = min(counts, key=lambda h: (-counts[h], h)) + print(f"{day} {per_day[day]} peak={peak:02d}") + + +if __name__ == "__main__": + main(sys.argv[1]) diff --git a/bench/local/reference_solutions/git-find-culprit/solve.sh b/bench/local/reference_solutions/git-find-culprit/solve.sh new file mode 100644 index 0000000..1ed8bb4 --- /dev/null +++ b/bench/local/reference_solutions/git-find-culprit/solve.sh @@ -0,0 +1,6 @@ +git bisect start >/dev/null +git bisect bad HEAD >/dev/null +git bisect good "$(git rev-list --max-parents=0 HEAD)" >/dev/null +git bisect run sh check.sh >/dev/null +git rev-parse refs/bisect/bad > culprit.txt +git bisect reset >/dev/null 2>&1 diff --git a/bench/local/reference_solutions/git-split-lib-history/solve.sh b/bench/local/reference_solutions/git-split-lib-history/solve.sh new file mode 100644 index 0000000..9c1cf0c --- /dev/null +++ b/bench/local/reference_solutions/git-split-lib-history/solve.sh @@ -0,0 +1,7 @@ +export FILTER_BRANCH_SQUELCH_WARNING=1 +git branch lib-only master +git filter-branch -f --prune-empty --index-filter ' + git ls-files | grep -v "^lib/" | xargs git rm -q --cached --ignore-unmatch +' lib-only >/dev/null 2>&1 +rm -rf .git/refs/original +git checkout -q master diff --git a/bench/local/reference_solutions/ini-to-json/ini2json.py b/bench/local/reference_solutions/ini-to-json/ini2json.py new file mode 100644 index 0000000..8a6266c --- /dev/null +++ b/bench/local/reference_solutions/ini-to-json/ini2json.py @@ -0,0 +1,105 @@ +import json +import re +import sys + + +class IniError(Exception): + pass + + +def parse_value(v): + if v.startswith('"'): + end = v.find('"', 1) + if end == -1: + raise IniError("unterminated quote") + rest = v[end + 1:].strip() + if rest and rest[0] not in ";#": + raise IniError("text after closing quote") + return v[1:end], True + if v and v[0] in ";#": + return "", False + m = re.search(r"\s[;#]", v) + if m: + v = v[:m.start()].rstrip() + return v, False + + +def load(path): + root, sections, defaults = {}, {}, {} + cur = None + with open(path, encoding="utf-8") as f: + lines = f.read().splitlines() + for raw in lines: + line = raw.strip() + if not line or line[0] in ";#": + continue + m = re.fullmatch(r"\[([^\]]+)\]", line) + if m: + cur = m.group(1).strip() + if cur != "DEFAULT": + sections.setdefault(cur, {}) + continue + if "=" not in line: + raise IniError(f"bad line: {line!r}") + key, _, val = line.partition("=") + target = root if cur is None else defaults if cur == "DEFAULT" else sections[cur] + target[key.strip()] = parse_value(val.strip()) + merged = {name: {**defaults, **body} for name, body in sections.items()} + merged[""] = root + return merged + + +def resolve(merged, sec, key, stack=()): + if (sec, key) in stack: + raise IniError(f"reference cycle at {key}") + if sec not in merged or key not in merged[sec]: + raise IniError(f"unresolved reference {sec}.{key}") + text, _ = merged[sec][key] + + def sub(m): + ref = m.group(1) + s2, k2 = ref.rsplit(".", 1) if "." in ref else (sec, ref) + return resolve(merged, s2, k2, stack + ((sec, key),)) + + return re.sub(r"\$\{([^}]*)\}", sub, text) + + +def convert(key, text, quoted): + if quoted: + return text + if key.endswith("_list"): + return [x.strip() for x in text.split(",")] if text.strip() else [] + if re.fullmatch(r"-?\d+", text): + return int(text) + if re.fullmatch(r"-?\d+\.\d+", text): + return float(text) + low = text.lower() + if low in ("true", "yes", "on"): + return True + if low in ("false", "no", "off"): + return False + if low == "null": + return None + return text + + +def main(src, dst): + try: + merged = load(src) + out = {} + for sec, body in merged.items(): + node = out + if sec: + for part in sec.split("."): + node = node.setdefault(part, {}) + for key, (_, quoted) in body.items(): + node[key] = convert(key, resolve(merged, sec, key), quoted) + except (IniError, OSError) as e: + print(f"ini2json: {e}", file=sys.stderr) + sys.exit(1) + with open(dst, "w", encoding="utf-8") as f: + json.dump(out, f, indent=2) + f.write("\n") + + +main(sys.argv[1], sys.argv[2]) diff --git a/bench/local/reference_solutions/json-diff-cli/jdiff.py b/bench/local/reference_solutions/json-diff-cli/jdiff.py new file mode 100644 index 0000000..71de1ae --- /dev/null +++ b/bench/local/reference_solutions/json-diff-cli/jdiff.py @@ -0,0 +1,62 @@ +import json +import re +import sys + + +def dump(v): + return json.dumps(v, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + +def seg(k): + return "." + k if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", k) else "[" + json.dumps(k, ensure_ascii=False) + "]" + + +def is_num(x): + return isinstance(x, (int, float)) and not isinstance(x, bool) + + +def equal_scalars(x, y): + if is_num(x) or is_num(y): + return is_num(x) and is_num(y) and x == y + return type(x) is type(y) and x == y + + +def walk(path, a, b, out): + if isinstance(a, dict) and isinstance(b, dict): + for k in sorted(set(a) | set(b)): + if k not in b: + out.append(f"- {path}{seg(k)}: {dump(a[k])}") + elif k not in a: + out.append(f"+ {path}{seg(k)}: {dump(b[k])}") + else: + walk(path + seg(k), a[k], b[k], out) + elif isinstance(a, list) and isinstance(b, list): + for i in range(max(len(a), len(b))): + if i >= len(b): + out.append(f"- {path}[{i}]: {dump(a[i])}") + elif i >= len(a): + out.append(f"+ {path}[{i}]: {dump(b[i])}") + else: + walk(f"{path}[{i}]", a[i], b[i], out) + elif not equal_scalars(a, b): + out.append(f"~ {path}: {dump(a)} -> {dump(b)}") + + +def load(p): + with open(p, encoding="utf-8") as f: + return json.load(f) + + +def main(): + try: + a, b = load(sys.argv[1]), load(sys.argv[2]) + except (OSError, ValueError) as e: + print(f"jdiff: {e}", file=sys.stderr) + return 2 + out = [] + walk("$", a, b, out) + sys.stdout.buffer.write(("".join(l + "\n" for l in out)).encode("utf-8")) + return 1 if out else 0 + + +sys.exit(main()) diff --git a/bench/local/reference_solutions/ledger-balance/ledger.py b/bench/local/reference_solutions/ledger-balance/ledger.py new file mode 100644 index 0000000..863c365 --- /dev/null +++ b/bench/local/reference_solutions/ledger-balance/ledger.py @@ -0,0 +1,93 @@ +import re +import sys + +AMOUNT = re.compile(r"(-?)\$(-?)(\d{1,3}(?:,\d{3})+|\d+)(?:\.(\d{1,2}))?") +DATE = re.compile(r"(\d{4}-\d\d-\d\d)(?: |$)") + + +class JournalError(Exception): + pass + + +def parse_amount(text, line): + m = AMOUNT.fullmatch(text) + if not m or (m.group(1) and m.group(2)): + raise JournalError(f"line {line}: malformed amount {text!r}") + cents = int(m.group(3).replace(",", "")) * 100 + int((m.group(4) or "0").ljust(2, "0")) + return -cents if (m.group(1) or m.group(2)) else cents + + +def money(cents): + s = f"${abs(cents) // 100:,}.{abs(cents) % 100:02d}" + return "-" + s if cents < 0 else s + + +def read(path): + txns, cur = [], None + with open(path, encoding="utf-8") as f: + for no, raw in enumerate(f.read().split("\n"), 1): + if not raw.strip() or raw[0] in ";#": + continue + if raw[0] in " \t": + if cur is not None and not raw.strip().startswith(";"): + cur["lines"].append(raw.strip()) + continue + m = DATE.match(raw) + if m: + cur = {"line": no, "date": m.group(1), "lines": []} + txns.append(cur) + return txns + + +def postings(txn): + out, missing, total = [], None, 0 + for content in txn["lines"]: + m = re.search(r"\t+| {2,}", content) + account, rest = (content[:m.start()], content[m.end():]) if m else (content, "") + rest = rest.split(";")[0].strip() + if not rest: + if missing is not None: + raise JournalError(f"line {txn['line']}: more than one posting without an amount") + missing = len(out) + out.append([account, 0]) + else: + cents = parse_amount(rest, txn["line"]) + total += cents + out.append([account, cents]) + if missing is not None: + out[missing][1] = -total + total = 0 + if total != 0: + raise JournalError(f"line {txn['line']}: transaction does not balance (off by {money(total)})") + return out + + +def main(argv): + until = None + args = list(argv) + if "--until" in args: + i = args.index("--until") + until = args[i + 1] + del args[i:i + 2] + try: + txns = read(args[0]) + lines = [(t["date"], postings(t)) for t in txns] + except JournalError as e: + print(e, file=sys.stderr) + return 1 + totals = {} + for date, ps in lines: + if until is not None and date > until: + continue + for account, cents in ps: + parts = account.split(":") + for i in range(1, len(parts) + 1): + key = ":".join(parts[:i]) + totals[key] = totals.get(key, 0) + cents + for key in sorted(totals): + if totals[key] != 0: + print(f"{money(totals[key]):>12} {key}") + return 0 + + +sys.exit(main(sys.argv[1:])) diff --git a/bench/local/reference_solutions/make-incremental/Makefile b/bench/local/reference_solutions/make-incremental/Makefile new file mode 100644 index 0000000..0647bec --- /dev/null +++ b/bench/local/reference_solutions/make-incremental/Makefile @@ -0,0 +1,18 @@ +SRCS := $(sort $(wildcard src/*.md)) +OUTS := $(SRCS:src/%.md=out/%.txt) + +all: out/all.txt + +out/all.txt: $(OUTS) + cat $(OUTS) > $@ + echo "build $@" >> build.log + +out/%.txt: src/%.md common/header.txt + mkdir -p out + (cat common/header.txt; tr a-z A-Z < $<) > $@ + echo "build $@" >> build.log + +clean: + rm -rf out build.log + +.PHONY: all clean diff --git a/bench/local/reference_solutions/money-split/alloc.py b/bench/local/reference_solutions/money-split/alloc.py new file mode 100644 index 0000000..4621018 --- /dev/null +++ b/bench/local/reference_solutions/money-split/alloc.py @@ -0,0 +1,35 @@ +import re +import sys +from decimal import Decimal + + +def fail(msg): + print(msg, file=sys.stderr) + sys.exit(1) + + +def main(argv): + if len(argv) < 2: + fail("usage: alloc.py TOTAL W1 W2 ...") + total, ws = argv[0], argv[1:] + if not re.fullmatch(r"\d+(\.\d{1,2})?", total): + fail(f"bad total: {total}") + if not ws: + fail("no weights") + if not all(re.fullmatch(r"\d+", w) for w in ws): + fail("weights must be non-negative integers") + weights = [int(w) for w in ws] + s = sum(weights) + if s == 0: + fail("weights are all zero") + cents = int(Decimal(total) * 100) + shares = [cents * w // s for w in weights] + rems = [cents * w % s for w in weights] + left = cents - sum(shares) + for i in sorted(range(len(weights)), key=lambda i: (-rems[i], i))[:left]: + shares[i] += 1 + for c in shares: + print(f"{c // 100}.{c % 100:02d}") + + +main(sys.argv[1:]) diff --git a/bench/local/reference_solutions/perf-asof-join/join.py b/bench/local/reference_solutions/perf-asof-join/join.py new file mode 100644 index 0000000..60a65ca --- /dev/null +++ b/bench/local/reference_solutions/perf-asof-join/join.py @@ -0,0 +1,21 @@ +import bisect +import csv +import sys + + +def main(events_path, rates_path, out_path): + rates = [] + with open(rates_path, newline="") as f: + for i, r in enumerate(csv.DictReader(f)): + rates.append((int(r["effective_ts"]), i, r["rate"])) + rates.sort() # by effective_ts, then file position: the later row wins a tie + keys = [r[0] for r in rates] + with open(events_path, newline="") as f, open(out_path, "w", newline="") as o: + w = csv.writer(o) + w.writerow(["ts", "id", "rate"]) + for e in csv.DictReader(f): + k = bisect.bisect_right(keys, int(e["ts"])) + w.writerow([e["ts"], e["id"], rates[k - 1][2] if k else ""]) + + +main(*sys.argv[1:4]) diff --git a/bench/local/reference_solutions/refactor-invoice/format.py b/bench/local/reference_solutions/refactor-invoice/format.py new file mode 100644 index 0000000..9cfbd09 --- /dev/null +++ b/bench/local/reference_solutions/refactor-invoice/format.py @@ -0,0 +1,11 @@ +EU = ("DE", "FR") + + +def money(cents, region): + neg = cents < 0 + whole, frac = divmod(abs(cents), 100) + if region in EU: + s = f"{whole:,}".replace(",", ".") + "," + f"{frac:02d}" + " €" + else: + s = "$" + f"{whole:,}" + "." + f"{frac:02d}" + return "-" + s if neg else s diff --git a/bench/local/reference_solutions/refactor-invoice/invoice.py b/bench/local/reference_solutions/refactor-invoice/invoice.py new file mode 100644 index 0000000..55617bc --- /dev/null +++ b/bench/local/reference_solutions/refactor-invoice/invoice.py @@ -0,0 +1,42 @@ +from format import EU, money as fmt_money +from tax import compute_tax, rate_bps + + +def _discount(sub, coupon): + if coupon == "SAVE10": + return sub * 10 // 100 + if coupon == "FIVER": + return 500 if sub >= 2000 else 0 + if coupon is not None and coupon != "": + raise ValueError("unknown coupon: " + str(coupon)) + return 0 + + +def render(order): + region = order["region"] + lines = order["lines"] + sub = sum(ln["qty"] * ln["cents"] for ln in lines) + coupon = order.get("coupon") + disc = _discount(sub, coupon) + base = sub - disc + tax = compute_tax(base, region) + ship = (0 if base >= 5000 else 599) + (300 if region in EU else 0) + total = base + tax + ship + + def money(c): + return fmt_money(c, region) + + out = ["INVOICE " + str(order["id"]), "Customer: " + order["customer"], "Region: " + region, "-" * 44] + for ln in lines: + desc = ln["desc"] + if len(desc) > 20: + desc = desc[:19] + "…" + out.append(f"{desc:<20} {ln['qty']:>3} x {money(ln['cents']):>10} {money(ln['qty'] * ln['cents']):>12}") + out.append("-" * 44) + out.append(f"{'Subtotal':<20}{money(sub):>24}") + if disc: + out.append(f"{'Discount (' + coupon + ')':<20}{money(-disc):>24}") + out.append(f"{'Tax (' + format(rate_bps(region) / 100, 'g') + '%)':<20}{money(tax):>24}") + out.append(f"{'Shipping':<20}{money(ship):>24}") + out.append(f"{'TOTAL':<20}{money(total):>24}") + return "\n".join(out) + "\n" diff --git a/bench/local/reference_solutions/refactor-invoice/tax.py b/bench/local/reference_solutions/refactor-invoice/tax.py new file mode 100644 index 0000000..2e881f4 --- /dev/null +++ b/bench/local/reference_solutions/refactor-invoice/tax.py @@ -0,0 +1,12 @@ +REGIONS = {"US-CA": 725, "US-NY": 400, "US-OR": 0, "DE": 1900, "FR": 2000} + + +def rate_bps(region): + if region not in REGIONS: + raise ValueError("unknown region: " + str(region)) + return REGIONS[region] + + +def compute_tax(base_cents, region): + """Tax in cents on base_cents, rounded half up (floor division, so also for credits).""" + return (base_cents * rate_bps(region) + 5000) // 10000 diff --git a/bench/local/reference_solutions/rename-api-signature/README.md b/bench/local/reference_solutions/rename-api-signature/README.md new file mode 100644 index 0000000..4b5c9f8 --- /dev/null +++ b/bench/local/reference_solutions/rename-api-signature/README.md @@ -0,0 +1,3 @@ +# shop + +Use `shop.db.fetch_user(uid, *, db)` to load a user; `get_user_by_name(db, name)` looks one up by name. diff --git a/bench/local/reference_solutions/rename-api-signature/docs/API.md b/bench/local/reference_solutions/rename-api-signature/docs/API.md new file mode 100644 index 0000000..f47921c --- /dev/null +++ b/bench/local/reference_solutions/rename-api-signature/docs/API.md @@ -0,0 +1,11 @@ +# API + +## fetch_user(uid, *, db) + +Returns a dict `{id, name, email}` or `None`. + +Example: `u = fetch_user(5, db=conn)` + +## get_user_by_name(db, name) + +Same, looked up by name. diff --git a/bench/local/reference_solutions/rename-api-signature/scripts/audit.py b/bench/local/reference_solutions/rename-api-signature/scripts/audit.py new file mode 100644 index 0000000..6c4e8c9 --- /dev/null +++ b/bench/local/reference_solutions/rename-api-signature/scripts/audit.py @@ -0,0 +1,11 @@ +"""Report users that look incomplete.""" +from shop.db import fetch_user + + +def audit(conn, ids): + bad = [] + for uid in ids: + u = fetch_user(uid, db=conn) + if u is None or not u["email"]: + bad.append(uid) + return bad diff --git a/bench/local/reference_solutions/rename-api-signature/shop/billing.py b/bench/local/reference_solutions/rename-api-signature/shop/billing.py new file mode 100644 index 0000000..f285c80 --- /dev/null +++ b/bench/local/reference_solutions/rename-api-signature/shop/billing.py @@ -0,0 +1,14 @@ +from shop.db import fetch_user as _fu + + +def invoice_owner(db, order): + owner = _fu( + order["user_id"], + db=db, + ) + return owner["name"] if owner else None + + +def total_for(db, orders): + # fetch_user is cheap, no caching needed + return sum(o["cents"] for o in orders if _fu(o["user_id"], db=db)) diff --git a/bench/local/reference_solutions/rename-api-signature/shop/db.py b/bench/local/reference_solutions/rename-api-signature/shop/db.py new file mode 100644 index 0000000..550cf0e --- /dev/null +++ b/bench/local/reference_solutions/rename-api-signature/shop/db.py @@ -0,0 +1,20 @@ +import sqlite3 + + +def connect(path=":memory:"): + conn = sqlite3.connect(path) + conn.execute("CREATE TABLE IF NOT EXISTS users(id INTEGER PRIMARY KEY, name TEXT, email TEXT)") + return conn + + +def fetch_user(uid, *, db): + """Return the user with this id as a dict, or None.""" + row = db.execute("SELECT id, name, email FROM users WHERE id = ?", (uid,)).fetchone() + if row is None: + return None + return {"id": row[0], "name": row[1], "email": row[2]} + + +def get_user_by_name(db, name): + row = db.execute("SELECT id FROM users WHERE name = ?", (name,)).fetchone() + return None if row is None else fetch_user(row[0], db=db) diff --git a/bench/local/reference_solutions/rename-api-signature/shop/notify.py b/bench/local/reference_solutions/rename-api-signature/shop/notify.py new file mode 100644 index 0000000..847dc6f --- /dev/null +++ b/bench/local/reference_solutions/rename-api-signature/shop/notify.py @@ -0,0 +1,10 @@ +import shop.db as sdb + + +def welcome(db, uid): + user = sdb.fetch_user(uid, db=db) + return "Welcome, %s!" % user["name"] if user else "Welcome!" + + +def by_keyword(db, uid): + return sdb.fetch_user(uid=uid, db=db) diff --git a/bench/local/reference_solutions/rename-api-signature/shop/plugins.json b/bench/local/reference_solutions/rename-api-signature/shop/plugins.json new file mode 100644 index 0000000..7c7f750 --- /dev/null +++ b/bench/local/reference_solutions/rename-api-signature/shop/plugins.json @@ -0,0 +1 @@ +{"user_loader": "fetch_user", "name_loader": "get_user_by_name"} diff --git a/bench/local/reference_solutions/rename-api-signature/shop/views.py b/bench/local/reference_solutions/rename-api-signature/shop/views.py new file mode 100644 index 0000000..51fb309 --- /dev/null +++ b/bench/local/reference_solutions/rename-api-signature/shop/views.py @@ -0,0 +1,19 @@ +import functools + +from shop.db import fetch_user + + +def profile(db, uid): + u = fetch_user(uid, db=db) + if u is None: + return "unknown" + return "%s <%s>" % (u["name"], u["email"]) + + +def profiles(db, uids): + return [profile(db, u) for u in uids if fetch_user(u, db=db)] + + +def make_loader(db): + """A one-argument loader bound to this connection.""" + return functools.partial(fetch_user, db=db) diff --git a/bench/local/reference_solutions/rename-api-signature/tests/test_shop.py b/bench/local/reference_solutions/rename-api-signature/tests/test_shop.py new file mode 100644 index 0000000..a1cb5ba --- /dev/null +++ b/bench/local/reference_solutions/rename-api-signature/tests/test_shop.py @@ -0,0 +1,36 @@ +import unittest + +from shop import billing, db, notify, registry, views +from shop.db import fetch_user + + +class ShopTests(unittest.TestCase): + def setUp(self): + self.conn = db.connect() + self.conn.execute("INSERT INTO users VALUES (1, 'Ada', 'ada@x.org')") + self.conn.execute("INSERT INTO users VALUES (2, 'Ben', '')") + + def test_fetch_user(self): + self.assertEqual(fetch_user(1, db=self.conn)["name"], "Ada") + self.assertIsNone(fetch_user(9, db=self.conn)) + + def test_by_name(self): + self.assertEqual(db.get_user_by_name(self.conn, "Ben")["id"], 2) + + def test_views(self): + self.assertEqual(views.profile(self.conn, 1), "Ada ") + self.assertEqual(views.profiles(self.conn, [1, 2, 3]), ["Ada ", "Ben <>"]) + self.assertEqual(views.make_loader(self.conn)(2)["name"], "Ben") + + def test_billing(self): + self.assertEqual(billing.invoice_owner(self.conn, {"user_id": 1}), "Ada") + self.assertEqual(billing.total_for(self.conn, [{"user_id": 1, "cents": 5}, {"user_id": 7, "cents": 9}]), 5) + + def test_notify_and_registry(self): + self.assertEqual(notify.welcome(self.conn, 1), "Welcome, Ada!") + self.assertEqual(notify.by_keyword(self.conn, 2)["name"], "Ben") + self.assertEqual(registry.load("user_loader")(1, db=self.conn)["email"], "ada@x.org") + + +if __name__ == "__main__": + unittest.main() diff --git a/bench/local/reference_solutions/semver-sort/vsort.py b/bench/local/reference_solutions/semver-sort/vsort.py new file mode 100644 index 0000000..1c2d75c --- /dev/null +++ b/bench/local/reference_solutions/semver-sort/vsort.py @@ -0,0 +1,39 @@ +import re +import sys + +ID = r"(?:0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*)" +RX = re.compile( + r"v?(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)" + r"(?:-(" + ID + r"(?:\." + ID + r")*))?" + r"(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?" +) + + +def key(m): + core = (int(m.group(1)), int(m.group(2)), int(m.group(3))) + pre = m.group(4) + if pre is None: + return core + (1, ()) + ids = tuple((0, int(x), "") if x.isdigit() else (1, 0, x) for x in pre.split(".")) + return core + (0, ids) + + +def main(): + good, bad = [], 0 + for line in sys.stdin.read().split("\n"): + line = line.rstrip("\r") + if not line.strip(): + continue + m = RX.fullmatch(line) + if m is None: + print(f"invalid version: {line!r}", file=sys.stderr) + bad += 1 + continue + good.append((key(m), line)) + good.sort(key=lambda kv: kv[0]) # stable + for _, line in good: + print(line) + return 1 if bad else 0 + + +sys.exit(main()) diff --git a/bench/local/reference_solutions/sessionize-logs/sessions.py b/bench/local/reference_solutions/sessionize-logs/sessions.py new file mode 100644 index 0000000..2ba238c --- /dev/null +++ b/bench/local/reference_solutions/sessionize-logs/sessions.py @@ -0,0 +1,60 @@ +import csv +import re +import sys +from datetime import datetime, timedelta, timezone + +MONTHS = {m: i for i, m in enumerate("Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec".split(), 1)} +LINE = re.compile( + r'(\S+) \S+ \S+ \[(\d\d)/(\w{3})/(\d{4}):(\d\d):(\d\d):(\d\d) ([+-])(\d\d)(\d\d)\] "([^"]*)" \d{3} (?:\d+|-) "[^"]*" "([^"]*)"' +) +GAP = 1800 + + +def parse(line): + m = LINE.fullmatch(line) + if not m: + return None + ip, dd, mon, yy, hh, mi, ss, sign, oh, om, request, ua = m.groups() + parts = request.split(" ") + if len(parts) != 3 or mon not in MONTHS: + return None + path = parts[1].split("?")[0] + if path.startswith("/static/") or path == "/favicon.ico": + return None + offset = timedelta(hours=int(oh), minutes=int(om)) + if sign == "-": + offset = -offset + t = datetime(int(yy), MONTHS[mon], int(dd), int(hh), int(mi), int(ss), tzinfo=timezone(offset)) + return ip, ua, t.astimezone(timezone.utc) + + +def main(src, dst): + hits = {} + with open(src, encoding="utf-8", errors="replace") as f: + for n, line in enumerate(f): + p = parse(line.rstrip("\r\n")) + if p: + hits.setdefault((p[0], p[1]), []).append((p[2], n)) + rows = [] + for (ip, ua), hs in hits.items(): + hs.sort() + cur = None + for t, _ in hs: + if cur is not None and (t - cur[1]).total_seconds() <= GAP: + cur[1] = t + cur[2] += 1 + else: + if cur: + rows.append((ip, ua, *cur)) + cur = [t, t, 1] + rows.append((ip, ua, *cur)) + fmt = "%Y-%m-%dT%H:%M:%SZ" + out = [(ip, ua, a.strftime(fmt), b.strftime(fmt), c, int((b - a).total_seconds())) for ip, ua, a, b, c in rows] + out.sort(key=lambda r: (r[2], r[0], r[1])) + with open(dst, "w", newline="", encoding="utf-8") as f: + w = csv.writer(f) + w.writerow(["ip", "user_agent", "start", "end", "hits", "duration_seconds"]) + w.writerows(out) + + +main(sys.argv[1], sys.argv[2]) diff --git a/bench/local/reference_solutions/slugify-batch/slugify.py b/bench/local/reference_solutions/slugify-batch/slugify.py new file mode 100644 index 0000000..bf87fdf --- /dev/null +++ b/bench/local/reference_solutions/slugify-batch/slugify.py @@ -0,0 +1,39 @@ +import re +import sys +import unicodedata + +TABLE = [("ß", "ss"), ("æ", "ae"), ("œ", "oe"), ("ø", "o"), ("ð", "d"), ("đ", "d"), ("þ", "th"), ("ł", "l")] + + +def slug(text): + text = text.lower() + for a, b in TABLE: + text = text.replace(a, b) + text = unicodedata.normalize("NFKD", text) + text = "".join(c for c in text if not unicodedata.combining(c)) + text = re.sub(r"[^a-z0-9]+", "-", text).strip("-") + text = text[:40].rstrip("-") + return text or "untitled" + + +def main(): + data = sys.stdin.buffer.read().decode("utf-8") + lines = data.split("\n") + if lines and lines[-1] == "": + lines.pop() + taken = set() + out = [] + for line in lines: + if line.endswith("\r"): + line = line[:-1] + base = slug(line) + cand, n = base, 1 + while cand in taken: + n += 1 + cand = f"{base}-{n}" + taken.add(cand) + out.append(cand) + sys.stdout.buffer.write(("".join(s + "\n" for s in out)).encode("utf-8")) + + +main() diff --git a/bench/local/reference_solutions/sql-longest-gaps/solve.sh b/bench/local/reference_solutions/sql-longest-gaps/solve.sh new file mode 100644 index 0000000..0a68f77 --- /dev/null +++ b/bench/local/reference_solutions/sql-longest-gaps/solve.sh @@ -0,0 +1,26 @@ +python3 - <<'PY' +import csv +import sqlite3 +from datetime import datetime, timezone + +db = sqlite3.connect("sensors.db") +by = {} +for sensor, ts in db.execute("SELECT sensor, ts FROM readings WHERE value IS NOT NULL"): + t = datetime.fromisoformat(ts.replace("Z", "+00:00")) + if t.tzinfo is None: + t = t.replace(tzinfo=timezone.utc) + by.setdefault(sensor, []).append(int(t.timestamp())) +rows = [] +for sensor, xs in by.items(): + xs.sort() + if len(xs) < 2: + continue + gap, start = max(((b - a), -a) for a, b in zip(xs, xs[1:])) + start = -start + rows.append((sensor, gap, datetime.fromtimestamp(start, timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"))) +rows.sort(key=lambda r: (-r[1], r[0])) +with open("longest_gaps.csv", "w", newline="") as f: + w = csv.writer(f) + w.writerow(["sensor", "gap_seconds", "gap_start"]) + w.writerows(rows) +PY diff --git a/bench/local/reference_solutions/sqlite-migrate/migrate.py b/bench/local/reference_solutions/sqlite-migrate/migrate.py new file mode 100644 index 0000000..4f34ae5 --- /dev/null +++ b/bench/local/reference_solutions/sqlite-migrate/migrate.py @@ -0,0 +1,50 @@ +import sqlite3 +import sys +from decimal import ROUND_HALF_UP, Decimal + + +def migrate(db): + db.execute("PRAGMA foreign_keys = OFF") + db.execute("BEGIN") + try: + db.execute("CREATE TABLE customers(id INTEGER PRIMARY KEY, email TEXT NOT NULL UNIQUE)") + db.execute( + "CREATE TABLE orders_v2(id INTEGER PRIMARY KEY, customer_id INTEGER NOT NULL REFERENCES customers(id), " + "amount_cents INTEGER NOT NULL, placed_at TEXT NOT NULL, note TEXT)" + ) + ids = {} + for oid, customer, amount, placed_at, note in db.execute( + "SELECT id, customer, amount, placed_at, note FROM orders ORDER BY id" + ).fetchall(): + email = customer.strip().lower() + if email not in ids: + ids[email] = len(ids) + 1 + db.execute("INSERT INTO customers(id, email) VALUES (?, ?)", (ids[email], email)) + cents = int((Decimal(repr(amount)) * 100).quantize(Decimal(1), rounding=ROUND_HALF_UP)) + db.execute("INSERT INTO orders_v2 VALUES (?, ?, ?, ?, ?)", (oid, ids[email], cents, placed_at, note)) + db.execute("DROP TABLE orders") + db.execute("ALTER TABLE orders_v2 RENAME TO orders") + db.execute("CREATE INDEX idx_orders_placed ON orders(placed_at)") + db.execute("CREATE INDEX idx_orders_customer ON orders(customer_id)") + if db.execute("PRAGMA foreign_key_check").fetchall(): + raise RuntimeError("foreign key violations after migration") + db.execute("PRAGMA user_version = 2") + db.execute("COMMIT") + except BaseException: + db.execute("ROLLBACK") + raise + + +def main(path): + db = sqlite3.connect(path, isolation_level=None) + version = db.execute("PRAGMA user_version").fetchone()[0] + if version == 2: + return 0 + if version != 1: + print(f"migrate.py: unsupported schema version {version}", file=sys.stderr) + return 1 + migrate(db) + return 0 + + +sys.exit(main(sys.argv[1])) diff --git a/bench/local/reference_solutions/text-wrap-cli/wrap.py b/bench/local/reference_solutions/text-wrap-cli/wrap.py new file mode 100644 index 0000000..d6d5b4f --- /dev/null +++ b/bench/local/reference_solutions/text-wrap-cli/wrap.py @@ -0,0 +1,42 @@ +import re +import sys + +ASCII_WS = " \t\r\n\f\v" + + +def usage(): + print("usage: wrap.py WIDTH (WIDTH a positive integer)", file=sys.stderr) + sys.exit(2) + + +def main(): + if len(sys.argv) != 2 or not re.fullmatch(r"[0-9]+", sys.argv[1]) or int(sys.argv[1]) < 1: + usage() + width = int(sys.argv[1]) + text = sys.stdin.buffer.read().decode("utf-8") + paragraphs, words = [], [] + for line in text.split("\n"): + line_words = [w for w in re.split("[" + ASCII_WS + "]+", line) if w] + if not line_words: + if words: + paragraphs.append(words) + words = [] + else: + words.extend(line_words) + if words: + paragraphs.append(words) + rendered = [] + for ws in paragraphs: + lines, cur = [], ws[0] + for w in ws[1:]: + if len(cur) + 1 + len(w) <= width: + cur += " " + w + else: + lines.append(cur) + cur = w + lines.append(cur) + rendered.append("\n".join(lines)) + sys.stdout.buffer.write(("\n\n".join(rendered) + "\n" if rendered else "").encode("utf-8")) + + +main() diff --git a/bench/local/reference_solutions/vendor-units-workaround/app/retry.py b/bench/local/reference_solutions/vendor-units-workaround/app/retry.py new file mode 100644 index 0000000..d8832f2 --- /dev/null +++ b/bench/local/reference_solutions/vendor-units-workaround/app/retry.py @@ -0,0 +1,29 @@ +"""Retry delays.""" +import re + +# The vendored vendor.units reads "m" as a month, so it is not used for durations any more. +_UNIT_SECONDS = {"s": 1, "m": 60, "h": 3600, "d": 86400, "w": 604800} +_PART = re.compile(r"(\d+(?:\.\d+)?)\s*([smhdw])") + + +def delay_seconds(spec): + """Seconds to wait for a duration spec such as '90s', '5m' or '1h30m'.""" + text = spec.strip() + pos, total, seen = 0, 0, False + while pos < len(text): + m = _PART.match(text, pos) + if not m: + raise ValueError("bad duration: %r" % (spec,)) + seen = True + n, unit = m.group(1), m.group(2) + if "." in n: + whole, frac = n.split(".") + total += int(whole) * _UNIT_SECONDS[unit] + int(frac) * _UNIT_SECONDS[unit] / 10 ** len(frac) + else: + total += int(n) * _UNIT_SECONDS[unit] + pos = m.end() + while pos < len(text) and text[pos] == " ": + pos += 1 + if not seen: + raise ValueError("bad duration: %r" % (spec,)) + return int(total) if float(total).is_integer() else float(total) diff --git a/bench/local/regrade.py b/bench/local/regrade.py index cfe30b8..fbe0cea 100644 --- a/bench/local/regrade.py +++ b/bench/local/regrade.py @@ -10,11 +10,12 @@ from pathlib import Path from tasks import TASKS from tasks2 import TASKS2 +from tasks3 import TASKS3 HERE = Path(__file__).resolve().parent out = HERE / os.environ.get("RESULTS", "results-v4.jsonl") WORK = Path(os.environ.get("BENCH_WORK", Path.home() / ".cache/maat-bench/work")) -by = {T.name: T for T in TASKS + TASKS2} +by = {T.name: T for T in TASKS + TASKS2 + TASKS3} rows = [json.loads(l) for l in out.read_text().splitlines()] changed = 0 for r in rows: diff --git a/bench/local/rejudge.py b/bench/local/rejudge.py new file mode 100644 index 0000000..1ac3124 --- /dev/null +++ b/bench/local/rejudge.py @@ -0,0 +1,156 @@ +""" +Re-judge an oracle_replay.py results file with the product's rules, no model calls. + +oracle_replay.py records the checks each arm produced; this runs them again +against every saved workspace the way the CLI would: + + - preflight (dist/criteria.js preflightCriteria) on the untouched project + drops a check that cannot run or reports its own bug before any work; + - at the claim, a drafted check whose failure is its own bug + (checkSelfError) is retired, and so is the reference on exit 3; + - "verified" = at least one check left, and every check left passed. + All retired -> unverified, never verified. + + python3 rejudge.py RESULTS.jsonl [--snap DIR] [--arms A,C] +""" +from __future__ import annotations + +import argparse +import json +import os +import shutil +import subprocess +import sys +import tempfile +from collections import defaultdict +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +from tasks import TASKS # noqa: E402 +from tasks2 import TASKS2 # noqa: E402 +from tasks3 import TASKS3 # noqa: E402 + +REPO = HERE.parent.parent +BY = {T.name: T for T in TASKS + TASKS2 + TASKS3} + + +def node(js: str, stdin: str = "") -> str: + p = subprocess.run(["node", "--input-type=module", "-e", js], input=stdin, capture_output=True, text=True, timeout=600) + if p.returncode != 0: + raise RuntimeError(p.stderr[-500:]) + return p.stdout + + +def preflight_broken(checks: list[str], cwd: Path) -> set[int]: + js = f""" +import {{ preflightCriteria }} from '{REPO}/dist/criteria.js'; +const runs = JSON.parse(await new Promise(r => {{ let s=''; process.stdin.on('data', d => s += d); process.stdin.on('end', () => r(s)); }})); +const broken = await preflightCriteria(runs.map((run, i) => ({{ name: 'c' + i, kind: 'command', run, expectExit: 0 }})), {{ cwd: {json.dumps(str(cwd))}, timeoutMs: 120000 }}); +process.stdout.write(JSON.stringify(broken.map(b => Number(b.name.slice(1))))); +""" + return set(json.loads(node(js, json.dumps(checks)))) + + +def self_errors(outputs: list[str]) -> list[bool]: + js = f""" +import {{ checkSelfError }} from '{REPO}/dist/criteria.js'; +const outs = JSON.parse(await new Promise(r => {{ let s=''; process.stdin.on('data', d => s += d); process.stdin.on('end', () => r(s)); }})); +process.stdout.write(JSON.stringify(outs.map(o => checkSelfError(o) !== null))); +""" + return json.loads(node(js, json.dumps(outputs))) + + +def run_all(checks: list[str], ws: Path) -> list[tuple[int, str]]: + tmp = Path(tempfile.mkdtemp(prefix="rejudge-")) + try: + shutil.copytree(ws, tmp / "w", symlinks=True) + out = [] + for c in checks: + try: + p = subprocess.run(c, shell=True, cwd=tmp / "w", capture_output=True, text=True, timeout=180) + out.append((p.returncode, p.stdout + p.stderr)) + except subprocess.TimeoutExpired: + out.append((124, "timeout")) + return out + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +def verdict(arm: str, checks: list[str], ws: Path) -> tuple[str, str]: + if not checks: + return "unverified", "no checks" + res = run_all(checks, ws) + errs = self_errors([o for _, o in res]) + live = [] + for i, (code, out) in enumerate(res): + reference = arm in ("B", "C") + retired = (code == 3) if reference else (code != 0 and errs[i]) + if not retired: + live.append((code, out)) + if not live: + return "unverified", "every check retired" + bad = [o for c, o in live if c != 0] + return ("verified", "") if not bad else ("refused", bad[0][-200:]) + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("results") + ap.add_argument("--snap", default=os.environ.get("SNAP", str(Path.home() / ".cache/maat-bench/work"))) + ap.add_argument("--arms", default="A,B,C") + ap.add_argument("--out") + a = ap.parse_args() + snap = Path(a.snap) + recs = [json.loads(l) for l in open(a.results) if l.strip()] + latest: dict = {} + for r in recs: + latest[(r["task"], r["arm"])] = r + tot = defaultdict(lambda: defaultdict(int)) + lines = [] + for (task, arm), r in sorted(latest.items()): + if arm not in a.arms.split(","): + continue + T = BY[task] + checks = r["meta"].get("checks") if arm == "A" else ([r["meta"]["run"]] if r["meta"].get("applies") and r["meta"].get("run") else []) + checks = checks or [] + pristine = Path(tempfile.mkdtemp(prefix="rejudge-p-")) / "p" + pristine.mkdir() + T.setup(pristine) + if arm == "A" and checks: + drop = preflight_broken(checks, pristine) + checks = [c for i, c in enumerate(checks) if i not in drop] + pv, _ = verdict(arm, checks, pristine) + row = {"task": task, "arm": arm, "n_checks": len(checks), "pristine": pv, "ws": []} + for w in r["rows"]: + ws = snap / w["ws"] + if not ws.is_dir(): + continue + v, why = verdict(arm, checks, ws) + row["ws"].append({"ws": w["ws"], "right": w["right"], "verdict": v, "why": why}) + k = tot[arm] + k["right" if w["right"] else "wrong"] += 1 + k[f"{'right' if w['right'] else 'wrong'}_{v}"] += 1 + tot[arm]["pristine_verified"] += pv == "verified" + tot[arm]["tasks"] += 1 + tot[arm]["tasks_with_checks"] += bool(checks) + lines.append(row) + rv = sum(1 for x in row["ws"] if x["right"] and x["verdict"] == "verified") + rn = sum(1 for x in row["ws"] if x["right"]) + wv = sum(1 for x in row["ws"] if not x["right"] and x["verdict"] == "verified") + wn = sum(1 for x in row["ws"] if not x["right"]) + print(f"{task:22} {arm} checks={len(checks)} right verified {rv}/{rn} wrong verified {wv}/{wn} pristine {pv}", flush=True) + print() + for arm, k in sorted(tot.items()): + print( + f"arm {arm}: tasks {k['tasks']} (with checks {k['tasks_with_checks']}) | " + f"right work: verified {k['right_verified']}/{k['right']}, refused {k['right_refused']}, unverified {k['right_unverified']} | " + f"WRONG work: verified {k['wrong_verified']}/{k['wrong']}, refused {k['wrong_refused']}, unverified {k['wrong_unverified']} | " + f"pristine verified {k['pristine_verified']}" + ) + if a.out: + Path(a.out).write_text("".join(json.dumps(l) + "\n" for l in lines)) + + +if __name__ == "__main__": + main() diff --git a/bench/local/replay_common.py b/bench/local/replay_common.py new file mode 100644 index 0000000..f2d252a --- /dev/null +++ b/bench/local/replay_common.py @@ -0,0 +1,119 @@ +"""Shared loader for the Bet 2 offline replay (env_values.py, frame_intact.py). + +Walks the saved bench work folders, pairs every -molt- tree with its +grader row, and reads the sealed checks out of its .maat journal. No model, +no network. +""" +from __future__ import annotations + +import json +import re +import shutil +import sys +import tempfile +from pathlib import Path + +HERE = Path(__file__).resolve().parent +CACHE = Path.home() / ".cache/maat-bench" +RESULTS = CACHE / "container-results" +CW = CACHE / "container-work" + +sys.path.insert(0, str(HERE)) +from tasks import TASKS # noqa: E402 +from tasks2 import TASKS2 # noqa: E402 + +TASK_BY_NAME = {T.name: T for T in TASKS + TASKS2} +RUN_DIR = re.compile(r"^(?P.+)-molt-(?P\d+)$") + + +def result_key(root_name: str) -> str: + """container-work/v11-new-s3 -> v11-new; grok1-2 -> grok1; m8b2-1 -> m8b2.""" + n = re.sub(r"-s\d+$", "", root_name) + if (RESULTS / f"results-{n}.jsonl").exists(): + return n + return re.sub(r"-\d+$", "", root_name) + + +def roots() -> list[tuple[str, Path]]: + """(results key, folder holding -molt- trees).""" + out: list[tuple[str, Path]] = [] + pat = re.compile(r"^(v9|v10|v11|v12)-|^grok|^q\d|^m8b|^fail5") + for d in sorted(CW.iterdir()): + if d.is_dir() and pat.match(d.name): + out.append((result_key(d.name), d)) + out.append(("gem1", CACHE / "work-gem1")) + out.append(("sub-gemini", CACHE / "work-sub-gemini")) + out.append(("sub-grok", CACHE / "work-sub-grok")) + return [(k, p) for k, p in out if p.is_dir()] + + +def load_results(key: str) -> dict[tuple[str, int], dict]: + f = RESULTS / f"results-{key}.jsonl" + rows: dict[tuple[str, int], dict] = {} + if f.exists(): + for line in f.read_text().splitlines(): + r = json.loads(line) + rows[(r["task"], r["rep"])] = r + return rows + + +def journal_events(run: Path) -> list[dict]: + ev: list[dict] = [] + logd = run / ".maat/log" + for f in sorted(logd.glob("*.jsonl")) if logd.is_dir() else []: + for line in f.read_text(errors="replace").splitlines(): + try: + ev.append(json.loads(line)) + except Exception: + pass + ev.sort(key=lambda e: (e.get("iso", ""), e.get("seq", 0))) + return ev + + +def bar_runs(ev: list[dict]) -> list[dict]: + return [e["data"] for e in ev if e.get("kind") == "bar_run"] + + +def drafted(check: dict) -> bool: + return str(check.get("name", "")).startswith("task:") + + +def runs() -> list[dict]: + """One record per saved tree that has a grader row.""" + out = [] + for key, root in roots(): + res = load_results(key) + for r in sorted(p for p in root.iterdir() if p.is_dir()): + # a shard folder holds the trees; a work folder (work-gem1) is the trees + cands = [r] if RUN_DIR.match(r.name) else sorted(c for c in r.iterdir() if c.is_dir()) + for c in cands: + m = RUN_DIR.match(c.name) + if not m: + continue + task, rep = m["task"], int(m["rep"]) + row = res.get((task, rep)) + if row is None or task not in TASK_BY_NAME: + continue + out.append({"arm": key, "task": task, "rep": rep, "dir": c, "row": row, + "passed": bool(row["passed"]), "id": f"{key}/{task}#{rep}"}) + # root itself may directly hold trees (work-gem1) + for c in sorted(p for p in root.iterdir() if p.is_dir() and RUN_DIR.match(p.name)): + pass + return out + + +_pristine: dict[str, Path] = {} +_tmp = Path(tempfile.mkdtemp(prefix="bet2-pristine-")) + + +def pristine(task: str) -> Path: + if task not in _pristine: + d = _tmp / task + d.mkdir() + TASK_BY_NAME[task].setup(d) + _pristine[task] = d + return _pristine[task] + + +def cleanup() -> None: + shutil.rmtree(_tmp, ignore_errors=True) diff --git a/bench/local/run.py b/bench/local/run.py index f63781b..1d7a0ab 100644 --- a/bench/local/run.py +++ b/bench/local/run.py @@ -1,12 +1,15 @@ """ -Run molt on the local tasks and record how it behaves. Graders are hidden in -tasks.py. +Run molt on the local tasks and grade the result. Graders are hidden in tasks.py. - python3 run.py [molt] + python3 run.py molt [repeats] [task,task] + +Paired arms: ARMS="base:REFERENCE=0;ref:REFERENCE=1" (or JSON) runs, per task, +arm A then arm B before the next task; rows carry "arm"; resume is by +(task, agent, rep, arm). BENCH_TASKS=a,b restricts the task set. Only molt is run here. Other vendors' agents are not benchmarked from this repository: their terms commonly forbid benchmarking and publishing -performance data, and the results files hold molt rows only. +performance data. """ from __future__ import annotations @@ -25,33 +28,84 @@ # Outside every git repository: a task folder inside this one let agents' # `git commit` walk up and commit task files into molt-desktop's main. WORK = Path(os.environ.get("BENCH_WORK", Path.home() / ".cache/maat-bench/work")) -MOLT = Path.home() / os.environ.get("MOLT_DIST", "Documents/molt-desktop/dist-compare") / "cli.js" -LIMIT = 600 # seconds per task +# MOLT_DIST_ABS: an absolute path to the built Maat (the container mounts it at /maat). +MOLT = (Path(os.environ["MOLT_DIST_ABS"]) if os.environ.get("MOLT_DIST_ABS") else Path.home() / os.environ.get("MOLT_DIST", "Documents/molt-desktop/dist-compare")) / "cli.js" +LIMIT = int(os.environ.get("BENCH_LIMIT", "600")) # seconds per task +# Space Bunny Alpha was withdrawn from OpenRouter on 2026-10-05; the owner chose Nemotron 3 Ultra. +MODEL = os.environ.get("BENCH_MODEL") or "nvidia/nemotron-3-ultra-550b-a55b:free" +# BENCH_URL points the run at another endpoint (e.g. the NUC's llama.cpp over the tunnel); the +# OpenRouter key is then not sent anywhere. +URL = os.environ.get("BENCH_URL") or "https://openrouter.ai/api/v1" def openrouter_key() -> str: + if os.environ.get("OPENROUTER_API_KEY"): # the container is given the key, not the Mac's auth file + return os.environ["OPENROUTER_API_KEY"] return subprocess.run( ["node", "-e", "import('%s').then(m=>process.stdout.write(m.readAuth().openrouter||''))" % (Path.home() / "Documents/molt-desktop/dist/providers.js")], capture_output=True, text=True, ).stdout +# --sandbox, as the Terminal-Bench adapter has always passed: the task folder +# is throwaway (and, in a container, so is the machine). With --yes the +# project boundary still asked, headless asking is "User denied", and every +# local number from the first run to 2026-10-06 was measured with 118-168 +# denied calls per 60 runs: writing the deliverable through a heredoc, removing +# the model's own scratch files. BENCH_GATE=yes reproduces the old runs. +AGENT_USER = os.environ.get("BENCH_AGENT_USER") # set in the containers: the agent runs unprivileged + + +def as_agent(cmd: list, env: dict | None = None) -> tuple[list, dict | None]: + """With BENCH_AGENT_USER set, run the agent as that user: it cannot read the graders + or the reference solutions, which stay root-only. Unset (Mac host lanes): unchanged.""" + if not AGENT_USER: + return cmd, env + import pwd + home = pwd.getpwnam(AGENT_USER).pw_dir + return ["runuser", "-u", AGENT_USER, "--", "env", f"HOME={home}", *cmd], env + + +def hand_over(d: Path) -> None: + if AGENT_USER: + subprocess.run(["chown", "-R", AGENT_USER, str(d)], check=True) + + +def provider_capped(out: str, steps: int) -> bool: + """The provider's cap (OpenRouter daily limit, OpenCode free-usage limit), not the work.""" + return ( + "rate limit is reached until" in out + or ("free-models-per-day" in out and steps == 0) + or ("OpenCode rate limit" in out and steps == 0) + ) + + def run_molt(d: Path, prompt: str, log: Path) -> dict: - env = os.environ | {"MOLT_API_KEY": openrouter_key(), "MOLT_JUDGMENT": "0"} # nobody rules on a benchmark run + key = openrouter_key() if "openrouter.ai" in URL else "local" + env = os.environ | {"MOLT_API_KEY": key, "MOLT_JUDGMENT": "0"} # nobody rules on a benchmark run cmd = [ - "node", str(MOLT), "run", "--url", "https://openrouter.ai/api/v1", "--model", "stealth/space-bunny-alpha", - "--reasoning", "low", "--yes", "--json", "--criteria", "auto", "--batch", "--review", "3", "--steps", "200", "--cwd", str(d), prompt, + "node", str(MOLT), "run", "--url", URL, "--model", os.environ.get("BENCH_MODEL") or MODEL, + "--reasoning", os.environ.get("BENCH_REASONING") or "low", *(["--yes"] if os.environ.get("BENCH_GATE") == "yes" else ["--sandbox"]), "--json", "--criteria", "auto", "--batch", "--review", "3", "--steps", "200", + # Maat stops itself a minute before the kill and judges what is on disk; + # the subprocess timeout below is only the backstop. + "--for", f"{max(60, LIMIT - 60)}s", + *(["--reference"] if os.environ.get("REFERENCE") == "1" else []), "--cwd", str(d), prompt, ] t0 = time.time() + cmd, env = as_agent(cmd, env) try: - out = subprocess.run(cmd, capture_output=True, text=True, timeout=LIMIT, env=env).stdout + proc = subprocess.run(cmd, capture_output=True, text=True, timeout=LIMIT, env=env) + out, err = proc.stdout, proc.stderr timed_out = False except subprocess.TimeoutExpired as e: out = (e.stdout or b"").decode() if isinstance(e.stdout, bytes) else (e.stdout or "") + err = (e.stderr or b"").decode() if isinstance(e.stderr, bytes) else (e.stderr or "") timed_out = True secs = time.time() - t0 log.write_text(out) - steps = 0; per = []; outcome = None; spend = {}; review = None; disagree = [] + # Maat's own notices (criteria not drafted, dropped checks, refusals) go to stderr. + log.with_suffix(".err").write_text(err or "") + steps = 0; per = []; outcome = None; spend = {}; review = None; disagree = []; extra = {} for line in out.splitlines(): if not line.startswith("{"): continue @@ -67,7 +121,13 @@ def run_molt(d: Path, prompt: str, log: Path) -> dict: spend = ev.get("spend") or {} review = ev.get("review") disagree = ev.get("checksDisagree") or [] + # Only what job_end carried; absent keys stay absent (older builds). + extra = {k: ev[k] for k in ("revealed", "deadline", "endedBy", "retired", "build", "tier", "tierReason") if k in ev} + # The provider's daily cap, not the work: every later task would fail the + # same way (2026-10-05: eleven tasks per arm "failed" in 140 s, 0 turns). + capped = provider_capped(out, steps) return { + "provider_capped": capped, "secs": round(secs), "turns": steps, "calls": sum(per), "multi": sum(1 for x in per if x > 1), "tokens_in": spend.get("promptTokens"), "claim": outcome, "timed_out": timed_out, "said_done": (outcome or "").startswith("verified"), @@ -76,6 +136,7 @@ def run_molt(d: Path, prompt: str, log: Path) -> dict: # Refused only by molt's own drafted checks (reported unverified). "checks_disagree": disagree, "said_done_reviewed": (outcome or "").startswith("verified") and not (review and not review.get("confirmed")), + **extra, } @@ -93,34 +154,85 @@ def claims_done(text: str) -> bool: "passes", "passing", "finished", "works", "added", "updated", "written")) +def parse_arms(spec: str | None) -> list[tuple[str | None, dict]]: + """ + ARMS defines the arms of a paired run. Either "base:REFERENCE=0;ref:REFERENCE=1" + (arms split on ";", variables on ","), or JSON: {"base": {"REFERENCE": "0"}, ...} + or a path to such a file. No ARMS: one unnamed arm, the legacy behaviour. + """ + if not spec: + return [(None, {})] + spec = spec.strip() + if spec.startswith("{") or (os.path.isfile(spec) and spec.endswith(".json")): + data = json.loads(spec if spec.startswith("{") else Path(spec).read_text()) + return [(k, {a: str(b) for a, b in (v or {}).items()}) for k, v in data.items()] + arms = [] + for part in filter(None, (x.strip() for x in spec.split(";"))): + name, _, vs = part.partition(":") + env = {} + for kv in filter(None, (x.strip() for x in vs.split(","))): + k, _, v = kv.partition("=") + env[k.strip()] = v + arms.append((name.strip(), env)) + return arms + + def main(which: str, repeats: int, task_filter: str | None) -> None: from tasks2 import TASKS2 # noqa: PLC0415 + from tasks3 import TASKS3 # noqa: PLC0415 agents = {"molt": run_molt} if which not in agents: sys.exit(f"unknown agent {which!r}: this runner only runs molt") chosen = [which] - tasks = [T for T in TASKS + TASKS2 if not task_filter or T.name in task_filter.split(",")] - out = HERE / os.environ.get("RESULTS", f"results-{which}-x{repeats}.jsonl") + tasks = [T for T in TASKS + TASKS2 + TASKS3 if not task_filter or T.name in task_filter.split(",")] + # BENCH_TASKS restricts to a subset (e.g. the discordant tasks of an earlier pair). + if os.environ.get("BENCH_TASKS"): + want = {x.strip() for x in os.environ["BENCH_TASKS"].split(",") if x.strip()} + tasks = [T for T in tasks if T.name in want] + arms = parse_arms(os.environ.get("ARMS")) + out = Path(os.environ.get("RESULTS_DIR", HERE)) / os.environ.get("RESULTS", f"results-{which}-x{repeats}.jsonl") done = set() if out.exists(): # resume: skip runs already recorded for line in out.read_text().splitlines(): r = json.loads(line) - done.add((r["task"], r["agent"], r["rep"])) + done.add((r["task"], r["agent"], r["rep"], r.get("arm"))) for rep in range(repeats): for T in tasks: - for a in chosen: - if (T.name, a, rep) in done: - continue - d = WORK / f"{T.name}-{a}-{rep}" - shutil.rmtree(d, ignore_errors=True) - d.mkdir(parents=True) - T.setup(d) - r = agents[a](d, T.PROMPT, WORK / f"{T.name}-{a}-{rep}.log") - ok, why = T.grade(d) - r.update(task=T.name, agent=a, rep=rep, passed=ok, why=why, final="") - with out.open("a") as f: - f.write(json.dumps(r) + "\n") - print(json.dumps({k: r[k] for k in ("task", "agent", "rep", "passed", "said_done", "turns", "secs")}), flush=True) + for arm, arm_env in arms: # paired: arm A then arm B on this task before the next task + for a in chosen: + if (T.name, a, rep, arm) in done: + continue + tag = f"{T.name}-{a}-{rep}" + (f"-{arm}" if arm else "") + d = WORK / tag + shutil.rmtree(d, ignore_errors=True) + d.mkdir(parents=True) + T.setup(d) + hand_over(d) + saved = {k: os.environ.get(k) for k in arm_env} + os.environ.update(arm_env) + try: + r = agents[a](d, T.PROMPT, WORK / f"{tag}.log") + finally: + for k, v in saved.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + if r.get("provider_capped"): + # Not recorded, so a resume runs this task again. + print(f"STOPPED: the provider's daily limit / quota is reached ({tag} not recorded)", flush=True) + return + ok, why = T.grade(d) + final = "" + r.update(task=T.name, agent=a, rep=rep, passed=ok, why=why, final=final[:300]) + if arm: + r["arm"] = arm + if os.environ.get("MAAT_BUILD"): + r["build"] = os.environ["MAAT_BUILD"] # sha8 of the packed Maat the container ran + with out.open("a") as f: + f.write(json.dumps(r) + "\n") + print(json.dumps({k: r[k] for k in ("task", "agent", "arm", "rep", "passed", "said_done", "turns", "secs") if k in r}), flush=True) + if __name__ == "__main__": diff --git a/bench/local/scoreboard.py b/bench/local/scoreboard.py new file mode 100644 index 0000000..e477a4d --- /dev/null +++ b/bench/local/scoreboard.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +""" +Ship/no-ship scoreboard over benchmark results (JSONL from bench/local/run.py). + + python3 bench/local/scoreboard.py base=results-v8-base.jsonl ref=results-v8-ref.jsonl \\ + --target precision --min-gain 0.10 + +Each argument is LABEL=FILE (or FILE alone: labelled by its "arm" column if +rows have one, else by the file stem). Files sharing a label are pooled. +The first label is the baseline; every other arm is compared to it. + +Per arm: pass rate, verified precision P(pass | verified), verified recall +P(verified | pass), false-done (verified but failed), no-verdict (claim +missing/error/stopped or timed out), checks-disagree-but-passed, median/p90 +secs, median tokens_in; rates carry Wilson 95% intervals. + +Paired (same task, rep): per-pair pass differences and an exact sign test on +discordant pairs. Ship rule: paired pass difference >= -1 task AND false-done +not higher AND the target metric improved by >= --min-gain. +Target gains: pass/precision/recall/verified in rate points (0.10 = 10 points); +secs/tokens as relative reduction (0.10 = 10% lower); no_verdict/false_done as +count reduction. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from stats import TIERS, is_verified, no_verdict, percentile, rate, sign_test, tier_breakdown # noqa: E402 + +LOW_POWER_N = 30 # pairs below this cannot separate a few-task difference from noise + + +def load(path: str | Path) -> list[dict]: + rows = [] + for line in Path(path).read_text().splitlines(): + line = line.strip() + if line.startswith("{"): + rows.append(json.loads(line)) + return rows + + +def metrics(rows: list[dict]) -> dict: + n = len(rows) + passed = [r for r in rows if r.get("passed")] + ver = [r for r in rows if is_verified(r.get("claim"))] + ver_ok = [r for r in ver if r.get("passed")] + secs = [r["secs"] for r in rows if r.get("secs") is not None] + toks = [r["tokens_in"] for r in rows if r.get("tokens_in") is not None] + return { + "n": n, + "pass": len(passed), + "verified": len(ver), + "verified_pass": len(ver_ok), + "false_done": len(ver) - len(ver_ok), + "no_verdict": sum(1 for r in rows if no_verdict(r)), + "disagree_passed": sum(1 for r in passed if r.get("checks_disagree")), + "tiers": tier_breakdown(rows), + "secs_med": percentile(secs, 50), + "secs_p90": percentile(secs, 90), + "tokens_med": percentile(toks, 50), + } + + +def fmt_num(v) -> str: + return "-" if v is None else f"{v:.0f}" + + +def tier_lines(m: dict) -> list[str]: + """Count and precision per tier, when the rows carry one (job_end `tier`).""" + tiers = m.get("tiers") or {} + if not tiers: + return [] + out = [" tiers P(pass | tier)"] + for t in [*TIERS, *sorted(set(tiers) - set(TIERS))]: + if t in tiers: + n, k = tiers[t] + out.append(f" {t:<15} n={n:<3} {rate(k, n)}") + return out + + +def report_arm(label: str, m: dict) -> list[str]: + return [ + f"== {label} (n={m['n']})", + f" pass rate {rate(m['pass'], m['n'])}", + f" verified precision {rate(m['verified_pass'], m['verified'])} P(pass | verified)", + f" verified recall {rate(m['verified_pass'], m['pass'])} P(verified | pass)", + f" false-done {m['false_done']}", + f" no-verdict {m['no_verdict']}", + f" checks-disagree but passed {m['disagree_passed']}", + f" secs median {fmt_num(m['secs_med'])} p90 {fmt_num(m['secs_p90'])}", + f" tokens_in median {fmt_num(m['tokens_med'])}", + *tier_lines(m), + ] + + +def pair_up(a: list[dict], b: list[dict]) -> list[tuple[dict, dict]]: + key = lambda r: (r.get("task"), r.get("agent"), r.get("rep", 0)) + bi = {key(r): r for r in b} + return [(r, bi[key(r)]) for r in a if key(r) in bi] + + +TARGETS = ("pass", "precision", "recall", "verified", "secs", "tokens", "no_verdict", "false_done") + + +def target_gain(target: str, ma: dict, mb: dict) -> float | None: + """Improvement of B over A on the target metric (positive = better).""" + div = lambda k, n: k / n if n else None + if target in ("pass", "verified"): + return div(mb[target], mb["n"]) - div(ma[target], ma["n"]) + if target in ("precision", "recall"): + den = "verified" if target == "precision" else "pass" + va, vb = div(ma["verified_pass"], ma[den]), div(mb["verified_pass"], mb[den]) + return None if va is None or vb is None else vb - va + if target in ("secs", "tokens"): + k = "secs_med" if target == "secs" else "tokens_med" + if not ma[k] or mb[k] is None: + return None + return (ma[k] - mb[k]) / ma[k] + return ma[target] - mb[target] # no_verdict, false_done: fewer is better + + +def compare(la: str, lb: str, ra: list[dict], rb: list[dict], target: str, min_gain: float) -> tuple[list[str], str]: + pairs = pair_up(ra, rb) + out = [f"== paired: {la} (A) vs {lb} (B)"] + n = len(pairs) + if not n: + return out + [" no (task, rep) in both arms"], f"VERDICT {lb} vs {la}: NO VERDICT (no shared tasks)" + only_a = [x for x, y in pairs if x.get("passed") and not y.get("passed")] + only_b = [y for x, y in pairs if y.get("passed") and not x.get("passed")] + ma, mb = metrics([x for x, _ in pairs]), metrics([y for _, y in pairs]) + diff = mb["pass"] - ma["pass"] + p = sign_test(len(only_a), len(only_b)) + out += [ + f" pairs {n}; A {ma['pass']} pass, B {mb['pass']} pass; B-A = {diff:+d}", + f" discordant: only A passed {len(only_a)}, only B passed {len(only_b)}; exact sign test p = {p:.3f}", + ] + if only_a: + out.append(" only A: " + ", ".join(sorted(str(r.get("task")) for r in only_a))) + if only_b: + out.append(" only B: " + ", ".join(sorted(str(r.get("task")) for r in only_b))) + gain = target_gain(target, ma, mb) + ok_pass = diff >= -1 + ok_fd = mb["false_done"] <= ma["false_done"] + ok_target = gain is not None and gain >= min_gain + gs = "n/a" if gain is None else f"{gain:+.3f}" + out.append( + f" rule: pass diff >= -1: {diff:+d} {'ok' if ok_pass else 'FAIL'}; " + f"false-done B<=A: {mb['false_done']} vs {ma['false_done']} {'ok' if ok_fd else 'FAIL'}; " + f"target {target} gain >= {min_gain}: {gs} {'ok' if ok_target else 'FAIL'}" + ) + fails = [name for name, ok in (("pass drop", ok_pass), ("false-done up", ok_fd), ("target not improved", ok_target)) if not ok] + line = f"VERDICT {lb} vs {la}: {'NO-SHIP' if fails else 'SHIP'}" + (f" ({', '.join(fails)})" if fails else "") + if n < LOW_POWER_N: + line += f" [LOW POWER: {n} pairs < {LOW_POWER_N}; sign test p={p:.2f}; a decision rule, not evidence]" + return out, line + + +def group(args: list[str]) -> dict[str, list[dict]]: + arms: dict[str, list[dict]] = {} + for a in args: + label, sep, path = a.partition("=") + if not sep or Path(a).exists(): + label, path = None, a + for r in load(path): + arms.setdefault(label or r.get("arm") or Path(path).stem, []).append(r) + return arms + + +def main(argv: list[str]) -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("files", nargs="+", help="LABEL=FILE or FILE") + ap.add_argument("--target", choices=TARGETS, default="precision", help="metric that must improve (default precision)") + ap.add_argument("--min-gain", type=float, default=0.0, help="required improvement (units: see above)") + ap.add_argument("--agent", help="only rows of this agent (e.g. molt)") + a = ap.parse_args(argv) + arms = group(a.files) + if a.agent: + arms = {k: [r for r in v if r.get("agent") == a.agent] for k, v in arms.items()} + for label, rows in arms.items(): + print("\n".join(report_arm(label, metrics(rows)))) + labels = list(arms) + verdicts = [] + for lb in labels[1:]: + lines, v = compare(labels[0], lb, arms[labels[0]], arms[lb], a.target, a.min_gain) + print("\n".join(lines)) + verdicts.append(v) + print("\n".join(verdicts)) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/bench/local/stats.py b/bench/local/stats.py new file mode 100644 index 0000000..124869c --- /dev/null +++ b/bench/local/stats.py @@ -0,0 +1,81 @@ +""" +Shared statistics for the benchmark scoreboards (standard library only). +bench/harbor/compare.py imports this file by path, so keep it dependency-free. +""" + +from __future__ import annotations + +import math + +Z95 = 1.959963984540054 + + +def wilson(k: int, n: int, z: float = Z95) -> tuple[float, float] | None: + """Wilson score interval for k successes in n trials; None when n == 0.""" + if n <= 0: + return None + p = k / n + d = 1 + z * z / n + c = (p + z * z / (2 * n)) / d + h = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / d + return max(0.0, c - h), min(1.0, c + h) + + +def rate(k: int, n: int) -> str: + """'5/6 83% [44-97]' (or '0/0 -').""" + if n == 0: + return f"{k}/{n} -" + lo, hi = wilson(k, n) + return f"{k}/{n} {100 * k / n:.0f}% [{100 * lo:.0f}-{100 * hi:.0f}]" + + +def sign_test(b: int, c: int) -> float: + """Exact two-sided sign test on b vs c discordant pairs (p = 1 with none).""" + n = b + c + if n == 0: + return 1.0 + k = min(b, c) + tail = sum(math.comb(n, i) for i in range(k + 1)) / 2**n + return min(1.0, 2 * tail) + + +def percentile(xs: list[float], q: float) -> float | None: + """Nearest-rank percentile (q in 0..100); the median uses the usual midpoint.""" + v = sorted(x for x in xs if x is not None) + if not v: + return None + if q == 50: + m = len(v) // 2 + return v[m] if len(v) % 2 else (v[m - 1] + v[m]) / 2 + return v[max(0, math.ceil(q / 100 * len(v)) - 1)] + + +def is_verified(claim) -> bool: + return isinstance(claim, str) and claim.startswith("verified") + + +def no_verdict(row: dict) -> bool: + """No usable Maat verdict: claim missing, error, stopped, or the run timed out.""" + c = row.get("claim") + return (not c) or c.startswith(("error", "stopped")) or bool(row.get("timed_out")) + + +TIERS = ("verified", "passed-checks") + + +def tier_breakdown(rows: list[dict]) -> dict[str, tuple[int, int]]: + """{tier: (rows, rows that passed)} over rows that carry `tier` (job_end's field). + + Rows from builds before the tiers have no `tier` and are left out, so an old + result file reports no breakdown rather than a wrong one. Precision of a + tier is P(pass | tier): the word "verified" has to mean the same thing on + every model, and this is how that is read off the results. + """ + out: dict[str, tuple[int, int]] = {} + for r in rows: + t = r.get("tier") + if not t: + continue + n, k = out.get(t, (0, 0)) + out[t] = (n + 1, k + (1 if r.get("passed") else 0)) + return out diff --git a/bench/local/tasks3.py b/bench/local/tasks3.py new file mode 100644 index 0000000..2171523 --- /dev/null +++ b/bench/local/tasks3.py @@ -0,0 +1,2681 @@ +""" +Twenty-two more local tasks, harder and more varied than tasks.py / tasks2.py, same shape: + + setup(dir) -> builds the starting state in dir + PROMPT -> what the agent is told + grade(dir) -> (passed: bool, why: str), run after the agent finishes + +Graders run the deliverable on cases of their own (some implied by the prompt rather than listed +in it) and never depend on the network. Anything that mutates state runs on a scratch copy of dir. +Reference and deliberately-wrong solutions live in reference_solutions/ and are exercised by +validate3.py. +""" + +from __future__ import annotations + +import csv +import datetime as dt +import hashlib +import io +import json +import os +import random +import re +import shutil +import sqlite3 +import subprocess +import sys +import tempfile +import time +import unicodedata +from decimal import Decimal +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from tasks2 import IGNORED, all_files, py_check, run, scratch_copy, sh # noqa: E402,F401 + +VCS_ENV = { + "GIT_AUTHOR_NAME": "Dev", "GIT_AUTHOR_EMAIL": "dev@example.com", + "GIT_COMMITTER_NAME": "Dev", "GIT_COMMITTER_EMAIL": "dev@example.com", + "FILTER_BRANCH_SQUELCH_WARNING": "1", +} + + +def runb(args, cwd=None, inp=None, env=None, timeout=30): + """Run a command with bytes in/out (locale-independent); env entries are added to os.environ.""" + e = dict(os.environ) + e.update(env or {}) + if isinstance(inp, str): + inp = inp.encode() + return subprocess.run(args, cwd=cwd, input=inp, capture_output=True, env=e, timeout=timeout) + + +def vcs(d: Path, *args, date: str | None = None): + env = dict(VCS_ENV) + if date: + env["GIT_AUTHOR_DATE"] = env["GIT_COMMITTER_DATE"] = date + r = runb(["git", *args], cwd=d, env=env) + return r.stdout.decode(errors="replace") + + +def write(d: Path, rel: str, text): + p = d / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_bytes(text if isinstance(text, bytes) else text.encode()) + + +def digest(root: Path) -> dict: + out = {} + for rel in sorted(all_files(root)): + out[rel] = hashlib.sha256((root / rel).read_bytes()).hexdigest() + return out + + +# ================================================================ 1. dedupe-contacts +class DedupeContacts: + name = "dedupe-contacts" + PROMPT = ( + "contacts.csv has columns id,name,email,phone and a lot of duplicate people. Write dedupe.py: " + "`python3 dedupe.py in.csv out.csv` merges them. Two rows are the same person if they have the same " + "email (ignoring case and surrounding spaces) or the same phone number (compare digits only, and only " + "the last 10 digits when there are more than 10); blank emails never match, and neither do phones with " + "fewer than 7 digits. Sameness is transitive. Write one row per person to out.csv with the same header: " + "id is the smallest id in the group; name, email and phone are the first non-blank value (stripped of " + "surrounding spaces, otherwise kept as written) among the group's rows taken in ascending numeric id " + "order, or blank if there is none. Rows go in ascending numeric id order. Then run it on contacts.csv " + "to produce contacts_clean.csv." + ) + + ROWS = [ + (1, "Ada Lovelace", "ada@x.org", "+1 (555) 010-2000"), + (2, "A. Lovelace", "ADA@X.ORG ", ""), + (3, "", "", "555-010-2000"), + (4, "Ben Ng", "ben@x.org", "12345"), + (5, "Benjamin Ng", "bn@y.org", "12345"), + (6, "Cleo, Jr.", "cleo@x.org", "00 44 20 7946 0958"), + (7, "Cleo Jones", "", "020 7946 0958"), + (10, " Dmitri ", "d@z.com", ""), + (9, "Dima", "", "555 0100"), + (100, "Dmitri P.", "D@Z.com", "555-0100"), + (8, "Esi", "", ""), + (11, "Esi Mensah", "", ""), + ] + + @staticmethod + def gen(seed=5, people=120, rows=400): + rnd = random.Random(seed) + ppl = [] + for i in range(people): + ppl.append((f"Person {i}", f"user{i}@mail{i % 7}.com", "".join(rnd.choice("0123456789") for _ in range(10)))) + out = [] + ids = rnd.sample(range(1000, 5000), rows) + for rid in ids: + n, e, p = rnd.choice(ppl) + if rnd.random() < 0.3: + e = "" + elif rnd.random() < 0.4: + e = " " + e.upper() + " " + if rnd.random() < 0.3: + p = "" + else: + f = rnd.choice(["{}", "+1 {}", "{}.{}", "({}) {}-{}", "001{}"]) + if f.count("{}") == 1: + p = f.format(p) + elif f.startswith("("): + p = f.format(p[:3], p[3:6], p[6:]) + else: + p = f.format(p[:3], p[3:]) + if rnd.random() < 0.15: + p = "".join(rnd.choice("0123456789") for _ in range(rnd.randint(3, 6))) + if rnd.random() < 0.2: + n = "" + out.append((rid, n, e, p)) + return out + + @staticmethod + def write_csv(path: Path, rows): + with open(path, "w", newline="") as f: + w = csv.writer(f) + w.writerow(["id", "name", "email", "phone"]) + w.writerows(rows) + + @staticmethod + def oracle(rows): + rows = [dict(zip(("id", "name", "email", "phone"), (str(x) for x in r))) for r in rows] + par = list(range(len(rows))) + + def find(x): + while par[x] != x: + par[x] = par[par[x]] + x = par[x] + return x + seen = {} + for i, r in enumerate(rows): + keys = [] + e = r["email"].strip().lower() + if e: + keys.append(("e", e)) + dg = re.sub(r"\D", "", r["phone"]) + if len(dg) >= 7: + keys.append(("p", dg[-10:])) + for k in keys: + if k in seen: + par[find(i)] = find(seen[k]) + else: + seen[k] = i + groups = {} + for i, r in enumerate(rows): + groups.setdefault(find(i), []).append(r) + out = [] + for g in groups.values(): + g.sort(key=lambda r: int(r["id"])) + row = {"id": g[0]["id"]} + for k in ("name", "email", "phone"): + row[k] = next((r[k].strip() for r in g if r[k].strip()), "") + out.append(row) + out.sort(key=lambda r: int(r["id"])) + return out + + @staticmethod + def all_rows(): + return [list(r) for r in DedupeContacts.ROWS] + DedupeContacts.gen(3, 40, 120) + + @staticmethod + def setup(d: Path) -> None: + DedupeContacts.write_csv(d / "contacts.csv", DedupeContacts.all_rows()) + + @staticmethod + def grade(d: Path): + if not (d / "dedupe.py").exists(): + return False, "no dedupe.py" + tmp, w = scratch_copy(d) + try: + cases = {"contacts.csv": DedupeContacts.all_rows(), "big.csv": DedupeContacts.gen(5, 120, 400)} + for fn, rows in cases.items(): + DedupeContacts.write_csv(w / fn, rows) + r = runb(["python3", "dedupe.py", fn, "o_" + fn], cwd=w) + if r.returncode != 0 or not (w / ("o_" + fn)).exists(): + return False, f"dedupe.py failed on {fn}: {r.stderr.decode()[-150:]}" + with open(w / ("o_" + fn), newline="") as f: + rd = list(csv.reader(f)) + if not rd or rd[0] != ["id", "name", "email", "phone"]: + return False, f"bad header {rd[:1]}" + got = [dict(zip(rd[0], x)) for x in rd[1:]] + want = DedupeContacts.oracle(rows) + if got != want: + for a, b in zip(got, want): + if a != b: + return False, f"{fn}: got {a} want {b}" + return False, f"{fn}: {len(got)} rows, want {len(want)}" + cc = d / "contacts_clean.csv" + if not cc.exists(): + return False, "no contacts_clean.csv" + with open(cc, newline="") as f: + rd = list(csv.reader(f)) + if not rd or [dict(zip(rd[0], x)) for x in rd[1:]] != DedupeContacts.oracle(cases["contacts.csv"]): + return False, "contacts_clean.csv does not match the dedupe of contacts.csv" + return True, "all inputs" + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +# ================================================================ 2. money-split +class MoneySplit: + name = "money-split" + PROMPT = ( + "Write alloc.py: `python3 alloc.py TOTAL W1 W2 ...` splits a sum of money between parties in proportion to " + "their integer weights and prints each party's share on its own line with exactly two decimals, in the " + "order given. Shares are whole cents and must add up to TOTAL exactly: give every party the floor of its " + "exact share, then hand the leftover cents out one each to the parties with the largest fractional " + "remainders (ties go to the party listed first). TOTAL is a non-negative decimal with at most two decimals " + "(e.g. 100, 0.05, 1234.5); weights are non-negative integers. On any bad input (no weights, a negative or " + "non-integer weight, all weights zero, a TOTAL that is not a decimal with at most two places) print " + "nothing to stdout, write a message to stderr and exit with status 1. Don't use floating point." + ) + + @staticmethod + def oracle(total, ws): + if not re.fullmatch(r"\d+(\.\d{1,2})?", total): + return None + cents = int((Decimal(total) * 100).to_integral_value()) + if not ws or not all(re.fullmatch(r"\d+", w) for w in ws): + return None + ws = [int(w) for w in ws] + S = sum(ws) + if S == 0: + return None + base = [cents * w // S for w in ws] + rem = [cents * w % S for w in ws] + left = cents - sum(base) + for i in sorted(range(len(ws)), key=lambda i: (-rem[i], i))[:left]: + base[i] += 1 + return [f"{c // 100}.{c % 100:02d}" for c in base] + + CASES = [ + ("100", ["1", "1", "1"]), ("0.05", ["1", "1", "1"]), ("0.01", ["1", "1"]), ("10.00", ["0", "3", "0", "7"]), + ("1234.5", ["5", "3", "2"]), ("0", ["1", "2"]), ("99.99", ["1"] * 7), + ("1000000000.07", ["17", "23", "59", "1"]), ("0.10", ["1", "2", "3"]), ("5", ["7"]), + ("100", ["0", "0"]), ("100", []), ("100", ["-1", "2"]), ("100", ["1.5", "2"]), ("1.234", ["1", "1"]), + ("abc", ["1"]), ("-5", ["1", "2"]), ("0.07", ["1"] * 10), + ("2.00", ["1", "1", "1", "0", "1", "1", "1"]), ("8.33", ["333", "333", "334"]), + ("0.03", ["1", "0", "1", "1"]), ("19.99", ["3", "3"]), + ] + + @staticmethod + def setup(d: Path) -> None: + (d / "README.txt").write_text("Splits invoices between cost centres.\n") + + @staticmethod + def grade(d: Path): + if not (d / "alloc.py").exists(): + return False, "no alloc.py" + rnd = random.Random(2) + cases = list(MoneySplit.CASES) + for _ in range(60): + n = rnd.randint(1, 8) + cases.append((f"{rnd.randint(0, 10 ** rnd.randint(1, 9))}.{rnd.randint(0, 99):02d}", [str(rnd.choice([0, 1, 1, 2, 3, 7, 10, 333])) for _ in range(n)])) + for total, ws in cases: + try: + r = runb(["python3", "alloc.py", total, *ws], cwd=d) + except subprocess.TimeoutExpired: + return False, "timeout" + want = MoneySplit.oracle(total, ws) + if want is None: + if r.returncode != 1 or r.stdout.strip(): + return False, f"{total} {ws}: want exit 1 and no stdout, got {r.returncode} {r.stdout[:30]!r}" + continue + got = r.stdout.decode().split("\n") + if r.returncode != 0 or got != want + [""]: + return False, f"{total} {ws}: got {got[:8]} want {want[:8]}" + return True, f"{len(cases)} cases" + + +# ================================================================ 3. fix-daily-buckets +class FixDailyBuckets: + name = "fix-daily-buckets" + PROMPT = ( + "Our nightly event report (`python3 report.py events.jsonl`) prints a different count per day on the " + "Berlin server than on my laptop, even though both read the same file: the numbers are fine most of the " + "day but wrong around midnight, and it only shows up for some events. The report must give exactly the " + "same output whatever timezone the machine is set to; days and hours are UTC (an event timestamp with an " + "explicit offset means that instant, a timestamp without one is already UTC). Fix it." + ) + + BUCKET = '''from datetime import datetime + + +def to_dt(ts): + """Event timestamp (epoch seconds, or an ISO-8601 string) -> datetime.""" + if isinstance(ts, str): + return datetime.fromisoformat(ts) + return datetime.fromtimestamp(ts) + + +def day_of(ts): + return to_dt(ts).strftime("%Y-%m-%d") + + +def hour_of(ts): + return to_dt(ts).hour +''' + REPORT = '''import json +import sys +from collections import Counter, defaultdict +from datetime import datetime + +import bucket + + +def hour(ts): + if isinstance(ts, (int, float)): + return datetime.fromtimestamp(ts).hour + return bucket.hour_of(ts) + + +def main(path): + per_day = Counter() + per_hour = defaultdict(Counter) + with open(path) as f: + for line in f: + line = line.strip() + if not line: + continue + ts = json.loads(line)["ts"] + day = bucket.day_of(ts) + per_day[day] += 1 + per_hour[day][hour(ts)] += 1 + for day in sorted(per_day): + counts = per_hour[day] + peak = min(counts, key=lambda h: (-counts[h], h)) + print(f"{day} {per_day[day]} peak={peak:02d}") + + +if __name__ == "__main__": + main(sys.argv[1]) +''' + + @staticmethod + def events(seed, n): + rnd = random.Random(seed) + base = 1700000000 - 1700000000 % 86400 + out = [] + for i in range(n): + t = base + rnd.choice([0, 1, 2, 3]) * 86400 + rnd.choice([rnd.randint(0, 86399), rnd.randint(0, 7200), rnd.randint(79200, 86399)]) + k = rnd.random() + if k < 0.5: + ts = t + elif k < 0.6: + ts = t + 0.5 + else: + off = rnd.choice(["+05:30", "-08:00", "+13:00", "Z", "+00:00", ""]) + sign = {"+05:30": 19800, "-08:00": -28800, "+13:00": 46800}.get(off, 0) + ts = (dt.datetime(1970, 1, 1) + dt.timedelta(seconds=t + sign)).strftime("%Y-%m-%dT%H:%M:%S") + off + out.append({"id": i, "ts": ts}) + return out + + @staticmethod + def oracle(evs): + per = {} + for e in evs: + ts = e["ts"] + if isinstance(ts, str): + m = re.fullmatch(r"(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d)(Z|[+-]\d\d:\d\d)?", ts) + t = dt.datetime.fromisoformat(m.group(1)) + o = m.group(2) + if o and o != "Z": + s = 1 if o[0] == "+" else -1 + t -= s * dt.timedelta(hours=int(o[1:3]), minutes=int(o[4:6])) + else: + t = dt.datetime(1970, 1, 1) + dt.timedelta(seconds=ts) + per.setdefault(t.strftime("%Y-%m-%d"), []).append(t.hour) + lines = [] + for day in sorted(per): + hs = per[day] + peak = min(set(hs), key=lambda h: (-hs.count(h), h)) + lines.append(f"{day} {len(hs)} peak={peak:02d}") + return "\n".join(lines) + "\n" + + @staticmethod + def setup(d: Path) -> None: + write(d, "bucket.py", FixDailyBuckets.BUCKET) + write(d, "report.py", FixDailyBuckets.REPORT) + write(d, "events.jsonl", "".join(json.dumps(e) + "\n" for e in FixDailyBuckets.events(1, 60))) + + @staticmethod + def grade(d: Path): + tmp, w = scratch_copy(d) + try: + for seed in (1, 7): + evs = FixDailyBuckets.events(seed, 400) + (w / "e.jsonl").write_text("".join(json.dumps(e) + "\n" for e in evs)) + want = FixDailyBuckets.oracle(evs) + for tz in ("UTC", "EST5", "IST-5:30", "NZST-12NZDT,M9.5.0,M4.1.0/3", "PST8PDT,M3.2.0,M11.1.0"): + r = runb(["python3", "report.py", "e.jsonl"], cwd=w, env={"TZ": tz}) + if r.returncode != 0: + return False, f"TZ={tz}: {r.stderr.decode()[-150:]}" + if r.stdout.decode() != want: + a, b = r.stdout.decode().splitlines(), want.splitlines() + return False, f"TZ={tz}: first diff {next(((x, y) for x, y in zip(a, b) if x != y), (len(a), len(b)))}" + return True, "same output in every timezone" + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +# ================================================================ 4. semver-sort +class SemverSort: + name = "semver-sort" + PROMPT = ( + "Write vsort.py: reads version strings from stdin, one per line, and writes them to stdout sorted in " + "ascending order by Semantic Versioning 2.0.0 precedence (the rules of semver.org item 11, including " + "pre-release identifiers; build metadata after + is ignored for ordering). A single leading `v` is " + "accepted (`v1.2.3`); strings are printed exactly as given (minus the line terminator). Versions of equal " + "precedence keep their input order. Blank lines are ignored. A line that is not a valid semantic version " + "(strict: three numeric parts without leading zeros, optional -prerelease with non-empty dot-separated " + "identifiers of [0-9A-Za-z-] where numeric identifiers have no leading zeros, optional +build with " + "non-empty dot-separated [0-9A-Za-z-] identifiers) is reported to stderr and skipped, and then the exit " + "status is 1 after the valid ones were still printed; otherwise 0." + ) + + ID = r"(?:0|[1-9]\d*|\d*[A-Za-z-][0-9A-Za-z-]*)" + RX = re.compile(r"v?(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-(" + ID + r"(?:\." + ID + r")*))?(?:\+([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?") + + @staticmethod + def oracle(lines): + good, bad = [], 0 + for l in lines: + if not l.strip(): + continue + m = SemverSort.RX.fullmatch(l) + if not m: + bad += 1 + continue + pre = m.group(4) + core = (int(m.group(1)), int(m.group(2)), int(m.group(3))) + if pre is None: + key = core + (1, ()) + else: + ids = tuple((0, int(x), "") if x.isdigit() else (1, 0, x) for x in pre.split(".")) + key = core + (0, ids) + good.append((key, l)) + good.sort(key=lambda x: x[0]) + return [l for _, l in good], (1 if bad else 0) + + CASES = [ + ["1.0.0", "1.0.0-alpha", "1.0.0-alpha.1", "1.0.0-alpha.beta", "1.0.0-beta", "1.0.0-beta.2", "1.0.0-beta.11", "1.0.0-rc.1"], + ["1.10.0", "1.2.0", "1.2.10", "1.2.9", "v1.2.9", "0.9.9", "10.0.0", "2.0.0"], + ["1.0.0+b", "1.0.0+a", "1.0.0", "v1.0.0+z", "1.0.0-0+x", "1.0.0-0"], + ["1.0.0-1", "1.0.0-a", "1.0.0-A", "1.0.0-a-b", "1.0.0-a.b", "1.0.0-a.1", "1.0.0-9", "1.0.0-10", "1.0.0-1a", "1.0.0-01a"], + ["01.0.0", "1.0", "1.0.0.0", "1.0.0-", "1.0.0-01", "1.0.0+", "1.0.0-a..b", "vv1.0.0", "x", "1.0.0-a_b"], + ["", "2.0.0", "", "1.0.0", "", "bad", "99999999999999999999.0.0", "99999999999999999998.0.0"], + ["1.0.0-alpha.9", "1.0.0-alpha.10", "1.0.0-alpha.-", "1.0.0-alpha.a", "1.0.0-alpha"], + ] + + @staticmethod + def setup(d: Path) -> None: + (d / "versions.txt").write_text("1.2.0\n1.10.0\n1.2.0-rc.1\n") + + @staticmethod + def grade(d: Path): + if not (d / "vsort.py").exists(): + return False, "no vsort.py" + rnd = random.Random(4) + cases = [list(c) for c in SemverSort.CASES] + pool = ["1.0.0", "1.0.0-alpha", "1.0.0-alpha.1", "1.0.0-1", "1.0.0-a", "1.0.1", "0.1.0", "1.0.0+build", "v1.0.0-rc.1", "1.0.0-rc.1+x"] + for _ in range(20): + cases.append([rnd.choice(pool) for _ in range(rnd.randint(2, 15))]) + for c in cases: + r = runb(["python3", "vsort.py"], cwd=d, inp="\n".join(c) + "\n") + want, code = SemverSort.oracle(c) + got = r.stdout.decode().splitlines() + if got != want: + return False, f"{c[:4]}...: got {got[:6]} want {want[:6]}" + if r.returncode != code: + return False, f"{c[:4]}...: exit {r.returncode}, want {code}" + r = runb(["python3", "vsort.py"], cwd=d, inp="1.0.0\n1.0.0-b") + if r.stdout.decode() != "1.0.0-b\n1.0.0\n": + return False, "input without trailing newline mishandled" + return True, f"{len(cases)} cases" + + +# ================================================================ 5. json-diff +class JsonDiff: + name = "json-diff-cli" + PROMPT = ( + "Write jdiff.py: `python3 jdiff.py A.json B.json` prints how B differs from A, one line per difference, " + "and exits 1 if there were any, 0 if none, 2 (message on stderr, nothing on stdout) if either file is not " + "readable valid JSON. Paths: the root is `$`; an object member is `.name` when name matches " + "[A-Za-z_][A-Za-z0-9_]*, otherwise `[\"name\"]` written as a JSON string (so `$[\"a.b\"].c`); an array " + "element is `[index]`. Objects are compared key by key in sorted key order and arrays index by index, " + "recursively, and lines are printed in that traversal order. A key or element only in A prints " + "`- PATH: VALUE`; only in B prints `+ PATH: VALUE`; present in both but different (any scalar change, a " + "type change, or object vs array) prints `~ PATH: OLD -> NEW`; two objects or two arrays are never " + "reported themselves, only their contents. VALUE/OLD/NEW are compact JSON: `json.dumps(v, " + "sort_keys=True, separators=(',', ':'), ensure_ascii=False)`. Numbers compare by numeric value (1 equals " + "1.0) but true/false are not numbers (`1` vs `true` is a difference)." + ) + + @staticmethod + def oracle(a, b): + out = [] + + def dump(v): + return json.dumps(v, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + def key(k): + return "." + k if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", k) else "[" + json.dumps(k, ensure_ascii=False) + "]" + + def isnum(x): + return isinstance(x, (int, float)) and not isinstance(x, bool) + + def walk(p, x, y): + if isinstance(x, dict) and isinstance(y, dict): + for k in sorted(set(x) | set(y)): + if k not in y: + out.append(f"- {p}{key(k)}: {dump(x[k])}") + elif k not in x: + out.append(f"+ {p}{key(k)}: {dump(y[k])}") + else: + walk(p + key(k), x[k], y[k]) + elif isinstance(x, list) and isinstance(y, list): + for i in range(max(len(x), len(y))): + if i >= len(y): + out.append(f"- {p}[{i}]: {dump(x[i])}") + elif i >= len(x): + out.append(f"+ {p}[{i}]: {dump(y[i])}") + else: + walk(f"{p}[{i}]", x[i], y[i]) + else: + same = (isnum(x) and isnum(y) and x == y) or (not isnum(x) and not isnum(y) and type(x) is type(y) and x == y) + if not same: + out.append(f"~ {p}: {dump(x)} -> {dump(y)}") + walk("$", a, b) + return out + + CASES = [ + ({"a": 1, "b": [1, 2, 3], "c": {"d": True}}, {"a": 1.0, "b": [1, 2], "c": {"d": 1, "e": None}}), + ({"a.b": {"c": 1}, "": 2, "x y": [1]}, {"a.b": {"c": 2}, "": 3, "x y": []}), + ([1, [2, {"k": "v"}]], [1, [2, {"k": "w"}], 5]), + (1, 2), ({"a": 1}, [1]), ("é", "e"), (None, False), ({"a": {"b": {}}}, {"a": {"b": []}}), + ({"z": 1, "a": 2, "m": {"q": [0]}}, {"z": 1, "a": 2, "m": {"q": [0]}}), + ({"k": "ünï", "n": 1.5, "t": [True, False]}, {"k": "üni", "n": 1.50, "t": [1, 0]}), + ({"1a": 1, "a-b": 2, "_ok": 3, "A9": 4}, {"1a": 2, "a-b": 3, "_ok": 4, "A9": 5}), + ([], {}), ({"a": [[1, 2], [3]]}, {"a": [[1, 2, 3], []]}), + ] + + @staticmethod + def setup(d: Path) -> None: + write(d, "old.json", json.dumps({"name": "svc", "ports": [80, 443], "env": {"DEBUG": False}}, indent=2)) + write(d, "new.json", json.dumps({"name": "svc", "ports": [80, 8443, 9000], "env": {"DEBUG": True, "LOG": "info"}}, indent=2)) + + @staticmethod + def grade(d: Path): + if not (d / "jdiff.py").exists(): + return False, "no jdiff.py" + tmp, w = scratch_copy(d) + try: + for i, (a, b) in enumerate(JsonDiff.CASES): + (w / "a.json").write_text(json.dumps(a, ensure_ascii=False), encoding="utf-8") + (w / "b.json").write_text(json.dumps(b, ensure_ascii=False), encoding="utf-8") + r = runb(["python3", "jdiff.py", "a.json", "b.json"], cwd=w) + want = JsonDiff.oracle(a, b) + got = r.stdout.decode("utf-8").splitlines() + if got != want: + return False, f"case {i}: got {got[:4]} want {want[:4]}" + if r.returncode != (1 if want else 0): + return False, f"case {i}: exit {r.returncode}" + (w / "bad.json").write_text("{not json") + r = runb(["python3", "jdiff.py", "a.json", "bad.json"], cwd=w) + if r.returncode != 2 or r.stdout.strip(): + return False, f"invalid JSON: exit {r.returncode}, stdout {r.stdout[:30]!r}" + r = runb(["python3", "jdiff.py", "a.json", "missing.json"], cwd=w) + if r.returncode != 2: + return False, f"missing file: exit {r.returncode}, want 2" + return True, f"{len(JsonDiff.CASES)} cases" + finally: + shutil.rmtree(tmp, ignore_errors=True) + + + +# ================================================================ 6. refactor-invoice +INVOICE_ORIG = '''REGIONS = {"US-CA": 725, "US-NY": 400, "US-OR": 0, "DE": 1900, "FR": 2000} + + +def render(order): + region = order["region"] + eu = region in ("DE", "FR") + lines = order["lines"] + sub = 0 + for ln in lines: + sub += ln["qty"] * ln["cents"] + disc = 0 + coupon = order.get("coupon") + if coupon == "SAVE10": + disc = sub * 10 // 100 + elif coupon == "FIVER": + if sub >= 2000: + disc = 500 + elif coupon is not None and coupon != "": + raise ValueError("unknown coupon: " + str(coupon)) + base = sub - disc + if region not in REGIONS: + raise ValueError("unknown region: " + str(region)) + bps = REGIONS[region] + tax = (base * bps + 5000) // 10000 + ship = 0 if base >= 5000 else 599 + if eu: + ship += 300 + total = base + tax + ship + + def money(cents): + neg = cents < 0 + whole, frac = divmod(abs(cents), 100) + if eu: + s = f"{whole:,}".replace(",", ".") + "," + f"{frac:02d}" + " \\u20ac" + else: + s = "$" + f"{whole:,}" + "." + f"{frac:02d}" + return "-" + s if neg else s + + out = [] + out.append("INVOICE " + str(order["id"])) + out.append("Customer: " + order["customer"]) + out.append("Region: " + region) + out.append("-" * 44) + for ln in lines: + desc = ln["desc"] + if len(desc) > 20: + desc = desc[:19] + "\\u2026" + out.append(f"{desc:<20} {ln['qty']:>3} x {money(ln['cents']):>10} {money(ln['qty'] * ln['cents']):>12}") + out.append("-" * 44) + out.append(f"{'Subtotal':<20}{money(sub):>24}") + if disc: + out.append(f"{'Discount (' + coupon + ')':<20}{money(-disc):>24}") + out.append(f"{'Tax (' + format(bps / 100, 'g') + '%)':<20}{money(tax):>24}") + out.append(f"{'Shipping':<20}{money(ship):>24}") + out.append(f"{'TOTAL':<20}{money(total):>24}") + return "\\n".join(out) + "\\n" +''' + +INVOICE_ORIG = INVOICE_ORIG.replace("\\u20ac", "\u20ac").replace("\\u2026", "\u2026") + + +def _rand_orders(seed, n): + rnd = random.Random(seed) + words = ["Widget", "Gadget with a very long descriptive name", "Cable", "Ünïcode thing", "Bolt", "Thing 12345678901234567890", "Exactly twenty chars"] + out = [] + for i in range(n): + lines = [{"sku": f"S{rnd.randint(1, 99)}", "desc": rnd.choice(words), "qty": rnd.randint(0, 40), "cents": rnd.choice([1, 99, 250, 1999, 100000, 123456, -500, 3333])} + for _ in range(rnd.randint(0, 5))] + o = {"id": rnd.choice([i, f"INV-{i}"]), "region": rnd.choice(["US-CA", "US-NY", "US-OR", "DE", "FR", "DE", "US-CA", "XX", "us-ca"]), + "customer": rnd.choice(["Acme", "Müller GmbH", "O'Neil"]), "lines": lines} + c = rnd.choice([None, None, "SAVE10", "FIVER", "", "NOPE"]) + if c is not None: + o["coupon"] = c + out.append(o) + return out + + +INVOICE_HARNESS = r''' +import importlib.util, json, sys +def load(name, path): + spec = importlib.util.spec_from_file_location(name, path); m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m); return m +sys.path.insert(0, ".") +old = load("_orig_invoice", "_orig_invoice.py") +import invoice, tax, format as fmt +orders = json.load(open("_orders.json")) +def call(f, o): + try: return ("ok", f(o)) + except Exception as e: return ("err", type(e).__name__, str(e)) +for i, o in enumerate(orders): + a, b = call(old.render, o), call(invoice.render, o) + if a != b: + print(f"order {i} ({o['region']}, coupon={o.get('coupon')!r}): old {a!r:.120} new {b!r:.120}"); sys.exit() +for base, region in [(10000, "US-CA"), (1, "US-CA"), (-1234, "DE"), (-1, "FR"), (0, "US-OR"), (99999, "US-NY"), (6, "US-CA"), (-6, "US-CA"), (13, "FR"), (-13, "FR"), (150, "DE"), (200, "US-CA"), (-150, "DE"), (250, "FR")]: + want = (base * old.REGIONS[region] + 5000) // 10000 + got = tax.compute_tax(base, region) + if got != want or type(got) is not int: + print(f"compute_tax({base}, {region!r}) = {got!r}, want {want}"); sys.exit() +try: + tax.compute_tax(100, "XX"); print("compute_tax accepted an unknown region"); sys.exit() +except ValueError as e: + if str(e) != "unknown region: XX": print(f"wrong message {e}"); sys.exit() +for c, region, want in [(123456, "US-CA", "$1,234.56"), (-5, "US-NY", "-$0.05"), (123456789, "DE", "1.234.567,89 \u20ac"), (-100, "FR", "-1,00 \u20ac"), (0, "US-OR", "$0.00"), (7, "FR", "0,07 \u20ac")]: + got = fmt.money(c, region) + if got != want: print(f"money({c}, {region!r}) = {got!r}, want {want!r}"); sys.exit() +src = open("invoice.py").read() +for bad in ("1900", "725", "\u20ac", "REGIONS", "divmod"): + if bad in src: print(f"invoice.py still contains {bad!r}: the logic was not moved"); sys.exit() +print("OK") +''' + + +class RefactorInvoice: + name = "refactor-invoice" + PROMPT = ( + "invoice.py's render(order) is one long function that mixes tax rules, money formatting and layout. " + "Refactor it: tax goes in tax.py as `compute_tax(base_cents, region)` (returns the tax in cents, raises " + "ValueError for an unknown region exactly as render does today), money formatting goes in format.py as " + "`money(cents, region)` (returns the string render prints today for that region), and render(order) stays " + "in invoice.py, calling them. Behaviour must not change in any way: byte-identical output for every order, " + "and the same exception types and messages for the bad ones, including which error you get when an order " + "has more than one problem. The rate table and the formatting rules must live in the new modules, not " + "still in invoice.py. golden.json has sample orders with today's output." + ) + + @staticmethod + def setup(d: Path) -> None: + write(d, "invoice.py", INVOICE_ORIG) + ns: dict = {} + exec(compile(INVOICE_ORIG, "orig", "exec"), ns) + gold = [] + for o in _rand_orders(99, 12): + try: + gold.append({"order": o, "output": ns["render"](o)}) + except Exception as e: + gold.append({"order": o, "error": f"{type(e).__name__}: {e}"}) + write(d, "golden.json", json.dumps(gold, indent=1, ensure_ascii=False)) + + @staticmethod + def grade(d: Path): + tmp, w = scratch_copy(d) + try: + for f in ("tax.py", "format.py", "invoice.py"): + if not (w / f).exists(): + return False, f"no {f}" + write(w, "_orig_invoice.py", INVOICE_ORIG) + write(w, "_orders.json", json.dumps(_rand_orders(5, 500), ensure_ascii=False)) + return py_check(w, INVOICE_HARNESS) + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +# ================================================================ 7. sql-longest-gaps +class SqlGaps: + name = "sql-longest-gaps" + PROMPT = ( + "sensors.db (SQLite) has a table readings(id, sensor, ts, value). ts is text in UTC unless it carries an " + "explicit offset, and comes in the forms 2024-03-01T10:00:00Z, 2024-03-01 10:00:00 and " + "2024-03-01T12:00:00+02:00. Rows whose value is NULL are faulty and must be ignored. For each sensor with " + "at least two valid readings, find its longest gap in seconds between two consecutive valid readings in " + "time order (compare instants, not text); if several gaps tie, take the earliest one. Write " + "longest_gaps.csv with the header sensor,gap_seconds,gap_start, where gap_start is the time of the earlier " + "reading of the gap in UTC as YYYY-MM-DDTHH:MM:SSZ, rows ordered by gap_seconds descending, then sensor " + "ascending." + ) + + @staticmethod + def parse(ts): + m = re.fullmatch(r"(\d{4})-(\d\d)-(\d\d)[T ](\d\d):(\d\d):(\d\d)(Z|[+-]\d\d:\d\d)?", ts) + t = dt.datetime(*(int(x) for x in m.groups()[:6])) + o = m.group(7) + if o and o != "Z": + t -= (1 if o[0] == "+" else -1) * dt.timedelta(hours=int(o[1:3]), minutes=int(o[4:6])) + return t + + @staticmethod + def make_db(path: Path): + rnd = random.Random(21) + db = sqlite3.connect(path) + db.execute("CREATE TABLE readings(id INTEGER PRIMARY KEY, sensor TEXT NOT NULL, ts TEXT NOT NULL, value REAL)") + base = dt.datetime(2024, 3, 1) + for s in ["s1", "s2", "s3", "s4", "s5", "s6", "t10", "t9"]: + n = {"s5": 3, "s6": 1}.get(s, rnd.randint(8, 30)) + for _ in range(n): + t = base + dt.timedelta(seconds=rnd.randint(0, 86400 * 5)) + k = rnd.random() + if k < 0.34: + txt = t.strftime("%Y-%m-%dT%H:%M:%SZ") + elif k < 0.67: + txt = t.strftime("%Y-%m-%d %H:%M:%S") + else: + h = rnd.choice([2, -5, 9]) + tt = t + dt.timedelta(hours=h) + txt = tt.strftime("%Y-%m-%dT%H:%M:%S") + ("+" if h > 0 else "-") + f"{abs(h):02d}:00" + val = None if (s == "s5" or rnd.random() < 0.2) else round(rnd.random() * 10, 2) + if s == "s5" and _ == 0: + val = 1.0 + db.execute("INSERT INTO readings(sensor,ts,value) VALUES(?,?,?)", (s, txt, val)) + # a tie: two equal 1-hour gaps for s7, written in a scrambled order + for txt in ["2024-03-02T10:00:00Z", "2024-03-02 08:00:00", "2024-03-02T09:00:00Z", "2024-03-02 09:00:00", "2024-03-02T11:00:00+00:00"]: + db.execute("INSERT INTO readings(sensor,ts,value) VALUES('s7',?,1.0)", (txt,)) + db.commit() + db.close() + + @staticmethod + def oracle(path: Path): + db = sqlite3.connect(path) + by = {} + for s, ts in db.execute("SELECT sensor, ts FROM readings WHERE value IS NOT NULL"): + by.setdefault(s, []).append(SqlGaps.parse(ts)) + rows = [] + for s, ts in by.items(): + ts.sort() + if len(ts) < 2: + continue + best = max(range(len(ts) - 1), key=lambda i: ((ts[i + 1] - ts[i]).total_seconds(), -i)) + rows.append([s, str(int((ts[best + 1] - ts[best]).total_seconds())), ts[best].strftime("%Y-%m-%dT%H:%M:%SZ")]) + rows.sort(key=lambda r: (-int(r[1]), r[0])) + db.close() + return rows + + @staticmethod + def setup(d: Path) -> None: + SqlGaps.make_db(d / "sensors.db") + + @staticmethod + def grade(d: Path): + p = d / "longest_gaps.csv" + if not p.exists(): + return False, "no longest_gaps.csv" + tmp = Path(tempfile.mkdtemp(prefix="grade3-")) + try: + SqlGaps.make_db(tmp / "s.db") + want = SqlGaps.oracle(tmp / "s.db") + finally: + shutil.rmtree(tmp, ignore_errors=True) + got = [r for r in csv.reader(p.read_text().splitlines()) if r] + if not got or got[0] != ["sensor", "gap_seconds", "gap_start"]: + return False, f"bad header {got[:1]}" + got = got[1:] + if got != want: + return False, f"first difference: {next(((a, b) for a, b in zip(got, want) if a != b), (len(got), len(want)))}" + return True, "matches" + + +# ================================================================ 8. git-extract-subtree-history +class GitSplitHistory: + name = "git-split-lib-history" + PROMPT = ( + "This repo keeps a library in lib/ and an application in app/ together. Create a branch called lib-only " + "whose history contains only the commits that changed something under lib/, each one reduced to its " + "changes under lib/ (files keep their paths, e.g. lib/parser.py; nothing outside lib/ in any of its " + "trees), in the same order, with the same commit messages and author dates. Commits that touched nothing " + "under lib/ must not appear on it. Do not alter master or any existing commit, and leave the working " + "tree clean." + ) + + @staticmethod + def setup(d: Path) -> None: + vcs(d, "init", "-q", "-b", "master") + vcs(d, "config", "commit.gpgsign", "false") + n = [0] + + def commit(msg, files: dict, remove=(), move=None): + n[0] += 1 + for rel, txt in files.items(): + write(d, rel, txt) + if move: + (d / move[1]).parent.mkdir(parents=True, exist_ok=True) + vcs(d, "mv", move[0], move[1]) + for rel in remove: + vcs(d, "rm", "-q", rel) + vcs(d, "add", "-A") + vcs(d, "commit", "-q", "-m", msg, date=f"2024-01-{n[0]:02d}T12:00:00+00:00") + + commit("Initial readme", {"README.md": "# proj\n"}) + commit("lib: add parser", {"lib/__init__.py": "", "lib/parser.py": "def parse(s):\n return s.split(',')\n"}) + commit("app: first screen", {"app/main.py": "print('hi')\n"}) + commit("Add util and glue", {"lib/util.py": "def clamp(x, lo, hi):\n return max(lo, min(x, hi))\n", "app/glue.py": "from lib import util\n"}) + commit("docs: usage", {"README.md": "# proj\nUsage: see app/main.py\n"}) + commit("lib: fix parser off-by-one", {"lib/parser.py": "def parse(s):\n return [p.strip() for p in s.split(',')]\n"}) + commit("lib: move util into core", {}, move=("lib/util.py", "lib/core/util.py")) + commit("app: tweak", {"app/main.py": "print('hello')\n"}) + commit("Parser speedup and app update", {"lib/parser.py": "def parse(s):\n return [p.strip() for p in s.split(',') if p]\n", "app/main.py": "print('hello, world')\n"}) + commit("app: drop glue", {}, remove=["app/glue.py"]) + + @staticmethod + def grade(d: Path): + tmp = Path(tempfile.mkdtemp(prefix="grade3-")) + try: + GitSplitHistory.setup(tmp) + master0 = vcs(tmp, "rev-parse", "master").strip() + if vcs(d, "rev-parse", "master").strip() != master0: + return False, "master was altered" + if vcs(d, "rev-parse", "--verify", "-q", "refs/heads/lib-only").strip() == "": + return False, "no branch lib-only" + fmt = "%s|%aI" + want = vcs(tmp, "log", "--reverse", f"--format={fmt}", "master", "--", "lib").splitlines() + want_sha = vcs(tmp, "rev-list", "--reverse", "master", "--", "lib").split() + got = vcs(d, "log", "--reverse", f"--format={fmt}", "lib-only").splitlines() + got_sha = vcs(d, "rev-list", "--reverse", "lib-only").split() + if got != want: + return False, f"lib-only history is {[g.split('|')[0] for g in got]}, want {[w.split('|')[0] for w in want]}" + for g, w in zip(got_sha, want_sha): + names = [x for x in vcs(d, "ls-tree", "--name-only", g).splitlines()] + if names != ["lib"]: + return False, f"commit {g[:7]} has top-level entries {names}" + if vcs(d, "rev-parse", f"{g}:lib").strip() != vcs(tmp, "rev-parse", f"{w}:lib").strip(): + return False, f"commit {g[:7]}: lib/ differs from the original commit's lib/" + if len(vcs(d, "rev-list", "--merges", "lib-only").split()): + return False, "merge commits on lib-only" + st = [l for l in vcs(d, "status", "--porcelain").splitlines() if not re.search(r"\.(maat|molt)/?$", l)] + if st: + return False, f"working tree not clean: {st[:3]}" + return True, f"{len(got)} commits, trees match" + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +# ================================================================ 9. git-find-culprit +CHECK_SH = '''#!/bin/sh +v=$(grep '^max_conns' limits.conf | head -1 | cut -d= -f2 | tr -d ' ') +if [ "$v" -lt 500 ]; then + echo ok +else + echo "max_conns too high: $v" + exit 1 +fi +''' + + +class GitFindCulprit: + name = "git-find-culprit" + PROMPT = ( + "`sh check.sh` passes on the first commit of this repo and fails on master now. Find the first commit " + "where it fails and write its full 40-character hash (and nothing else) to culprit.txt. Don't rewrite " + "history or move branches: leave the repo on master with no bisect or checkout left in progress, and " + "don't commit culprit.txt." + ) + + @staticmethod + def setup(d: Path) -> None: + vcs(d, "init", "-q", "-b", "master") + vcs(d, "config", "commit.gpgsign", "false") + limits = {1: "max_conns = 100\ntimeout = 30\n", 9: "max_conns = 450\ntimeout = 30\n", 13: "max_conns = 450\ntimeout = 60\n", + 15: "max_conns= 900\ntimeout = 60\n", 18: "max_conns = 900\ntimeout = 60\nretries = 3\n"} + msgs = {9: "raise connection limit for load test", 13: "longer timeout", 15: "tidy whitespace in limits.conf", + 18: "limits.conf: add retries", 21: "fix flaky check"} + write(d, "check.sh", CHECK_SH) + for i in range(1, 25): + if i in limits: + write(d, "limits.conf", limits[i]) + write(d, f"notes/n{i:02d}.txt", f"note {i}\n") + if i == 1: + write(d, "README", "service config\n") + vcs(d, "add", "-A") + vcs(d, "commit", "-q", "-m", msgs.get(i, f"notes {i}"), date=f"2024-02-{i:02d}T09:00:00+00:00") + + @staticmethod + def grade(d: Path): + p = d / "culprit.txt" + if not p.exists(): + return False, "no culprit.txt" + tmp = Path(tempfile.mkdtemp(prefix="grade3-")) + try: + GitFindCulprit.setup(tmp) + revs = vcs(tmp, "rev-list", "--reverse", "master").split() + first_bad = None + for r in revs: + x = Path(tempfile.mkdtemp(prefix="co-", dir=tmp)) + subprocess.run(f"git archive {r} | tar -x -C {x}", shell=True, cwd=tmp, capture_output=True) + if subprocess.run(["sh", "check.sh"], cwd=x, capture_output=True).returncode != 0: + first_bad = r + break + if vcs(d, "rev-list", "--reverse", "master").split() != revs: + return False, "history of master changed" + if vcs(d, "rev-parse", "--abbrev-ref", "HEAD").strip() != "master": + return False, "HEAD is not on master" + if (d / ".git" / "BISECT_START").exists() or (d / ".git" / "BISECT_LOG").exists(): + return False, "a bisect is still in progress" + if vcs(d, "ls-files", "culprit.txt").strip(): + return False, "culprit.txt was committed" + got = p.read_text().strip() + if got != first_bad: + return False, f"culprit.txt has {got[:12]}, the first failing commit is {first_bad[:12]}" + if not re.fullmatch(r"[0-9a-f]{40}\n?", p.read_text()): + return False, "culprit.txt must hold only the 40-character hash" + return True, "right commit" + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +# ================================================================ 10. bash-backup +class BackupScript: + name = "bash-backup" + PROMPT = ( + "Write backup.sh (bash): `bash backup.sh SRC DEST KEEP` archives the directory SRC into " + "DEST/backup-STAMP.tar.gz and then prunes old archives. STAMP is the value of the environment variable " + "BACKUP_NOW if it is set (format YYYYmmdd-HHMMSS), otherwise the current local time in that format. DEST is " + "created if missing. Archive member paths are relative to SRC itself (docs/a.txt, not SRC/docs/a.txt), " + "dotfiles included, names with spaces preserved; files whose name ends in .tmp and any directory named " + ".cache (at any depth) are left out, and so is anything inside DEST, which may itself be inside SRC. " + "After writing the archive, keep only the KEEP newest files in DEST named exactly " + "backup-YYYYmmdd-HHMMSS.tar.gz, newest by the STAMP in the name, and delete the others; never touch any " + "other file in DEST. Errors: if SRC is not a directory, or KEEP is not a positive integer, exit 2 and " + "change nothing (do not even create DEST); if the archive for that STAMP already exists, exit 3 and " + "change nothing." + ) + + FILES = { + "a.txt": "A\n", "docs/b.md": "B\n", "docs/x y.txt": "XY\n", ".hidden": "H\n", "my.cache/keep.txt": "K\n", + ".cachefile": "C\n", "deep/er/est/f.bin": "F\n", "skip.tmp": "T\n", "docs/notes.tmp": "T\n", + "docs/.cache/z": "Z\n", ".cache/q": "Q\n", "deep/.cache/er/w": "W\n", + } + EXPECT = {"a.txt", "docs/b.md", "docs/x y.txt", ".hidden", "my.cache/keep.txt", ".cachefile", "deep/er/est/f.bin"} + + @staticmethod + def members(archive: Path): + r = runb(["tar", "-tzf", str(archive)]) + if r.returncode != 0: + return None + out = set() + for l in r.stdout.decode().splitlines(): + l = l[2:] if l.startswith("./") else l + if l and not l.endswith("/"): + out.add(l) + return out + + @staticmethod + def setup(d: Path) -> None: + for rel, txt in BackupScript.FILES.items(): + write(d, "data/" + rel, txt) + write(d, "README.txt", "Run backup.sh nightly from cron.\n") + + @staticmethod + def grade(d: Path): + if not (d / "backup.sh").exists(): + return False, "no backup.sh" + tmp = Path(tempfile.mkdtemp(prefix="grade3-")) + try: + shutil.copy(d / "backup.sh", tmp / "backup.sh") + src = tmp / "my src" + for rel, txt in BackupScript.FILES.items(): + write(src, rel, txt) + before = digest(src) + + def bk(src_arg, dest, keep, stamp=None): + env = {"BACKUP_NOW": stamp} if stamp else {} + return runb(["bash", "backup.sh", src_arg, dest, str(keep)], cwd=tmp, env=env) + + dest = tmp / "out" + for st in ("20240101-000000", "20240102-000000", "20240103-120000", "20240104-235959"): + r = bk("my src", "out", 3, st) + if r.returncode != 0: + return False, f"run {st} exited {r.returncode}: {r.stderr.decode()[-120:]}" + if st == "20240101-000000": + for name, txt in (("notes.txt", "n"), ("backup-latest.tar.gz", "l"), ("backup-20240101-0000.tar.gz", "s"), ("backup-20240101-000000.tar.gz.sig", "g"), ("backup-20230101-000000.txt", "t")): + write(dest, name, txt) + have = sorted(x.name for x in dest.iterdir()) + want = sorted(["notes.txt", "backup-latest.tar.gz", "backup-20240101-0000.tar.gz", "backup-20240101-000000.tar.gz.sig", "backup-20230101-000000.txt", + "backup-20240102-000000.tar.gz", "backup-20240103-120000.tar.gz", "backup-20240104-235959.tar.gz"]) + if have != want: + return False, f"DEST holds {have}" + m = BackupScript.members(dest / "backup-20240104-235959.tar.gz") + if m != BackupScript.EXPECT: + return False, f"archive members differ: extra {sorted((m or set()) - BackupScript.EXPECT)[:3]}, missing {sorted(BackupScript.EXPECT - (m or set()))[:3]}" + if digest(src) != before: + return False, "SRC was modified" + # same stamp again -> exit 3, nothing changed + snap = digest(dest) + r = bk("my src", "out", 1, "20240104-235959") + if r.returncode != 3 or digest(dest) != snap: + return False, f"existing archive: exit {r.returncode}, DEST changed={digest(dest) != snap}" + # an older stamp than KEEP newest is pruned straight away + r = bk("my src", "out", 3, "20240101-000000") + if r.returncode != 0 or (dest / "backup-20240101-000000.tar.gz").exists() or not (dest / "backup-20240102-000000.tar.gz").exists(): + return False, "stamp older than the KEEP newest must be pruned, newer ones kept (ordering is by name, not mtime)" + # a stamp in the middle + r = bk("my src", "out", 3, "20240103-000000") + left = sorted(x.name for x in dest.glob("backup-????????-??????.tar.gz")) + if left != ["backup-20240103-000000.tar.gz", "backup-20240103-120000.tar.gz", "backup-20240104-235959.tar.gz"]: + return False, f"after mid-stamp run archives are {left}" + # errors change nothing, not even DEST + write(tmp, "a.txt", "x") + for args in (("nope", "never1", 3), ("my src", "never2", 0), ("my src", "never3", "abc"), ("my src", "never4", -1), ("my src", "never5", "2x"), ("a.txt", "never6", 2)): + r = bk(args[0], args[1], args[2], "20240110-000000") + if r.returncode != 2 or (tmp / args[1]).exists(): + return False, f"bad args {args}: exit {r.returncode}, DEST created={(tmp / args[1]).exists()}" + # DEST inside SRC + r = bk("my src", "my src/backups", 2, "20240201-000000") + r2 = bk("my src", "my src/backups", 2, "20240202-000000") + if r.returncode or r2.returncode: + return False, f"DEST inside SRC exited {r.returncode}/{r2.returncode}" + m = BackupScript.members(src / "backups" / "backup-20240202-000000.tar.gz") + if m != BackupScript.EXPECT: + return False, f"DEST inside SRC: archive members {sorted((m or set()) ^ BackupScript.EXPECT)[:4]}" + return True, "all scenarios" + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +# ================================================================ 11. make-incremental +MAKEFILE_BAD = """SRCS := $(sort $(wildcard src/*.md)) +OUTS := $(SRCS:src/%.md=out/%.txt) + +all: out/all.txt + +out/all.txt: $(OUTS) +\tcat $(OUTS) > $@ +\techo "build $@" >> build.log + +out/%.txt: src/%.md FORCE +\tmkdir -p out +\t(cat common/header.txt; tr a-z A-Z < $<) > $@ +\techo "build $@" >> build.log + +FORCE: + +clean: +\trm -rf out + +.PHONY: all clean +""" + + +class MakeIncremental: + name = "make-incremental" + PROMPT = ( + "`make` in this project rebuilds everything every time. It should only rebuild what is stale: after a " + "full build, running make again does nothing; editing one src/x.md rebuilds out/x.txt and out/all.txt and " + "nothing else; editing common/header.txt (every out/*.txt starts with it) rebuilds every out/*.txt and " + "out/all.txt; adding a new src/y.md builds out/y.txt and out/all.txt only. `make clean` should leave the " + "tree as it was before the first build (out/ and build.log gone). A full `make -j4` from clean must also " + "give correct output. Fix the Makefile; don't change what the recipes produce or the lines they log." + ) + + FILES = {"src/alpha.md": "first file\nline two\n", "src/beta.md": "second file\n", "src/gamma.md": "third\n", + "common/header.txt": "# generated\n"} + + @staticmethod + def setup(d: Path) -> None: + for k, v in MakeIncremental.FILES.items(): + write(d, k, v) + write(d, "Makefile", MAKEFILE_BAD) + + @staticmethod + def expect_all(w: Path): + header = (w / "common/header.txt").read_text() + return "".join(header + (w / f"src/{n}.md").read_text().upper() for n in sorted(p.stem for p in (w / "src").glob("*.md"))) + + @staticmethod + def grade(d: Path): + if not (d / "Makefile").exists(): + return False, "no Makefile" + tmp, w = scratch_copy(d) + try: + shutil.rmtree(w / "out", ignore_errors=True) + (w / "build.log").unlink(missing_ok=True) + + def mk(*a): + r = runb(["make", *a], cwd=w, timeout=60) + return r.returncode, ((w / "build.log").read_text().splitlines() if (w / "build.log").exists() else []) + + def age_outputs(t): + for p in (w / "out").glob("*"): + os.utime(p, (t, t)) + + t0 = int(time.time()) - 100000 + for p in w.rglob("*"): + if p.is_file() and ".git" not in p.parts: + os.utime(p, (t0, t0)) + rc, log = mk() + if rc != 0: + return False, "make failed on a clean tree" + if sorted(log[:-1]) != ["build out/alpha.txt", "build out/beta.txt", "build out/gamma.txt"] or log[-1] != "build out/all.txt": + return False, f"first build logged {log}" + if (w / "out/all.txt").read_text() != MakeIncremental.expect_all(w): + return False, "out/all.txt content is wrong" + if (w / "out/beta.txt").read_text() != "# generated\nSECOND FILE\n": + return False, "out/beta.txt content is wrong" + age_outputs(t0 + 100) + n = len(log) + rc, log = mk() + if rc != 0 or len(log) != n: + return False, f"a second make rebuilt {log[n:]}" + # edit one source + write(w, "src/beta.md", "second file, edited\n") + os.utime(w / "src/beta.md", (t0 + 200, t0 + 200)) + rc, log2 = mk() + if log2[n:] != ["build out/beta.txt", "build out/all.txt"]: + return False, f"editing src/beta.md rebuilt {log2[n:]}" + if "SECOND FILE, EDITED" not in (w / "out/all.txt").read_text(): + return False, "all.txt not refreshed after editing a source" + # edit the header + age_outputs(t0 + 250) + n = len(log2) + write(w, "common/header.txt", "# generated v2\n") + os.utime(w / "common/header.txt", (t0 + 300, t0 + 300)) + rc, log3 = mk() + new = log3[n:] + if sorted(new[:-1]) != ["build out/alpha.txt", "build out/beta.txt", "build out/gamma.txt"] or new[-1:] != ["build out/all.txt"]: + return False, f"editing common/header.txt rebuilt {new}" + if (w / "out/gamma.txt").read_text() != "# generated v2\nTHIRD\n": + return False, "header edit did not reach out/gamma.txt" + # add a source + age_outputs(t0 + 350) + n = len(log3) + write(w, "src/delta.md", "fourth\n") + os.utime(w / "src/delta.md", (t0 + 400, t0 + 400)) + rc, log4 = mk() + if log4[n:] != ["build out/delta.txt", "build out/all.txt"]: + return False, f"adding src/delta.md rebuilt {log4[n:]}" + if (w / "out/all.txt").read_text() != MakeIncremental.expect_all(w): + return False, "out/all.txt wrong after adding a source" + # clean, then a parallel build + rc, _ = mk("clean") + if rc != 0 or (w / "out").exists() or (w / "build.log").exists(): + return False, "make clean left out/ or build.log behind" + rc, log5 = mk("-j4") + if rc != 0 or len(log5) != 5 or (w / "out/all.txt").read_text() != MakeIncremental.expect_all(w): + return False, f"make -j4 from clean: rc={rc}, log={log5}" + return True, "incremental, clean and parallel builds right" + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +# ================================================================ 12. csv-to-json-node +class CsvToJsonNode: + name = "csv-to-json-node" + PROMPT = ( + "Write csv2json.js (Node 22, no npm packages): it reads CSV from stdin and writes to stdout a JSON array " + "with one object per data row, keyed by the header row, all values strings. Format: fields separated by " + "commas; a field may be wrapped in double quotes, and then may contain commas, line breaks (kept exactly " + "as they are) and doubled quotes (\"\" is one quote); a quote in the middle of an unquoted field is " + "literal. Records end at LF or CRLF. A UTF-8 byte-order mark at the very start is ignored. Completely empty " + "lines between records are skipped, and a trailing newline does not make a record. If a header name " + "repeats, its second occurrence becomes name_2, the third name_3; an empty header becomes colN, N being " + "its 1-based column number. A row with fewer fields than the header is padded with empty strings. A row " + "with more fields than the header, or a quoted field that is never closed, is an error: a message on " + "stderr, nothing on stdout, exit status 1. A header with no rows, or empty input, gives []." + ) + + @staticmethod + def oracle(data: bytes): + s = data.decode("utf-8") + if s.startswith("\ufeff"): + s = s[1:] + recs, rec, field = [], [], [] + fstart, inq, any_char = True, False, False + i, n = 0, len(s) + while i < n: + c = s[i] + if inq: + if c == '"': + if s[i + 1:i + 2] == '"': + field.append('"') + i += 1 + else: + inq = False + else: + field.append(c) + elif c == '"' and fstart: + inq, fstart, any_char = True, False, True + elif c == ",": + rec.append("".join(field)) + field, fstart, any_char = [], True, True + elif c == "\n" or (c == "\r" and s[i + 1:i + 2] == "\n"): + if c == "\r": + i += 1 + if any_char: + rec.append("".join(field)) + recs.append(rec) + rec, field, fstart, any_char = [], [], True, False + else: + field.append(c) + fstart, any_char = False, True + i += 1 + if inq: + raise ValueError("unterminated") + if any_char: + rec.append("".join(field)) + recs.append(rec) + if not recs: + return [] + head = recs[0] + names, seen = [], {} + for k, h in enumerate(head, 1): + h = h if h != "" else f"col{k}" + seen[h] = seen.get(h, 0) + 1 + names.append(h if seen[h] == 1 else f"{h}_{seen[h]}") + out = [] + for r in recs[1:]: + if len(r) > len(names): + raise ValueError("too many fields") + out.append(dict(zip(names, r + [""] * (len(names) - len(r))))) + return out + + CASES = [ + b"a,b,c\n1,2,3\n4,5,6\n", + b"a,b\r\n1,2\r\n3,4", + b"\xef\xbb\xbfname,note\n\"Smith, J\",\"said \"\"hi\"\"\"\n", + b"k,v\n\"line1\nline2\",x\n\"crlf\r\ninside\",y\n", + b"a,b,c\n1,2\n1\n", + b"a,b\n\n1,2\n\n\n3,4\n\n", + b"x,x,x,y\n1,2,3,4\n", + b",b,\n1,2,3\n", + b"", + b"a,b\n", + b"\n\na,b\n1,2\n", + b"a,b\n1,2,3\n", + b"a,b\n\"unclosed,2\n", + b"a,b\nab\"c,d\"e\n", + b"a\n\"\"\n\"x\"\n", + b"a,b\n,\n 1 , 2 \n", + b"h\n\xc3\xa9\xe2\x82\xac\xf0\x9f\x98\x80\n", + b"a,b\n\"\",\"\"\n", + b"a\n\r\n1\r\n", + ] + + @staticmethod + def setup(d: Path) -> None: + write(d, "sample.csv", 'id,name,comment\n1,"Ada, Countess","wrote ""the"" notes"\n2,Ben,\n') + + @staticmethod + def grade(d: Path): + if not (d / "csv2json.js").exists(): + return False, "no csv2json.js" + for i, data in enumerate(CsvToJsonNode.CASES): + try: + want = CsvToJsonNode.oracle(data) + except ValueError: + want = None + try: + r = runb(["node", "csv2json.js"], cwd=d, inp=data) + except subprocess.TimeoutExpired: + return False, f"case {i}: timeout" + if want is None: + if r.returncode != 1 or r.stdout.strip(): + return False, f"case {i} {data[:30]!r}: want error exit 1 with empty stdout, got exit {r.returncode}, stdout {r.stdout[:40]!r}" + continue + try: + got = json.loads(r.stdout.decode("utf-8")) + except ValueError: + return False, f"case {i} {data[:30]!r}: stdout is not JSON (exit {r.returncode}): {r.stdout[:40]!r} {r.stderr[-100:]!r}" + if r.returncode != 0 or got != want: + return False, f"case {i} {data[:30]!r}: got {str(got)[:80]} want {str(want)[:80]}" + return True, f"{len(CsvToJsonNode.CASES)} cases" + + +# ================================================================ 13. ini-to-json +class IniToJson: + name = "ini-to-json" + PROMPT = ( + "Write ini2json.py: `python3 ini2json.py in.ini out.json` converts our INI config to JSON. Rules: " + "`[section]` headers; `key = value` lines (split at the first `=`, key and value stripped of surrounding " + "whitespace); keys before the first section go at the top level of the JSON object. Lines starting " + "(after whitespace) with ; or # are comments, and so is the rest of a value from a ; or # that is the " + "first character of the value or is preceded by whitespace (`a;b` is literal, `a ;b` is `a`). A value " + "wrapped in double quotes is taken literally between the quotes (no escapes, comment characters inside " + "are literal) and is always a string; only a comment may follow the closing quote. The section " + "[DEFAULT], wherever it appears, supplies keys to every other section (the section's own value wins); it " + "does not appear in the output. A dot in a section name nests: [db.pool] is the object db -> pool, and " + "[db] and [db.pool] merge. Repeated keys: the last wins; a repeated section header continues the " + "section. `${key}` in a value is replaced by that key's value text from the same section (including " + "inherited DEFAULT keys), and `${section.key}` (split at the last dot) by the value text of that key in " + "that section; this applies to quoted values too and works through chains of references. Types for " + "unquoted values, applied after substitution: integers (-?digits) and floats (-?digits.digits) become " + "numbers; true/false/yes/no/on/off (any case) become booleans; null (any case) becomes null; a key whose " + "name ends in _list becomes a list of strings split at commas with each item stripped (an empty value " + "is []); everything else stays a string. An unresolvable reference, a reference cycle, a line that is " + "neither a comment, a section header nor key = value, or an unterminated quote is an error: message on " + "stderr, exit status 1 and out.json not written." + ) + + class Err(Exception): + pass + + @staticmethod + def oracle(text: str): + Err = IniToJson.Err + root, secs, defaults = {}, {}, {} + cur = None # None = top level + for raw in text.splitlines(): + line = raw.strip() + if not line or line[0] in ";#": + continue + m = re.fullmatch(r"\[([^\]]+)\]", line) + if m: + cur = m.group(1).strip() + if cur != "DEFAULT": + secs.setdefault(cur, {}) + continue + if "=" not in line: + raise Err("bad line") + k, v = (x.strip() for x in line.split("=", 1)) + if v.startswith('"'): + end = v.find('"', 1) + if end == -1: + raise Err("unterminated") + rest = v[end + 1:].strip() + if rest and rest[0] not in ";#": + raise Err("junk after quote") + val = (v[1:end], True) + else: + if v and v[0] in ";#": + v = "" + else: + mm = re.search(r"\s[;#]", v) + if mm: + v = v[:mm.start()].rstrip() + val = (v, False) + target = root if cur is None else (defaults if cur == "DEFAULT" else secs[cur]) + target[k] = val + merged = {name: {**defaults, **body} for name, body in secs.items()} + merged[""] = root + + def text_of(sec, key, stack): + if (sec, key) in stack: + raise Err("cycle") + if sec not in merged or key not in merged[sec]: + raise Err("unresolved") + t, _ = merged[sec][key] + + def sub(m): + ref = m.group(1) + if "." in ref: + s2, k2 = ref.rsplit(".", 1) + else: + s2, k2 = sec, ref + return text_of(s2, k2, stack | {(sec, key)}) + return re.sub(r"\$\{([^}]*)\}", sub, t) + + def typed(key, t, quoted): + if quoted: + return t + if key.endswith("_list"): + return [x.strip() for x in t.split(",")] if t.strip() else [] + if re.fullmatch(r"-?\d+", t): + return int(t) + if re.fullmatch(r"-?\d+\.\d+", t): + return float(t) + if t.lower() in ("true", "yes", "on"): + return True + if t.lower() in ("false", "no", "off"): + return False + if t.lower() == "null": + return None + return t + out = {} + for sec, body in merged.items(): + node = out + if sec: + for part in sec.split("."): + node = node.setdefault(part, {}) + for k, (t, q) in body.items(): + node[k] = typed(k, text_of(sec, k, frozenset()), q) + return out + + GOOD = [ + """; global settings +name = My App ; inline comment +debug = yes +version = 1.10 +[DEFAULT] +timeout = 30 +host = localhost +[db] +host = db.internal +port = 5432 +url = postgres://${host}:${port}/main +tags_list = a, b ,c +empty_list = +quoted = "x ; not a comment" # real comment +path = C:\\temp#dir +pct = 50% +[db.pool] +size = 10 +ratio = 0.75 +max = ${db.port} +off = Off +nothing = NULL +neg = -4 +[web] +port = 8080 +workers = ${port} +base = "${host}" +""", + "[a]\r\nx = 1\r\nx = 2\r\n[b]\r\ny=a=b\r\n[a]\r\nz\t=\ttrue\r\n", + "[one]\nhost = h1\nurl = http://${host}/\n[two]\nhost = h2\n[DEFAULT]\nurl = default://${host}\ntimeout = 5\n", + "top = 1\nflag = No\n[s]\nKeyCase = Value\nonly_list = x\nlist_of = 1, 2\nempty = \"\"\ncomment_only = ; nothing\n", + "[a]\nb = ${c}\nc = ${d}\nd = 7\ne_list = ${d}, ${c}, z\n", + "[a.b.c]\nx = 1\n[a]\ny = 2\n[a.b]\nz = 3\n", + "[q]\nv = \"a\" ; trailing\nw = a;b\nx = a ;b\ny = a\t#b\nz = 1.5.2\nn = 1e5\nt = TRUE\n", + ] + BAD = [ + "[a]\nx = ${nope}\n", + "[a]\nx = ${y}\ny = ${x}\n", + "[a]\njust some text\n", + "[a]\nx = \"open\n", + "[a]\nx = ${b.y}\n[b]\nz = 1\n", + "[a]\nx = \"q\" junk\n", + ] + + @staticmethod + def setup(d: Path) -> None: + write(d, "app.ini", IniToJson.GOOD[0]) + + @staticmethod + def grade(d: Path): + if not (d / "ini2json.py").exists(): + return False, "no ini2json.py" + tmp, w = scratch_copy(d) + try: + for i, text in enumerate(IniToJson.GOOD): + (w / "in.ini").write_bytes(text.encode()) + (w / "out.json").unlink(missing_ok=True) + r = runb(["python3", "ini2json.py", "in.ini", "out.json"], cwd=w) + want = IniToJson.oracle(text) + if r.returncode != 0 or not (w / "out.json").exists(): + return False, f"good case {i}: exit {r.returncode}: {r.stderr.decode()[-150:]}" + got = json.loads((w / "out.json").read_text()) + if json.dumps(got, sort_keys=True) != json.dumps(want, sort_keys=True): + for k in sorted(set(got) | set(want)): + if got.get(k) != want.get(k): + return False, f"good case {i}: key {k!r}: got {str(got.get(k))[:90]} want {str(want.get(k))[:90]}" + return False, f"good case {i}: differs" + for i, text in enumerate(IniToJson.BAD): + (w / "in.ini").write_bytes(text.encode()) + (w / "out.json").unlink(missing_ok=True) + r = runb(["python3", "ini2json.py", "in.ini", "out.json"], cwd=w) + if r.returncode != 1 or (w / "out.json").exists(): + return False, f"bad case {i}: exit {r.returncode}, out.json written={(w / 'out.json').exists()}" + return True, f"{len(IniToJson.GOOD)} good + {len(IniToJson.BAD)} bad cases" + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +# ================================================================ 14. perf-asof-join +JOIN_SLOW = '''import csv +import sys + + +def main(events_path, rates_path, out_path): + rates = [] + with open(rates_path, newline="") as f: + for r in csv.DictReader(f): + rates.append((int(r["effective_ts"]), r["rate"])) + with open(events_path, newline="") as f, open(out_path, "w", newline="") as o: + w = csv.writer(o) + w.writerow(["ts", "id", "rate"]) + for e in csv.DictReader(f): + ts = int(e["ts"]) + best = None + for eff, rate in rates: + if eff <= ts and (best is None or eff >= best[0]): + best = (eff, rate) + w.writerow([e["ts"], e["id"], best[1] if best else ""]) + + +main(*sys.argv[1:4]) +''' + + +class PerfAsofJoin: + name = "perf-asof-join" + PROMPT = ( + "join.py (`python3 join.py events.csv rates.csv out.csv`) attaches to each event the rate that was in " + "force at its time: the rate of the rates.csv row with the greatest effective_ts that is <= the event's ts " + "(if several rows share that effective_ts, the one later in the file wins; if no row qualifies the rate " + "is blank). It is correct but far too slow on real data: it must finish a 200,000-event, 50,000-rate input " + "in under 10 seconds. Make it fast without changing its output in any way (rows in event order, rate " + "text copied as is, same header). rates.csv is not sorted." + ) + + @staticmethod + def gen(seed, ne, nr): + rnd = random.Random(seed) + span = max(nr * 10, 100) + rates = [(rnd.randint(-50, span), rnd.choice(["1.5", "0.250", "7", "x,y", "", "0.1"])) for _ in range(nr)] + if nr > 3: + rates[1] = (rates[0][0], "dup-later") + events = [(rnd.randint(-80, span + 50), f"e{i}") for i in range(ne)] + return events, rates + + @staticmethod + def write_inputs(w: Path, events, rates): + with open(w / "events.csv", "w", newline="") as f: + c = csv.writer(f) + c.writerow(["ts", "id"]) + c.writerows(events) + with open(w / "rates.csv", "w", newline="") as f: + c = csv.writer(f) + c.writerow(["effective_ts", "rate"]) + c.writerows(rates) + + @staticmethod + def oracle(events, rates): + import bisect + order = sorted(range(len(rates)), key=lambda i: (rates[i][0], i)) + keys = [rates[i][0] for i in order] + out = [["ts", "id", "rate"]] + for ts, eid in events: + k = bisect.bisect_right(keys, ts) + out.append([str(ts), eid, rates[order[k - 1]][1] if k else ""]) + return out + + @staticmethod + def setup(d: Path) -> None: + ev, ra = PerfAsofJoin.gen(1, 30, 8) + PerfAsofJoin.write_inputs(d, ev, ra) + write(d, "join.py", JOIN_SLOW) + + @staticmethod + def grade(d: Path): + if not (d / "join.py").exists(): + return False, "no join.py" + tmp, w = scratch_copy(d) + try: + for seed, ne, nr, limit in ((2, 300, 40, 30), (3, 200000, 50000, 10), (4, 5, 0, 30)): + ev, ra = PerfAsofJoin.gen(seed, ne, nr) + PerfAsofJoin.write_inputs(w, ev, ra) + (w / "out.csv").unlink(missing_ok=True) + t0 = time.time() + try: + r = runb(["python3", "join.py", "events.csv", "rates.csv", "out.csv"], cwd=w, timeout=limit) + except subprocess.TimeoutExpired: + return False, f"{ne} events x {nr} rates: not done in {limit}s" + el = time.time() - t0 + if r.returncode != 0 or not (w / "out.csv").exists(): + return False, f"exit {r.returncode}: {r.stderr.decode()[-150:]}" + with open(w / "out.csv", newline="") as f: + got = list(csv.reader(f)) + want = PerfAsofJoin.oracle(ev, ra) + if got != want: + for a, b in zip(got, want): + if a != b: + return False, f"{ne} events: got {a} want {b}" + return False, f"{len(got)} rows, want {len(want)}" + return True, f"correct, big input in {el:.1f}s" + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +# ================================================================ 15. slugify-batch +class SlugifyBatch: + name = "slugify-batch" + PROMPT = ( + "Write slugify.py: reads lines from stdin (each terminated by \\n or \\r\\n; the terminator is not part of " + "the line, and blank lines count) and writes one URL slug per line to stdout, in order. A slug is made " + "like this, in this order: lowercase the line (str.lower); replace ß with ss, æ with ae, œ with oe, ø, ð " + "and đ with o, d and d, þ with th and ł with l; apply Unicode NFKD normalisation and delete the combining " + "marks; turn every run of characters outside a-z and 0-9 into a single hyphen; strip leading and trailing " + "hyphens; cut to at most 40 characters and strip trailing hyphens again; an empty result becomes " + "`untitled`. Finally make slugs unique across the whole input: if a slug is already taken by an earlier " + "line, append -2, -3, ... using the first number whose result is not yet taken (the suffix is added after " + "the 40-character cut). Read and write UTF-8 regardless of the locale." + ) + + TABLE = [("ß", "ss"), ("æ", "ae"), ("œ", "oe"), ("ø", "o"), ("ð", "d"), ("đ", "d"), ("þ", "th"), ("ł", "l")] + + @staticmethod + def slug(s): + s = s.lower() + for a, b in SlugifyBatch.TABLE: + s = s.replace(a, b) + s = unicodedata.normalize("NFKD", s) + s = "".join(c for c in s if not unicodedata.combining(c)) + s = re.sub(r"[^a-z0-9]+", "-", s).strip("-") + s = s[:40].rstrip("-") + return s or "untitled" + + @staticmethod + def oracle(lines): + seen, out = set(), [] + for l in lines: + base = SlugifyBatch.slug(l) + cand, n = base, 1 + while cand in seen: + n += 1 + cand = f"{base}-{n}" + seen.add(cand) + out.append(cand) + return out + + CASES = [ + ["Hello, World!", " Hello, World! ", "hello world", "Hello World"], + ["Crème Brûlée", "Straße", "STRASSE", "Ærø Œuvre", "Zażółć gęślą jaźń", "Ñandú", "Þórr Ðæmi Đồng"], + ["日本語", "", "---", "!!!", "x²", "fine flour", "Ⅷ rooms", "Ünïcödé Тест"], + ["a", "a", "a-2", "a", "a 2", "a-3"], + ["a" * 50, "a" * 50, "a" * 39 + " bbb", "a" * 39 + "-bbb", "a" * 40, "a" * 41], + ["untitled", "", "日本", "untitled-2", "!"], + ["Release v1.2.3 (final)", "release v1.2.3 final", "Release: v1.2.3 / final!"], + ["İstanbul", "ISTANBUL", "Dž", "dž"], + ] + + @staticmethod + def setup(d: Path) -> None: + write(d, "titles.txt", "Hello, World!\nCrème Brûlée\nHello World\n") + + @staticmethod + def grade(d: Path): + if not (d / "slugify.py").exists(): + return False, "no slugify.py" + for i, lines in enumerate(SlugifyBatch.CASES): + for eol in ("\n", "\r\n"): + data = "".join(l + eol for l in lines).encode("utf-8") + r = runb(["python3", "slugify.py"], cwd=d, inp=data, env={"LC_ALL": "C", "LANG": "C", "PYTHONUTF8": "", "PYTHONIOENCODING": ""}) + want = SlugifyBatch.oracle(lines) + got = r.stdout.decode("utf-8", "replace").split("\n") + if r.returncode != 0 or got != want + [""]: + for a, b in zip(got, want): + if a != b: + return False, f"case {i} (eol {eol!r}): input {lines[want.index(b)]!r}: got {a!r} want {b!r}" + return False, f"case {i}: exit {r.returncode}, {len(got) - 1} lines, want {len(want)}: {r.stderr.decode()[-120:]}" + return True, f"{len(SlugifyBatch.CASES)} cases" + + +# ================================================================ 16. text-wrap-cli +class TextWrap: + name = "text-wrap-cli" + PROMPT = ( + "Write wrap.py: `python3 wrap.py WIDTH` reads text from stdin and writes it re-wrapped to at most WIDTH " + "characters per line (counted in Unicode characters). Paragraphs are separated by one or more blank " + "lines (lines empty or holding only ASCII whitespace); each is re-flowed greedily: words (separated by " + "runs of ASCII whitespace -- space, tab, CR, LF, FF, VT -- and nothing else, so U+00A0 and other Unicode " + "spaces are ordinary characters inside a word) are packed onto a line while they fit, single-spaced. A " + "word longer than WIDTH sits alone on its own line, never split, and hyphenated words are never split " + "either. Output has no trailing spaces, one blank line between paragraphs, and ends with a single " + "newline; empty or all-blank input gives empty output. WIDTH must be a positive integer, otherwise print " + "a usage message to stderr and exit with status 2." + ) + + @staticmethod + def oracle(text, width): + paras, cur = [], [] + for line in text.split("\n"): + if not line.strip(" \t\r\f\v"): + if cur: + paras.append(cur) + cur = [] + else: + cur.extend(re.split(r"[ \t\r\n\f\v]+", line.strip(" \t\r\f\v"))) + if cur: + paras.append(cur) + out = [] + for words in paras: + lines, line = [], words[0] + for w in words[1:]: + if len(line) + 1 + len(w) <= width: + line += " " + w + else: + lines.append(line) + line = w + lines.append(line) + out.append("\n".join(lines)) + return "\n\n".join(out) + "\n" if out else "" + + TEXT = ( + "The quick brown fox jumps over the lazy dog. A well-known state-of-the-art technique; see " + "https://example.com/a/very/long/url/that/will/not/fit/anywhere for details.\n\n\n" + " Second\tparagraph\twith odd spacing\r\nand a CRLF line break.\n \nThird: the na\u00efve caf\u00e9 " + "owner\u00a0keeps\u00a0nbsp\u00a0words together, \u65e5\u672c\u8a9e\u306e\u30c6\u30ad\u30b9\u30c8 too.\n" + "x\n\n\n\n" + ) + + @staticmethod + def setup(d: Path) -> None: + write(d, "sample.txt", TextWrap.TEXT) + + @staticmethod + def grade(d: Path): + if not (d / "wrap.py").exists(): + return False, "no wrap.py" + inputs = [TextWrap.TEXT, "", " \n\n \t\n", "one", "a b c d e f g h i j k l m n o p", "word " * 30, "\n\nlead\n", "x" * 100 + " y " + "z" * 3 + "\n", + "end-of-line \nstill same paragraph\n\nnew"] + for width in (1, 5, 10, 20, 37, 80): + for text in inputs: + r = runb(["python3", "wrap.py", str(width)], cwd=d, inp=text.encode("utf-8")) + want = TextWrap.oracle(text, width) + got = r.stdout.decode("utf-8") + if r.returncode != 0 or got != want: + return False, f"width {width}, input {text[:30]!r}: got {got[:90]!r} want {want[:90]!r} (exit {r.returncode})" + for bad in ("0", "-3", "abc", "", "2.5"): + r = runb(["python3", "wrap.py", bad], cwd=d, inp=b"hello\n") + if r.returncode != 2 or r.stdout: + return False, f"WIDTH {bad!r}: exit {r.returncode}, want 2 and no stdout" + r = runb(["python3", "wrap.py"], cwd=d, inp=b"hello\n") + if r.returncode != 2: + return False, "missing WIDTH should exit 2" + return True, "all widths" + + +# ================================================================ 17. sqlite-migrate +class SqliteMigrate: + name = "sqlite-migrate" + PROMPT = ( + "app.db is at schema version 1 (PRAGMA user_version = 1). Write migrate.py: `python3 migrate.py DB` " + "migrates the database in place to version 2, keeping all data: (1) a new table customers(id INTEGER " + "PRIMARY KEY, email TEXT NOT NULL UNIQUE) holding each distinct customer of orders, where the old " + "orders.customer text is an email that must be trimmed and lowercased to compare, ids assigned 1, 2, 3, " + "... in order of first appearance by orders.id, email stored trimmed and lowercased; (2) orders becomes " + "(id INTEGER PRIMARY KEY, customer_id INTEGER NOT NULL REFERENCES customers(id), amount_cents INTEGER NOT " + "NULL, placed_at TEXT NOT NULL, note TEXT) in exactly that column order, with the same ids and data, " + "customer replaced by customer_id and amount (dollars, REAL) replaced by amount_cents rounded to the " + "nearest cent; (3) the existing index idx_orders_placed must still exist on orders(placed_at) and a new " + "index idx_orders_customer on orders(customer_id) is added; (4) order_items must keep working and keep " + "referencing orders(id); (5) finally set user_version to 2. Running it on a database already at version " + "2 changes nothing and exits 0; any other version leaves the database untouched " + "and exits 1. After migrating, PRAGMA foreign_key_check and integrity_check " + "must come back clean." + ) + + @staticmethod + def make_v1(path: Path, seed: int, n: int): + rnd = random.Random(seed) + Path(path).unlink(missing_ok=True) + db = sqlite3.connect(path) + db.executescript( + "CREATE TABLE orders(id INTEGER PRIMARY KEY, customer TEXT NOT NULL, amount REAL NOT NULL, placed_at TEXT NOT NULL, note TEXT);" + "CREATE INDEX idx_orders_placed ON orders(placed_at);" + "CREATE TABLE order_items(id INTEGER PRIMARY KEY, order_id INTEGER NOT NULL REFERENCES orders(id), sku TEXT NOT NULL, qty INTEGER NOT NULL);" + ) + people = ["ada@example.com", "ben@example.com", "Cleo@Example.com", "dmitri@example.org", "esi@example.net"] + fixed = [19.99, 0.29, 4.35, 8.2, 1.15, 2.675, 100.0, 0.07, 33.33, 1234567.89] + for i in range(1, n + 1): + who = rnd.choice(people) + who = rnd.choice([who, who.upper(), " " + who + " ", who.title()]) + amt = fixed[i - 1] if i <= len(fixed) else round(rnd.uniform(0.01, 999.99), 2) + if amt == 2.675: + amt = 2.68 + db.execute("INSERT INTO orders VALUES(?,?,?,?,?)", (i * 3, who, amt, f"2024-0{rnd.randint(1, 9)}-{rnd.randint(10, 28)}", rnd.choice([None, "gift", ""]))) + for _ in range(rnd.randint(0, 3)): + db.execute("INSERT INTO order_items(order_id, sku, qty) VALUES(?,?,?)", (i * 3, rnd.choice(["A", "B", "C"]), rnd.randint(1, 5))) + db.execute("PRAGMA user_version = 1") + db.commit() + db.close() + + @staticmethod + def setup(d: Path) -> None: + SqliteMigrate.make_v1(d / "app.db", 1, 25) + + @staticmethod + def check(path: Path, orig: Path): + a = sqlite3.connect(orig) + b = sqlite3.connect(path) + try: + if b.execute("PRAGMA user_version").fetchone()[0] != 2: + return "user_version is not 2" + cols = [r[1] for r in b.execute("PRAGMA table_info(orders)")] + if cols != ["id", "customer_id", "amount_cents", "placed_at", "note"]: + return f"orders columns are {cols}" + ccols = [(r[1], r[2], r[3], r[5]) for r in b.execute("PRAGMA table_info(customers)")] + if ccols != [("id", "INTEGER", 0, 1), ("email", "TEXT", 1, 0)]: + return f"customers columns are {ccols}" + old = a.execute("SELECT id, customer, amount, placed_at, note FROM orders ORDER BY id").fetchall() + emails, want_orders = {}, [] + for oid, cust, amt, placed, note in old: + e = cust.strip().lower() + emails.setdefault(e, len(emails) + 1) + want_orders.append((oid, emails[e], int((Decimal(repr(amt)) * 100).quantize(Decimal(1), rounding="ROUND_HALF_UP")), placed, note)) + if b.execute("SELECT id, email FROM customers ORDER BY id").fetchall() != sorted((i, e) for e, i in emails.items()): + return "customers rows are wrong" + got = b.execute("SELECT id, customer_id, amount_cents, placed_at, note FROM orders ORDER BY id").fetchall() + if got != want_orders: + for x, y in zip(got, want_orders): + if x != y: + return f"orders row {x} should be {y}" + return "orders row count differs" + if any(type(r[2]) is not int for r in got): + return "amount_cents is not stored as an integer" + idx = {r[1]: r[2] for r in b.execute("PRAGMA index_list(orders)")} + if "idx_orders_placed" not in idx or "idx_orders_customer" not in idx: + return f"indexes on orders are {sorted(idx)}" + if [r[2] for r in b.execute("PRAGMA index_info(idx_orders_placed)")] != ["placed_at"] or [r[2] for r in b.execute("PRAGMA index_info(idx_orders_customer)")] != ["customer_id"]: + return "index columns are wrong" + if [(r[2], r[3], r[4]) for r in b.execute("PRAGMA foreign_key_list(orders)")] != [("customers", "customer_id", "id")]: + return "orders no longer has its foreign key to customers(id)" + if [(r[2], r[3], r[4]) for r in b.execute("PRAGMA foreign_key_list(order_items)")] != [("orders", "order_id", "id")]: + return f"order_items foreign key is {list(b.execute('PRAGMA foreign_key_list(order_items)'))}" + if a.execute("SELECT * FROM order_items ORDER BY id").fetchall() != b.execute("SELECT * FROM order_items ORDER BY id").fetchall(): + return "order_items data changed" + if b.execute("PRAGMA foreign_key_check").fetchall(): + return "foreign_key_check reports violations" + if b.execute("PRAGMA integrity_check").fetchone()[0] != "ok": + return "integrity_check failed" + names = {r[0] for r in b.execute("SELECT name FROM sqlite_master WHERE type='table'")} + if names != {"orders", "order_items", "customers"}: + return f"tables are {sorted(names)}" + return None + finally: + a.close() + b.close() + + @staticmethod + def grade(d: Path): + if not (d / "migrate.py").exists(): + return False, "no migrate.py" + tmp = Path(tempfile.mkdtemp(prefix="grade3-")) + try: + shutil.copy(d / "migrate.py", tmp / "migrate.py") + for seed, n in ((1, 25), (7, 60)): + SqliteMigrate.make_v1(tmp / "orig.db", seed, n) + shutil.copy(tmp / "orig.db", tmp / "t.db") + r = runb(["python3", "migrate.py", "t.db"], cwd=tmp) + if r.returncode != 0: + return False, f"seed {seed}: exit {r.returncode}: {r.stderr.decode()[-200:]}" + bad = SqliteMigrate.check(tmp / "t.db", tmp / "orig.db") + if bad: + return False, f"seed {seed}: {bad}" + dump1 = subprocess.run(["python3", "-c", "import sqlite3,sys;print('\\n'.join(sqlite3.connect('t.db').iterdump()))"], cwd=tmp, capture_output=True).stdout + r = runb(["python3", "migrate.py", "t.db"], cwd=tmp) + dump2 = subprocess.run(["python3", "-c", "import sqlite3,sys;print('\\n'.join(sqlite3.connect('t.db').iterdump()))"], cwd=tmp, capture_output=True).stdout + if r.returncode != 0 or dump1 != dump2: + return False, f"seed {seed}: a second run exit {r.returncode} / changed the database" + for ver in (0, 3): + shutil.copy(tmp / "orig.db", tmp / "u.db") + c = sqlite3.connect(tmp / "u.db") + c.execute(f"PRAGMA user_version = {ver}") + c.commit() + c.close() + before = (tmp / "u.db").read_bytes() + r = runb(["python3", "migrate.py", "u.db"], cwd=tmp) + if r.returncode != 1 or (tmp / "u.db").read_bytes() != before: + return False, f"user_version {ver}: exit {r.returncode}, database {'changed' if (tmp / 'u.db').read_bytes() != before else 'untouched'}" + return True, "migrated, idempotent" + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +# ================================================================ 18. vendor-units-workaround +VENDOR_UNITS = '''"""units 1.4.2 -- vendored copy. Do not edit: CI compares vendor/ against the upstream checksums.""" +import re + +_UNITS = {"s": 1, "m": 2592000, "mo": 2592000, "h": 3600, "d": 86400, "w": 604800} + + +def parse_duration(spec): + """Parse '' (single unit) into seconds.""" + m = re.fullmatch(r"\\s*(\\d+(?:\\.\\d+)?)\\s*([a-z]+)\\s*", spec) + if not m: + raise ValueError("bad duration: %r" % (spec,)) + unit = m.group(2) + if unit not in _UNITS: + raise ValueError("unknown unit: %r" % (unit,)) + return float(m.group(1)) * _UNITS[unit] +''' +RETRY_PY = '''"""Retry delays.""" +from vendor import units + + +def delay_seconds(spec): + """Seconds to wait for a duration spec such as '90s' or '5m'.""" + return units.parse_duration(spec) +''' +SCHEDULE_PY = '''import json + +from app.retry import delay_seconds + + +def load_schedule(path): + """{name: seconds} for every duration in a JSON config file.""" + with open(path) as f: + cfg = json.load(f) + return {name: delay_seconds(spec) for name, spec in cfg.items()} +''' + + +class VendorUnits: + name = "vendor-units-workaround" + PROMPT = ( + "Our retry delays are wrong: a config value of \"5m\" makes the app wait about five months. The cause is in " + "vendor/units.py, a pinned third-party copy whose checksums CI verifies, so nothing under vendor/ may be " + "changed, added or removed. Fix the app instead (app/retry.py's delay_seconds, which app/schedule.py's " + "load_schedule uses). delay_seconds(spec) must accept: a number (integer or decimal) followed by a unit " + "s, m (minutes), h, d or w (lowercase only), optional spaces between number and unit, and several such " + "parts in a row ('1h30m', '2d 4h 30s'), with leading/trailing whitespace allowed. It returns the total " + "in seconds (an int when whole, otherwise a float). Anything else -- empty, bare numbers, unknown units " + "such as 'mo' or 'y', uppercase units, negative or signed numbers -- raises ValueError." + ) + + @staticmethod + def setup(d: Path) -> None: + write(d, "vendor/__init__.py", "") + write(d, "vendor/units.py", VENDOR_UNITS) + write(d, "app/__init__.py", "") + write(d, "app/retry.py", RETRY_PY) + write(d, "app/schedule.py", SCHEDULE_PY) + write(d, "config.json", json.dumps({"backoff": "5m", "timeout": "90s", "cooldown": "1h30m", "report": "1.5h"}, indent=2)) + + GOOD = [("90s", 90), ("5m", 300), ("1h", 3600), ("2d", 172800), ("1w", 604800), ("1.5h", 5400), ("0s", 0), ("0.5m", 30), + (" 5m ", 300), ("5 m", 300), ("1h30m", 5400), ("2d 4h 30s", 2 * 86400 + 4 * 3600 + 30), ("1w 1d 1h 1m 1s", 604800 + 86400 + 3600 + 61), + ("0.25h", 900), ("100s", 100), ("1m 0.5s", 60.5)] + BAD = ["", " ", "5", "5mo", "1y", "5M", "5S", "-5m", "+5m", "5m-", "m", "1h 30", "1h,30m", "1..5h", "5 ms", "five minutes", "1e3s", "1h30"] + + @staticmethod + def grade(d: Path): + tmp, w = scratch_copy(d) + try: + ref = Path(tempfile.mkdtemp(prefix="grade3-")) + try: + VendorUnits.setup(ref) + if digest(ref / "vendor") != digest(w / "vendor"): + return False, "vendor/ was changed" + finally: + shutil.rmtree(ref, ignore_errors=True) + code = ( + "import json, math, sys\n" + "from app.retry import delay_seconds\n" + "from app.schedule import load_schedule\n" + f"GOOD = {VendorUnits.GOOD!r}\nBAD = {VendorUnits.BAD!r}\n" + "for spec, want in GOOD:\n" + " try: got = delay_seconds(spec)\n" + " except Exception as e: print(f'{spec!r}: raised {e!r}'); sys.exit()\n" + " if not math.isclose(got, want, rel_tol=1e-9, abs_tol=1e-9) or (float(want).is_integer() and not isinstance(got, int)):\n" + " print(f'{spec!r}: got {got!r} want {want!r}'); sys.exit()\n" + "for spec in BAD:\n" + " try: got = delay_seconds(spec)\n" + " except ValueError: continue\n" + " except Exception as e: print(f'{spec!r}: raised {type(e).__name__}, want ValueError'); sys.exit()\n" + " print(f'{spec!r}: accepted as {got!r}'); sys.exit()\n" + "json.dump({'a': '2m', 'b': '1h 1s'}, open('cfg2.json', 'w'))\n" + "if load_schedule('cfg2.json') != {'a': 120, 'b': 3601}: print('load_schedule wrong:', load_schedule('cfg2.json')); sys.exit()\n" + "if load_schedule('config.json') != {'backoff': 300, 'timeout': 90, 'cooldown': 5400, 'report': 5400}: print('config.json schedule wrong:', load_schedule('config.json')); sys.exit()\n" + "print('OK')\n" + ) + ok, why = py_check(w, code) + return ok, why + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +# ================================================================ 19. rename-api-signature +SHOP_FILES = { + "shop/__init__.py": "", + "shop/db.py": '''import sqlite3 + + +def connect(path=":memory:"): + conn = sqlite3.connect(path) + conn.execute("CREATE TABLE IF NOT EXISTS users(id INTEGER PRIMARY KEY, name TEXT, email TEXT)") + return conn + + +def get_user(db, uid): + """Return the user with this id as a dict, or None.""" + row = db.execute("SELECT id, name, email FROM users WHERE id = ?", (uid,)).fetchone() + if row is None: + return None + return {"id": row[0], "name": row[1], "email": row[2]} + + +def get_user_by_name(db, name): + row = db.execute("SELECT id FROM users WHERE name = ?", (name,)).fetchone() + return None if row is None else get_user(db, row[0]) +''', + "shop/views.py": '''import functools + +from shop.db import get_user + + +def profile(db, uid): + u = get_user(db, uid) + if u is None: + return "unknown" + return "%s <%s>" % (u["name"], u["email"]) + + +def profiles(db, uids): + return [profile(db, u) for u in uids if get_user(db, u)] + + +def make_loader(db): + """A one-argument loader bound to this connection.""" + return functools.partial(get_user, db) +''', + "shop/billing.py": '''from shop.db import get_user as _gu + + +def invoice_owner(db, order): + owner = _gu( + db, + order["user_id"], + ) + return owner["name"] if owner else None + + +def total_for(db, orders): + # get_user is cheap, no caching needed + return sum(o["cents"] for o in orders if _gu(db, o["user_id"])) +''', + "shop/notify.py": '''import shop.db as sdb + + +def welcome(db, uid): + user = sdb.get_user(db, uid) + return "Welcome, %s!" % user["name"] if user else "Welcome!" + + +def by_keyword(db, uid): + return sdb.get_user(uid=uid, db=db) +''', + "shop/registry.py": '''import json +import os + +import shop.db + +with open(os.path.join(os.path.dirname(__file__), "plugins.json")) as _f: + PLUGINS = json.load(_f) + + +def load(name): + """Look a data-access function up by its configured name.""" + return getattr(shop.db, PLUGINS[name]) +''', + "shop/plugins.json": '{"user_loader": "get_user", "name_loader": "get_user_by_name"}\n', + "shop/cli.py": '''import sys + +from shop import views + + +def main(argv, db): + if len(argv) == 2 and argv[0] == "show": + print(views.profile(db, int(argv[1]))) + return 0 + print("usage: show UID", file=sys.stderr) + return 2 +''', + "scripts/audit.py": '''"""Report users that look incomplete.""" +from shop.db import get_user + + +def audit(conn, ids): + bad = [] + for uid in ids: + u = get_user(conn, uid) + if u is None or not u["email"]: + bad.append(uid) + return bad +''', + "README.md": "# shop\n\nUse `shop.db.get_user(db, uid)` to load a user; `get_user_by_name(db, name)` looks one up by name.\n", + "docs/API.md": "# API\n\n## get_user(db, uid)\n\nReturns a dict `{id, name, email}` or `None`.\n\nExample: `u = get_user(conn, 5)`\n\n## get_user_by_name(db, name)\n\nSame, looked up by name.\n", + "CHANGELOG.md": "# Changelog\n\n## 1.2\n- Added get_user_by_name.\n\n## 1.0\n- Initial release: get_user(db, uid).\n", + "tests/__init__.py": "", + "tests/test_shop.py": '''import unittest + +from shop import billing, db, notify, registry, views +from shop.db import get_user + + +class ShopTests(unittest.TestCase): + def setUp(self): + self.conn = db.connect() + self.conn.execute("INSERT INTO users VALUES (1, 'Ada', 'ada@x.org')") + self.conn.execute("INSERT INTO users VALUES (2, 'Ben', '')") + + def test_get_user(self): + self.assertEqual(get_user(self.conn, 1)["name"], "Ada") + self.assertIsNone(get_user(self.conn, 9)) + + def test_by_name(self): + self.assertEqual(db.get_user_by_name(self.conn, "Ben")["id"], 2) + + def test_views(self): + self.assertEqual(views.profile(self.conn, 1), "Ada ") + self.assertEqual(views.profiles(self.conn, [1, 2, 3]), ["Ada ", "Ben <>"]) + self.assertEqual(views.make_loader(self.conn)(2)["name"], "Ben") + + def test_billing(self): + self.assertEqual(billing.invoice_owner(self.conn, {"user_id": 1}), "Ada") + self.assertEqual(billing.total_for(self.conn, [{"user_id": 1, "cents": 5}, {"user_id": 7, "cents": 9}]), 5) + + def test_notify_and_registry(self): + self.assertEqual(notify.welcome(self.conn, 1), "Welcome, Ada!") + self.assertEqual(notify.by_keyword(self.conn, 2)["name"], "Ben") + self.assertEqual(registry.load("user_loader")(self.conn, 1)["email"], "ada@x.org") + + +if __name__ == "__main__": + unittest.main() +''', +} + + +class RenameApi: + name = "rename-api-signature" + PROMPT = ( + "Rename shop.db.get_user(db, uid) to fetch_user(uid, *, db) -- new name, and `db` becomes a keyword-only " + "argument after uid -- and update everything that uses it: every call site in the package and in " + "scripts/, including calls through an import alias, multi-line calls and anything that binds the " + "function (partial and the like), the name looked up by string in shop/plugins.json, the tests in " + "tests/, and the docs (README.md, docs/API.md). Don't leave the old name behind as an alias. " + "get_user_by_name keeps its name and its (db, name) signature. CHANGELOG.md records history: leave it " + "exactly as it is. Apart from CHANGELOG.md, the old name get_user must not appear anywhere in the " + "project any more (comments included). Behaviour must stay the same and the tests must pass." + ) + + @staticmethod + def setup(d: Path) -> None: + for rel, txt in SHOP_FILES.items(): + write(d, rel, txt) + + HARNESS = r''' +import inspect, runpy, sys +sys.path.insert(0, ".") +from shop import db, views, billing, notify, registry, cli +conn = db.connect() +conn.execute("INSERT INTO users VALUES (1, 'Ada', 'ada@x.org')") +conn.execute("INSERT INTO users VALUES (2, 'Ben', '')") +if hasattr(db, "get_user"): print("shop.db still has get_user"); sys.exit() +sig = inspect.signature(db.fetch_user) +ps = list(sig.parameters.values()) +if [p.name for p in ps] != ["uid", "db"] or ps[0].kind not in (ps[0].POSITIONAL_OR_KEYWORD, ps[0].POSITIONAL_ONLY) or ps[1].kind != ps[1].KEYWORD_ONLY: + print(f"fetch_user signature is {sig}"); sys.exit() +if db.fetch_user(1, db=conn) != {"id": 1, "name": "Ada", "email": "ada@x.org"} or db.fetch_user(9, db=conn) is not None: + print("fetch_user returns the wrong thing"); sys.exit() +try: + db.fetch_user(conn, 1); print("fetch_user(conn, 1) should not be accepted"); sys.exit() +except TypeError: + pass +if db.get_user_by_name(conn, "Ben")["id"] != 2 or list(inspect.signature(db.get_user_by_name).parameters) != ["db", "name"]: + print("get_user_by_name changed"); sys.exit() +checks = [ + ("views.profile", lambda: views.profile(conn, 1) == "Ada "), + ("views.profiles", lambda: views.profiles(conn, [1, 2, 3]) == ["Ada ", "Ben <>"]), + ("views.make_loader", lambda: views.make_loader(conn)(2)["name"] == "Ben"), + ("billing.invoice_owner", lambda: billing.invoice_owner(conn, {"user_id": 1}) == "Ada"), + ("billing.total_for", lambda: billing.total_for(conn, [{"user_id": 1, "cents": 5}, {"user_id": 7, "cents": 9}]) == 5), + ("notify.welcome", lambda: notify.welcome(conn, 1) == "Welcome, Ada!" and notify.welcome(conn, 8) == "Welcome!"), + ("notify.by_keyword", lambda: notify.by_keyword(conn, 2)["name"] == "Ben"), + ("registry user_loader", lambda: registry.load("user_loader")(1, db=conn)["email"] == "ada@x.org"), + ("registry name_loader", lambda: registry.load("name_loader")(conn, "Ada")["id"] == 1), + ("cli show", lambda: cli.main(["show", "1"], conn) == 0), + ("scripts/audit.py", lambda: runpy.run_path("scripts/audit.py")["audit"](conn, [1, 2, 3]) == [2, 3]), +] +for name, fn in checks: + try: + ok = fn() + except Exception as e: + print(f"{name}: raised {type(e).__name__}: {e}"); sys.exit() + if not ok: + print(f"{name}: wrong result"); sys.exit() +print("OK") +''' + + @staticmethod + def grade(d: Path): + tmp, w = scratch_copy(d) + try: + if (d / "CHANGELOG.md").read_text() != SHOP_FILES["CHANGELOG.md"]: + return False, "CHANGELOG.md was modified" + for rel in sorted(all_files(w)): + if rel == "CHANGELOG.md" or not rel.endswith((".py", ".md", ".json", ".txt", ".toml", ".cfg", ".sh")): + continue + txt = (w / rel).read_text(errors="replace") + if re.search(r"\bget_user\b", txt): + return False, f"{rel} still mentions get_user" + if "fetch_user" not in (w / "docs/API.md").read_text() or "fetch_user" not in (w / "README.md").read_text(): + return False, "docs do not mention fetch_user" + if '"fetch_user"' not in (w / "shop/plugins.json").read_text(): + return False, "shop/plugins.json does not name fetch_user" + if "get_user_by_name" not in (w / "shop/db.py").read_text(): + return False, "get_user_by_name is gone" + ok, why = py_check(w, RenameApi.HARNESS) + if not ok: + return False, why + r = runb(["python3", "-m", "unittest", "discover", "-s", "tests", "-t", "."], cwd=w, timeout=60) + out = (r.stdout + r.stderr).decode() + m = re.search(r"Ran (\d+) tests?", out) + if r.returncode != 0 or not m: + return False, f"the project's tests fail: {out[-200:]}" + if int(m.group(1)) < 5: + return False, f"only {m.group(1)} tests ran, the suite had 5" + return True, "renamed everywhere" + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +# ================================================================ 20. sessionize-logs +class SessionizeLogs: + name = "sessionize-logs" + PROMPT = ( + "access.log is an Apache combined-format log (ip ident user [dd/Mon/yyyy:HH:MM:SS +zzzz] \"request\" " + "status bytes \"referer\" \"user-agent\"). Write sessions.py: `python3 sessions.py access.log out.csv` " + "groups hits into visitor sessions. A visitor is an (ip, user-agent) pair. A hit is a line that parses in " + "that format with a request of the form `METHOD PATH PROTOCOL` (three space-separated parts), except " + "static assets: a PATH (ignoring any ?query) starting with /static/ or equal to /favicon.ico is not a " + "hit; lines that do not parse are skipped. The file is NOT in time order and timestamps carry different " + "UTC offsets: order each visitor's hits by the real instant (ties keep file order). A visitor's hit " + "starts a new session when it comes more than 30 minutes (1800 seconds; exactly 1800 still continues) " + "after the visitor's previous hit. Write out.csv with the header ip,user_agent,start,end,hits," + "duration_seconds, one row per session, start and end in UTC as YYYY-MM-DDTHH:MM:SSZ (first and last " + "hit), duration_seconds = end - start, rows sorted by start, then ip, then user_agent." + ) + + MONTHS = {m: i for i, m in enumerate(["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"], 1)} + RX = re.compile(r'(\S+) \S+ \S+ \[(\d\d)/(\w{3})/(\d{4}):(\d\d):(\d\d):(\d\d) ([+-])(\d\d)(\d\d)\] "([^"]*)" (\d{3}) (\d+|-) "([^"]*)" "([^"]*)"') + + @staticmethod + def gen(seed, n): + rnd = random.Random(seed) + visitors = [("10.0.0.%d" % i, ua) for i in range(1, 5) for ua in ("Mozilla/5.0 (X11; Linux)", "curl/8.0, test")][:6] + [("2001:db8::1", "Mozilla/5.0 (X11; Linux)")] + inv = {v: k for k, v in SessionizeLogs.MONTHS.items()} + lines = [] + base = dt.datetime(2024, 3, 9, 22, 0, 0) + for _ in range(n): + ip, ua = rnd.choice(visitors) + t = base + dt.timedelta(seconds=rnd.choice([rnd.randint(0, 40000), rnd.randint(0, 4000), rnd.randint(0, 40000)])) + if rnd.random() < 0.15: + t = t.replace(second=0) + off = rnd.choice([0, -420, 120, 330, 0]) + local = t + dt.timedelta(minutes=off) + sign = "+" if off >= 0 else "-" + path = rnd.choice(["/", "/a", "/b?x=1", "/static/app.js", "/static/", "/favicon.ico", "/favicon.ico?v=2", "/staticfoo", "/api/x"]) + req = rnd.choice(['GET %s HTTP/1.1', 'POST %s HTTP/2.0', 'GET %s HTTP/1.1']) % path + if rnd.random() < 0.04: + req = "-" + if rnd.random() < 0.03: + req = "GET %s" % path + line = '%s - - [%02d/%s/%d:%02d:%02d:%02d %s%02d%02d] "%s" %s %s "-" "%s"' % ( + ip, local.day, inv[local.month], local.year, local.hour, local.minute, local.second, sign, abs(off) // 60, abs(off) % 60, req, + rnd.choice(["200", "404", "500"]), rnd.choice(["123", "-"]), ua) + if rnd.random() < 0.03: + line = "garbage line without structure" + lines.append(line) + # exact 1800s and 1801s boundaries for one visitor + for k, (ip, ua) in enumerate([("9.9.9.9", "edge/1")]): + for ts in ("2024-03-10 00:00:00", "2024-03-10 00:30:00", "2024-03-10 01:00:01", "2024-03-10 01:00:01", "2024-03-10 05:00:00"): + t = dt.datetime.fromisoformat(ts) + lines.append('%s - - [%02d/%s/%d:%02d:%02d:%02d +0000] "GET /e HTTP/1.1" 200 1 "-" "%s"' % (ip, t.day, inv[t.month], t.year, t.hour, t.minute, t.second, ua)) + rnd.shuffle(lines) + return lines + + @staticmethod + def oracle(lines): + by = {} + for idx, line in enumerate(lines): + m = SessionizeLogs.RX.fullmatch(line) + if not m: + continue + ip, dd, mon, yy, hh, mi, ss, sg, oh, om, req, _st, _b, _ref, ua = m.groups() + parts = req.split(" ") + if len(parts) != 3: + continue + path = parts[1].split("?")[0] + if path.startswith("/static/") or path == "/favicon.ico": + continue + t = dt.datetime(int(yy), SessionizeLogs.MONTHS[mon], int(dd), int(hh), int(mi), int(ss)) + t -= (1 if sg == "+" else -1) * dt.timedelta(hours=int(oh), minutes=int(om)) + by.setdefault((ip, ua), []).append((t, idx)) + rows = [] + for (ip, ua), hits in by.items(): + hits.sort() + sess = [[hits[0][0], hits[0][0], 1]] + for t, _ in hits[1:]: + if (t - sess[-1][1]).total_seconds() > 1800: + sess.append([t, t, 1]) + else: + sess[-1][1] = t + sess[-1][2] += 1 + for a, b, c in sess: + rows.append([ip, ua, a.strftime("%Y-%m-%dT%H:%M:%SZ"), b.strftime("%Y-%m-%dT%H:%M:%SZ"), str(c), str(int((b - a).total_seconds()))]) + rows.sort(key=lambda r: (r[2], r[0], r[1])) + return rows + + @staticmethod + def setup(d: Path) -> None: + write(d, "access.log", "\n".join(SessionizeLogs.gen(1, 60)) + "\n") + + @staticmethod + def grade(d: Path): + if not (d / "sessions.py").exists(): + return False, "no sessions.py" + tmp, w = scratch_copy(d) + try: + for seed, n in ((1, 60), (2, 600)): + lines = SessionizeLogs.gen(seed, n) + (w / "t.log").write_bytes(("\n".join(lines) + "\n").encode()) + (w / "o.csv").unlink(missing_ok=True) + r = runb(["python3", "sessions.py", "t.log", "o.csv"], cwd=w) + if r.returncode != 0 or not (w / "o.csv").exists(): + return False, f"exit {r.returncode}: {r.stderr.decode()[-150:]}" + with open(w / "o.csv", newline="", encoding="utf-8") as f: + rd = list(csv.reader(f)) + if rd[:1] != [["ip", "user_agent", "start", "end", "hits", "duration_seconds"]]: + return False, f"bad header {rd[:1]}" + want = SessionizeLogs.oracle(lines) + if rd[1:] != want: + for a, b in zip(rd[1:], want): + if a != b: + return False, f"seed {seed}: got {a} want {b}" + return False, f"seed {seed}: {len(rd) - 1} sessions, want {len(want)}" + return True, "sessions match" + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +# ================================================================ 21. ledger-balance +class LedgerBalance: + name = "ledger-balance" + PROMPT = ( + "Write ledger.py: `python3 ledger.py FILE [--until YYYY-MM-DD]` reads a plain-text double-entry journal " + "and prints a balance report. Journal: blank lines and lines starting with ; or # are ignored. A " + "transaction starts at a line beginning (column 0) with a date YYYY-MM-DD followed by a space and a " + "description; its postings are the following lines that start with a space or tab, up to the next line " + "that does not (indented comment lines starting with ; are skipped). A posting is `ACCOUNT` or " + "`ACCOUNT` followed by two or more spaces or a tab and then `AMOUNT`, optionally followed by a comment " + "starting with ;. Account names may contain single spaces, and colons make a hierarchy " + "(Assets:Bank:Checking). AMOUNT is dollars: `$` required, with an optional minus before the $ or after " + "it (`-$5`, `$-5`), optional thousands commas, 0 to 2 decimals (`$1,234.50`, `$7`). At most one " + "posting of a transaction may omit its amount: it receives whatever makes the transaction sum to zero. " + "Every transaction must sum to exactly zero after that; a transaction that does not, one with two or " + "more amount-less postings, or one with a malformed amount is an error: print `line N: ...` (N = the " + "line number of its date line) to stderr, nothing to stdout, exit 1. All transactions are validated even " + "when --until excludes them. Report: only transactions dated <= the --until date are counted (all of " + "them without it). Every account and every ancestor of an account (each prefix ending before a colon) " + "gets the sum of the postings in it and below it; accounts whose total is exactly zero are left out. " + "One line per account, sorted by full account name in plain codepoint order, formatted as the amount " + "right-aligned in 12 columns, two spaces, then the account name. Amounts print as $1,234.50 or " + "-$1,234.50, always with two decimals. Use exact decimal arithmetic." + ) + + AMT = re.compile(r"(-?)\$(-?)(\d{1,3}(?:,\d{3})+|\d+)(?:\.(\d{1,2}))?") + + @staticmethod + def fmt(c): + s = f"${abs(c) // 100:,}.{abs(c) % 100:02d}" + return "-" + s if c < 0 else s + + @staticmethod + def oracle(text, until=None): + lines = text.split("\n") + txns = [] + cur = None + for no, raw in enumerate(lines, 1): + if not raw.strip() or raw[0] in ";#": + continue + if raw[0] in " \t": + if raw.strip().startswith(";"): + continue + if cur is not None: + cur["raw"].append((no, raw.strip())) + continue + m = re.match(r"(\d{4}-\d\d-\d\d)(?: |$)", raw) + if m: + cur = {"line": no, "date": m.group(1), "raw": []} + txns.append(cur) + postings = [] + for t in txns: + ps, missing, tot = [], 0, 0 + for no, content in t["raw"]: + m = re.search(r"\t+| {2,}", content) + if m: + acct, rest = content[:m.start()], content[m.end():] + else: + acct, rest = content, "" + rest = rest.split(";")[0].strip() + if acct.startswith(";"): + continue + if not rest: + missing += 1 + ps.append([acct, None]) + continue + am = LedgerBalance.AMT.fullmatch(rest) + if not am or (am.group(1) and am.group(2)): + raise ValueError(f"line {t['line']}: bad amount") + c = int(am.group(3).replace(",", "")) * 100 + int((am.group(4) or "0").ljust(2, "0")) + c = -c if (am.group(1) or am.group(2)) else c + tot += c + ps.append([acct, c]) + if missing > 1: + raise ValueError(f"line {t['line']}: more than one posting without an amount") + for p in ps: + if p[1] is None: + p[1] = -tot + tot = 0 + if tot != 0: + raise ValueError(f"line {t['line']}: unbalanced") + if until is None or t["date"] <= until: + postings.extend(ps) + totals = {} + for acct, c in postings: + parts = acct.split(":") + for i in range(1, len(parts) + 1): + k = ":".join(parts[:i]) + totals[k] = totals.get(k, 0) + c + return "".join(f"{LedgerBalance.fmt(c):>12} {k}\n" for k, c in sorted(totals.items()) if c != 0) + + CASES = [ + ("""; sample +2024-01-05 Groceries + Expenses:Food $12.50 + Assets:Cash + +2024-01-06 Paycheck + Assets:Bank:Checking $2,500.00 ; march + Income:Salary $-2,500.00 +# comment +2024-01-07 Rent +\tExpenses:Housing:Rent\t$1,000 +\tAssets:Bank:Checking +2024-02-01 Transfer to savings + Assets:Bank:Savings Account $500 + Assets:Bank:Checking -$500 +""", None), + ("2024-01-01 a\n A:b $0.10\n A:c $0.20\n B -$0.30\n2024-01-02 b\n A:b -$0.10\n Z\n", None), + ("2024-01-01 open\n Assets:Cash $100\n Equity -$100\n2024-03-01 later\n Assets:Cash $50\n Income -$50\n", "2024-02-01"), + ("2024-01-01 x\n A $1,000,000.01\n B\n", None), + ("2024-01-01 round trip\n A $5\n B -$5\n2024-01-02 undo\n A -$5\n B $5\n", None), + ("2024-01-01 x\n A $1\n B $-1\n C $0.5\n D -$0.50\n", None), + ("2024-01-01 odd spacing\n Assets:My Wallet $3.5\n Equity:Opening Balances\n", None), + ("", None), + # errors + ("2024-01-01 x\n A $1\n B -$2\n", None), + ("2024-01-01 x\n A $1\n B\n C\n", None), + ("2024-01-01 x\n A $1.234\n B -$1.234\n", None), + ("2024-01-01 ok\n A $1\n B -$1\n2024-01-02 bad\n A $1\n B -$2\n", "2024-01-01"), + ("2024-01-01 x\n A 1.00\n B -1.00\n", None), + ] + + @staticmethod + def setup(d: Path) -> None: + write(d, "journal.txt", LedgerBalance.CASES[0][0]) + + @staticmethod + def grade(d: Path): + if not (d / "ledger.py").exists(): + return False, "no ledger.py" + tmp, w = scratch_copy(d) + try: + for i, (text, until) in enumerate(LedgerBalance.CASES): + (w / "j.txt").write_text(text, encoding="utf-8") + args = ["python3", "ledger.py", "j.txt"] + (["--until", until] if until else []) + r = runb(args, cwd=w) + try: + want = LedgerBalance.oracle(text, until) + except ValueError as e: + if r.returncode != 1 or r.stdout.strip(): + return False, f"case {i}: want error ({e}), got exit {r.returncode}, stdout {r.stdout[:60]!r}" + if not re.search(rb"line \d+", r.stderr): + return False, f"case {i}: stderr should say `line N: ...`, got {r.stderr[:80]!r}" + continue + if r.returncode != 0 or r.stdout.decode() != want: + return False, f"case {i}: exit {r.returncode}, got {r.stdout.decode()[:120]!r} want {want[:120]!r}" + r = runb(["python3", "ledger.py", "j.txt", "--until", "2024-02-01"], cwd=w) + (w / "j.txt").write_text(LedgerBalance.CASES[8][0]) + r = runb(["python3", "ledger.py", "j.txt"], cwd=w) + if r.returncode != 1 or not re.search(rb"line 1\b", r.stderr): + return False, f"error message should name line 1, got {r.stderr[:80]!r}" + return True, f"{len(LedgerBalance.CASES)} journals" + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +# ================================================================ 22. async-pool-node +POOL_TESTS = r''' +"use strict"; +const assert = require("assert"); +let mapLimit; +try { ({ mapLimit } = require("./pool.js")); } catch (e) { console.log("cannot load pool.js: " + e.message); process.exit(); } +if (typeof mapLimit !== "function") { console.log("pool.js must export mapLimit"); process.exit(); } +let unhandled = 0; +process.on("unhandledRejection", () => { unhandled++; }); +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); +const tests = []; +const test = (name, fn) => tests.push([name, fn]); + +test("results keep input order and index is passed", async () => { + const items = [...Array(20).keys()]; + const res = await mapLimit(items, 3, async (x, i) => { await sleep((x * 7) % 11); return x * 2 + i; }); + assert.deepStrictEqual(res, items.map((x, i) => x * 2 + i)); +}); +test("never more than `limit` in flight, and uses all of them", async () => { + let running = 0, max = 0; + await mapLimit([...Array(15).keys()], 4, async (x) => { running++; max = Math.max(max, running); await sleep(5 + (x % 3)); running--; }); + assert.strictEqual(max, 4); +}); +test("limit larger than the item count", async () => { + let running = 0, max = 0; + const res = await mapLimit([1, 2, 3], 10, async (x) => { running++; max = Math.max(max, running); await sleep(5); running--; return x; }); + assert.deepStrictEqual(res, [1, 2, 3]); assert.strictEqual(max, 3); +}); +test("empty input", async () => { assert.deepStrictEqual(await mapLimit([], 2, async () => 1), []); }); +test("plain (non-promise) return values and falsy results", async () => { + assert.deepStrictEqual(await mapLimit([1, 2, 3], 2, (x) => (x === 2 ? 0 : x)), [1, 0, 3]); + assert.deepStrictEqual(await mapLimit([1, 2], 1, (x) => (x === 1 ? undefined : null)), [undefined, null]); +}); +test("a free slot is refilled at once (no batches)", async () => { + const start = []; + const t0 = Date.now(); + await mapLimit([80, 10, 10, 10], 2, async (ms, i) => { start[i] = Date.now() - t0; await sleep(ms); }); + assert.ok(start[2] < 50, "item 2 started at " + start[2] + "ms, should be ~10ms"); + assert.ok(start[3] < 60, "item 3 started at " + start[3] + "ms, should be ~20ms"); +}); +test("limit 1 runs strictly one after another, in order", async () => { + const order = []; + await mapLimit([3, 1, 2], 1, async (x) => { order.push("s" + x); await sleep(x); order.push("e" + x); }); + assert.deepStrictEqual(order, ["s3", "e3", "s1", "e1", "s2", "e2"]); +}); +test("first rejection rejects the result; nothing new is started afterwards", async () => { + const started = []; + const p = mapLimit([0, 1, 2, 3, 4, 5], 2, async (i) => { started.push(i); if (i === 0) { await sleep(5); throw new Error("boom0"); } await sleep(30); return i; }); + await assert.rejects(p, /boom0/); + await sleep(100); + assert.deepStrictEqual(started, [0, 1]); +}); +test("a synchronous throw becomes a rejection, not an exception", async () => { + let p; + try { p = mapLimit([0, 1, 2], 2, (i) => { if (i === 1) throw new TypeError("sync"); return sleep(5); }); } + catch (e) { assert.fail("mapLimit threw synchronously: " + e.message); } + await assert.rejects(p, TypeError); +}); +test("later failures after the first do not become unhandled rejections", async () => { + const before = unhandled; + const p = mapLimit([0, 1, 2], 3, async (i) => { await sleep(5 + i * 10); throw new Error("e" + i); }); + await assert.rejects(p, /e0/); + await sleep(80); + assert.strictEqual(unhandled, before, "unhandled rejections leaked"); +}); +test("invalid limits reject with RangeError and run nothing", async () => { + for (const bad of [0, -1, 1.5, NaN, "2", Infinity, undefined, null]) { + let called = 0, p; + try { p = mapLimit([1, 2], bad, async () => { called++; }); } catch (e) { assert.fail("threw synchronously for " + String(bad)); } + await assert.rejects(p, (e) => e instanceof RangeError, "limit " + String(bad)); + assert.strictEqual(called, 0); + } +}); +test("many items, many rounds", async () => { + const items = [...Array(500).keys()]; + const res = await mapLimit(items, 7, async (x) => { if (x % 50 === 0) await sleep(1); return x * x; }); + assert.deepStrictEqual(res, items.map((x) => x * x)); +}); + +(async () => { + for (const [name, fn] of tests) { + try { await fn(); } catch (e) { console.log(name + ": " + String(e.message || e).split("\n")[0].slice(0, 160)); process.exit(); } + } + await sleep(20); + if (unhandled) { console.log("unhandled rejections: " + unhandled); process.exit(); } + console.log("OK"); +})(); +''' + + +class AsyncPoolNode: + name = "async-pool-node" + PROMPT = ( + "Write pool.js (Node 22, CommonJS, no packages) exporting `mapLimit(items, limit, fn)`. It returns a " + "promise for the array of results in input order, calling fn(item, index) (which may return a value or a " + "promise) so that at most `limit` calls are in flight at any moment: calls start in input order, and a " + "new one starts as soon as a running one settles (not in batches). If any call rejects or throws " + "synchronously, the returned promise rejects with that first error, no further items are started, and " + "calls already running are left to finish with their outcome ignored -- a later failure must not surface " + "as an unhandled rejection. An empty array resolves to []. A limit that is not an integer >= 1 makes the " + "returned promise reject with a RangeError without calling fn; mapLimit itself never throws " + "synchronously." + ) + + @staticmethod + def setup(d: Path) -> None: + write(d, "example.js", "const { mapLimit } = require('./pool.js');\nmapLimit([1, 2, 3], 2, async (x) => x * 2).then(console.log);\n") + + @staticmethod + def grade(d: Path): + if not (d / "pool.js").exists(): + return False, "no pool.js" + tmp, w = scratch_copy(d) + try: + (w / "pool_test_hidden.js").write_text(POOL_TESTS.replace('require("./pool.js")', 'require("./pool.js")')) + try: + r = runb(["node", "pool_test_hidden.js"], cwd=w, timeout=60) + except subprocess.TimeoutExpired: + return False, "tests hung" + out = (r.stdout.decode().strip().splitlines() or [""])[-1] + if out == "OK" and r.returncode == 0: + return True, "all pool tests" + return False, (out or r.stderr.decode()[-200:] or f"exit {r.returncode}")[:240] + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +TASKS3 = [DedupeContacts, MoneySplit, FixDailyBuckets, SemverSort, JsonDiff, RefactorInvoice, SqlGaps, GitSplitHistory, + GitFindCulprit, BackupScript, MakeIncremental, CsvToJsonNode, IniToJson, PerfAsofJoin, SlugifyBatch, TextWrap, + SqliteMigrate, VendorUnits, RenameApi, SessionizeLogs, LedgerBalance, AsyncPoolNode] diff --git a/bench/local/tests/test_provider_cap.py b/bench/local/tests/test_provider_cap.py new file mode 100644 index 0000000..5b837de --- /dev/null +++ b/bench/local/tests/test_provider_cap.py @@ -0,0 +1,24 @@ +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import run # noqa: E402 + + +class ProviderCap(unittest.TestCase): + def test_openrouter_daily_cap(self): + self.assertTrue(run.provider_capped('..."rate limit is reached until 17:00 - x"', 3)) + self.assertTrue(run.provider_capped("free-models-per-day", 0)) + self.assertFalse(run.provider_capped("free-models-per-day", 4)) + + def test_opencode_limit(self): + self.assertTrue(run.provider_capped("OpenCode rate limit (HTTP 429): Free usage limit reached", 0)) + self.assertFalse(run.provider_capped("OpenCode rate limit (HTTP 429): Free usage limit reached", 3)) + + def test_ordinary_failure_is_not_a_cap(self): + self.assertFalse(run.provider_capped("tests failed", 5)) + + +if __name__ == "__main__": + unittest.main() diff --git a/bench/local/tests/test_scoreboard.py b/bench/local/tests/test_scoreboard.py new file mode 100644 index 0000000..b20c1e3 --- /dev/null +++ b/bench/local/tests/test_scoreboard.py @@ -0,0 +1,177 @@ +import io +import json +import sys +import tempfile +import unittest +from contextlib import redirect_stdout +from pathlib import Path + +LOCAL = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(LOCAL)) +sys.path.insert(0, str(LOCAL.parent / "harbor")) +import scoreboard as sb # noqa: E402 +import stats # noqa: E402 + +REAL = Path.home() / ".cache/maat-bench/container-results" +V = "verified (self-checked)" + + +def row(task, passed, claim, rep=0, **kw): + return {"task": task, "agent": "molt", "rep": rep, "passed": passed, "claim": claim, + "secs": 100, "turns": 5, "tokens_in": 1000, "timed_out": False, "checks_disagree": [], **kw} + + +class Stats(unittest.TestCase): + def test_wilson(self): + lo, hi = stats.wilson(5, 6) + self.assertAlmostEqual(lo, 0.4365, 3) + self.assertAlmostEqual(hi, 0.9699, 3) + lo, hi = stats.wilson(15, 20) + self.assertAlmostEqual(lo, 0.5313, 3) + self.assertAlmostEqual(hi, 0.8881, 3) + self.assertIsNone(stats.wilson(0, 0)) + self.assertAlmostEqual(stats.wilson(10, 10)[1], 1.0, 9) + self.assertAlmostEqual(stats.wilson(0, 10)[0], 0.0, 9) + + def test_sign_test(self): + self.assertEqual(stats.sign_test(0, 0), 1.0) + self.assertAlmostEqual(stats.sign_test(5, 2), 0.453125) + self.assertAlmostEqual(stats.sign_test(0, 5), 0.0625) + self.assertAlmostEqual(stats.sign_test(6, 0), 0.03125) + self.assertEqual(stats.sign_test(3, 3), 1.0) + + def test_percentile(self): + self.assertEqual(stats.percentile([1, 2, 3, 4], 50), 2.5) + self.assertEqual(stats.percentile(list(range(1, 11)), 90), 9) + self.assertIsNone(stats.percentile([], 50)) + + def test_no_verdict(self): + self.assertTrue(stats.no_verdict({"claim": None})) + self.assertTrue(stats.no_verdict({"claim": "error (self-checked)"})) + self.assertTrue(stats.no_verdict({"claim": "stopped"})) + self.assertTrue(stats.no_verdict({"claim": V, "timed_out": True})) + self.assertFalse(stats.no_verdict({"claim": "unverified"})) + + +class Metrics(unittest.TestCase): + def test_precision_recall(self): + rows = [row("a", True, V), row("b", True, V), row("c", False, V), row("d", True, "unverified"), + row("e", False, None), row("f", True, "error"), + row("g", True, "unverified", checks_disagree=["x"])] + m = sb.metrics(rows) + self.assertEqual((m["n"], m["pass"], m["verified"], m["verified_pass"]), (7, 5, 3, 2)) + self.assertEqual(m["false_done"], 1) + self.assertEqual(m["no_verdict"], 2) + self.assertEqual(m["disagree_passed"], 1) + out = "\n".join(sb.report_arm("x", m)) + self.assertIn("2/3 67%", out) # precision + self.assertIn("2/5 40%", out) # recall + + +class Tiers(unittest.TestCase): + def test_breakdown_counts_and_precision_per_tier(self): + rows = [row("a", True, "verified", tier="verified"), row("b", True, "verified", tier="verified"), + row("c", False, "verified", tier="verified"), + row("d", True, "unverified", tier="passed-checks"), row("e", False, "unverified", tier="passed-checks"), + row("f", True, "unverified")] # an older build's row carries no tier + self.assertEqual(stats.tier_breakdown(rows), {"verified": (3, 2), "passed-checks": (2, 1)}) + out = "\n".join(sb.report_arm("x", sb.metrics(rows))) + self.assertIn("verified n=3 2/3 67%", out) + self.assertIn("passed-checks n=2 1/2 50%", out) + + def test_no_tier_section_for_rows_without_one(self): + out = "\n".join(sb.report_arm("x", sb.metrics([row("a", True, V)]))) + self.assertNotIn("tiers", out) + + +class Compare(unittest.TestCase): + def arms(self, a_pass, b_pass, b_claims=None): + tasks = [f"t{i}" for i in range(len(a_pass))] + ra = [row(t, bool(p), V if p else "unverified") for t, p in zip(tasks, a_pass)] + rb = [row(t, bool(p), (b_claims or {}).get(t, V if p else "unverified")) for t, p in zip(tasks, b_pass)] + return ra, rb + + def test_ship(self): + ra, rb = self.arms([1, 1, 0, 0], [1, 1, 1, 0]) + _, v = sb.compare("a", "b", ra, rb, "pass", 0.1) + self.assertIn("VERDICT b vs a: SHIP", v) + self.assertIn("LOW POWER", v) + + def test_pass_drop(self): + ra, rb = self.arms([1, 1, 1, 0], [1, 0, 0, 0]) + _, v = sb.compare("a", "b", ra, rb, "secs", -1) + self.assertIn("NO-SHIP (pass drop)", v) + + def test_one_task_drop_is_allowed(self): + ra, rb = self.arms([1, 1, 1, 0], [1, 1, 0, 0]) + _, v = sb.compare("a", "b", ra, rb, "pass", -1) + self.assertIn(": SHIP", v) + + def test_false_done_up_blocks(self): + ra, rb = self.arms([1, 0, 0, 0], [1, 0, 0, 0], {"t1": V}) + _, v = sb.compare("a", "b", ra, rb, "pass", 0.0) + self.assertIn("false-done up", v) + + def test_target_not_improved(self): + ra, rb = self.arms([1, 0], [1, 0]) + _, v = sb.compare("a", "b", ra, rb, "precision", 0.1) + self.assertIn("target not improved", v) + + def test_sign_test_reported(self): + ra, rb = self.arms([1] * 5 + [0] * 2, [0] * 5 + [1] * 2) + lines, _ = sb.compare("a", "b", ra, rb, "pass", -9) + self.assertTrue(any("only A passed 5, only B passed 2" in x and "p = 0.453" in x for x in lines)) + + def test_pairs_by_task_and_rep(self): + a = [row("t", True, V, rep=0), row("t", False, "unverified", rep=1)] + b = [row("t", True, V, rep=1)] + self.assertEqual(len(sb.pair_up(a, b)), 1) + + def test_arm_column_grouping(self): + with tempfile.TemporaryDirectory() as d: + f = Path(d) / "r.jsonl" + f.write_text("\n".join(json.dumps(r) for r in [row("t", True, V, arm="base"), row("t", False, V, arm="ref")])) + self.assertEqual(sorted(sb.group([str(f)])), ["base", "ref"]) + + +class RealFiles(unittest.TestCase): + @unittest.skipUnless((REAL / "results-v8-base.jsonl").exists(), "container results not on this machine") + def test_v8(self): + b = sb.metrics(sb.load(REAL / "results-v8-base.jsonl")) + self.assertEqual((b["n"], b["pass"], b["verified"], b["verified_pass"]), (20, 15, 6, 5)) + r = sb.metrics(sb.load(REAL / "results-v8-ref.jsonl")) + self.assertEqual((r["n"], r["pass"]), (20, 12)) + buf = io.StringIO() + with redirect_stdout(buf): + sb.main([f"base={REAL / 'results-v8-base.jsonl'}", f"ref={REAL / 'results-v8-ref.jsonl'}"]) + self.assertIn("5/6 83%", buf.getvalue()) + self.assertIn("5/15 33%", buf.getvalue()) + + +class HarborCompare(unittest.TestCase): + def test_precision_recall_lines(self): + import compare + + def job(root, name, trials): + for i, (reward, said) in enumerate(trials): + d = Path(root) / name / f"t{i}" + (d / "agent").mkdir(parents=True) + (d / "result.json").write_text(json.dumps({"task_name": f"t{i}", "verifier_result": {"rewards": {"reward": reward}}})) + (d / "agent" / "molt.jsonl").write_text(json.dumps({"kind": "job_end", "outcome": said}) + "\n") + return Path(root) / name + + with tempfile.TemporaryDirectory() as root: + a = job(root, "a", [(1, "verified"), (1, "verified"), (0, "verified"), (1, "unverified")]) + b = job(root, "b", [(1, "verified"), (0, "unverified"), (0, "unverified"), (1, "unverified")]) + buf = io.StringIO() + with redirect_stdout(buf): + compare.main([str(a), str(b)]) + out = buf.getvalue() + self.assertIn("precision", out) + self.assertIn("2/3 67% [21-94]", out) # a: precision; recall is 2/3 too + self.assertIn("1/1 100%", out) # b precision + self.assertIn("1/2 50%", out) # b recall + + +if __name__ == "__main__": + unittest.main() diff --git a/bench/local/validate3.py b/bench/local/validate3.py new file mode 100644 index 0000000..29404db --- /dev/null +++ b/bench/local/validate3.py @@ -0,0 +1,108 @@ +""" +Validate tasks3.py. For every task: the grader must FAIL on the untouched start state, PASS (twice, same +verdict) after the reference solution, and FAIL after each deliberately wrong solution. + + $REF (env for solve.sh) is the task's reference directory, so a wrong solution can start from it. + reference_solutions// files dropped into the task folder (after setup); + an optional solve.sh is run there (bash) instead of being copied + reference_solutions/_wrong/// the same layout, for a plausible-but-wrong solution + + python3 validate3.py [task-name ...] +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +from tasks3 import TASKS3 # noqa: E402 + +REFS = HERE / "reference_solutions" + + +def fresh(task) -> Path: + d = Path(tempfile.mkdtemp(prefix=f"v3-{task.name}-")) + task.setup(d) + return d + + +def apply(src: Path, d: Path, ref: Path) -> None: + for p in sorted(src.rglob("*")): + rel = p.relative_to(src) + if rel.parts[0] == "solve.sh" or p.is_dir(): + continue + (d / rel).parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(p, d / rel) + if (src / "solve.sh").exists(): + r = subprocess.run(["bash", str(src / "solve.sh")], cwd=d, capture_output=True, text=True, env={**os.environ, "REF": str(ref)}) + if r.returncode != 0: + raise RuntimeError(f"solve.sh failed in {src}: {r.stderr[-300:]}") + + +def main() -> int: + only = set(sys.argv[1:]) + bad = 0 + n = 0 + names = [t.name for t in TASKS3] + assert len(names) == len(set(names)), "duplicate task names" + for task in TASKS3: + if only and task.name not in only: + continue + n += 1 + t0 = time.time() + problems = [] + d = fresh(task) + try: + ok0, why0 = task.grade(d) + finally: + shutil.rmtree(d, ignore_errors=True) + if ok0: + problems.append("grader PASSES the untouched start state") + ref = REFS / task.name + if not ref.is_dir(): + problems.append("no reference solution") + ok1 = ok2 = False + why1 = "" + else: + d = fresh(task) + try: + apply(ref, d, ref) + ok1, why1 = task.grade(d) + ok2, why2 = task.grade(d) + finally: + shutil.rmtree(d, ignore_errors=True) + if not ok1: + problems.append(f"reference fails: {why1}") + elif not ok2 or why1 != why2: + problems.append(f"grader not repeatable: {why1!r} then {why2!r}") + wrongs = sorted((REFS / "_wrong" / task.name).glob("*")) if (REFS / "_wrong" / task.name).is_dir() else [] + wnote = [] + for w in wrongs: + d = fresh(task) + try: + apply(w, d, ref) + okw, whyw = task.grade(d) + finally: + shutil.rmtree(d, ignore_errors=True) + if okw: + problems.append(f"wrong solution {w.name} PASSES") + wnote.append(f"{w.name}:{whyw[:45]}") + bad += bool(problems) + print(f"{'FAIL' if problems else 'PASS'} {task.name:24} start: {why0[:50]!r} | ref: {ok1} | wrong x{len(wrongs)} [{time.time() - t0:.1f}s]") + for p in problems: + print(" !!", p) + for w in wnote: + print(" wrong", w) + print(f"\n{n} tasks, {bad} problems") + return 1 if bad else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/missions.md b/docs/missions.md index faa4264..396744a 100644 --- a/docs/missions.md +++ b/docs/missions.md @@ -55,7 +55,7 @@ one JSON per worker attempt. ## What there is not -**An orchestrator model.** Factory's missions put a frontier model in a loop +**An orchestrator model.** A competitor's missions put a frontier model in a loop with seventeen tools to decide what runs next, read each worker's handoff, delegate a review of it to more subagents, and decide whether a milestone is done. It is the most expensive process in their system, and it is doing a job @@ -63,7 +63,7 @@ a loop can do. Here the orchestrator is `src/mission.ts`. It reads a queue, starts a worker, reads a verdict the bar produced, and moves on. It costs no tokens. -**Validators that are models.** Factory injects a "scrutiny validator" and a +**Validators that are models.** A competitor injects a "scrutiny validator" and a "user-testing validator" at each milestone: subagents that read the work and say whether it is good. Their CTO, on the record: when the model cannot run something "it's going to validate this by just looking at it, and it will @@ -71,7 +71,7 @@ fundamentally make a mistake." A milestone here is sealed by running the assertions. If an assertion cannot be a command, it is a note, it is on every receipt as stated intent, and it is never reported as passed. -**A validation contract written in prose.** Factory's contract is a markdown +**A validation contract written in prose.** A competitor's contract is a markdown checklist with "evidence requirements" a worker is asked to satisfy. Here an assertion is a command or it is not an assertion. That is the whole difference, and it is the difference between a checklist a model reads and a diff --git a/docs/prior-art.md b/docs/prior-art.md index edc76b7..331117d 100644 --- a/docs/prior-art.md +++ b/docs/prior-art.md @@ -130,7 +130,7 @@ it requires owning both compaction and verification in one harness. ## Long-horizon work -**Factory Missions** (Factory AI, 2026). A frontier-model orchestrator +**A competitor's missions** (2026). A frontier-model orchestrator interviews the user, writes a validation contract of prose assertions with stable ids, decomposes the goal into `features.json`, designs worker skills, then runs a worker session per feature and injects model "validators" at each @@ -145,14 +145,14 @@ the work looks done. Their CTO's own account of what a model does when it cannot run something — "it will validate this by just looking at it, and it will fundamentally make a mistake" — is the argument for all three. -**Ralph loops.** The pattern Factory credits as prior: run an agent in a +**Ralph loops.** The pattern a competitor credits as prior: run an agent in a `while true` against a fixed prompt until a condition holds. molt's proof loop is the same idea with the condition made mechanical and the claim refused rather than the loop merely continued. ## Terminal-Bench harness engineering -**Factory's Terminal-Bench write-up** lists what moved their score with the +**A competitor's Terminal-Bench write-up** lists what moved their score with the same model: a small tool repertoire, bootstrapping each session with system information, short default timeouts the agent may raise, a planning tool that leans on recency bias, a controlled background-execution primitive, and diff --git a/finetune/extract.mjs b/finetune/extract.mjs index 01cd365..8ef7a32 100755 --- a/finetune/extract.mjs +++ b/finetune/extract.mjs @@ -23,7 +23,9 @@ * which is adjudicated by running them. It is never asked to decide. * - Attempts made by a provider whose terms bar using its output to develop * machine learning models are left out entirely — not in records.jsonl, - * not in a split. See EXCLUDED_PROVIDERS below. + * not in a split. That covers the judge as well as the worker: a run whose + * checks were drafted or whose claim was reviewed by such a model carries + * its output too. See EXCLUDED_PROVIDERS below. * * Usage: * node finetune/extract.mjs [--out finetune/data/] [--valid 0.15] @@ -56,8 +58,28 @@ import { basename, join, resolve } from "node:path"; */ const EXCLUDED_PROVIDERS = new Set(["xai", "x.ai", "api.x.ai", "grok-build", "grok build"]); const EXCLUDED_MODEL = /(^|\/)grok-/iu; -function isExcluded(provider, model) { - return EXCLUDED_PROVIDERS.has(String(provider ?? "").trim().toLowerCase()) || EXCLUDED_MODEL.test(String(model ?? "")); +/** An endpoint that is xAI's, directly or through the Grok Build CLI. */ +const EXCLUDED_URL = /^grok-build:\/\/|^https?:\/\/([^/]*\.)?x\.ai(\/|:|$)/iu; +/** + * The receipt's `- judge: at ` line (src/receipts.ts), or the + * index row's `judge` / `judgeUrl`, as { model, url }. No judge line: the + * worker judged itself, and the worker is checked already. + */ +function judgeOf(meta, idx) { + const line = String(meta["judge"] ?? "").trim(); + const at = line.lastIndexOf(" at "); + const model = (at >= 0 ? line.slice(0, at) : line).trim() || String(idx.judge ?? "").trim(); + const url = (at >= 0 ? line.slice(at + 4) : "").trim() || String(idx.judgeUrl ?? "").trim(); + return { model, url }; +} +function isExcluded(provider, model, judge = { model: "", url: "" }) { + return ( + EXCLUDED_PROVIDERS.has(String(provider ?? "").trim().toLowerCase()) || + EXCLUDED_MODEL.test(String(model ?? "")) || + EXCLUDED_MODEL.test(judge.model) || + EXCLUDED_URL.test(judge.url) || + EXCLUDED_PROVIDERS.has(judge.url.toLowerCase()) + ); } const args = process.argv.slice(2); @@ -192,7 +214,7 @@ for (const { root, state } of sources) { const idx = index.filter((x) => x.file === f).at(-1) ?? {}; // Left out before anything is built from it, so no field of an excluded // attempt reaches records.jsonl either. - if (isExcluded(r.meta["provider"] ?? idx.provider, r.meta["model"] ?? idx.model)) { excludedIds.add(id); continue; } + if (isExcluded(r.meta["provider"] ?? idx.provider, r.meta["model"] ?? idx.model, judgeOf(r.meta, idx))) { excludedIds.add(id); continue; } // Which session: the index row says, or a journal names this receipt. let session = idx.session ?? null; if (!session) for (const [sid, es] of journals) if (es.some((e) => e.kind === "receipt" && basename(String(e.data.file ?? "")) === f)) { session = sid; break; } @@ -303,6 +325,6 @@ const manifest = { }; writeFileSync(join(out, "manifest.json"), JSON.stringify(manifest, null, 2) + "\n"); console.log(`${all.length} unique receipts from ${seenFiles} files across ${roots.length} root(s) → ${out}`); -if (excludedIds.size) console.log(`left out ${excludedIds.size} receipt(s) from providers whose terms bar training on their output`); +if (excludedIds.size) console.log(`left out ${excludedIds.size} receipt(s) whose worker or judge was a provider whose terms bar training on their output`); console.log(`train ${train.length} · valid ${valid.length} (${validSessions.size} held-out session(s)) · with journal turn ${manifest.withJournalTurn}`); console.log(`verdicts ${JSON.stringify(manifest.verdicts)} · failing checks ${JSON.stringify(manifest.failingChecks)}`); diff --git a/src/acp.ts b/src/acp.ts index dfd7066..89e8286 100644 --- a/src/acp.ts +++ b/src/acp.ts @@ -14,7 +14,7 @@ * and that process authenticates itself. molt is the client on the other end * of a documented protocol the vendor shipped for exactly this. * - * Grok Build is the only CLI driven this way. The Claude Code, Antigravity + * Grok Build and OpenCode are the CLIs driven this way. The Claude Code, Antigravity * and Gemini CLI backends were removed over those vendors' terms on running * a third-party harness on a plan; see `docs/provider-terms.md`. * @@ -83,7 +83,8 @@ import { promisify } from "node:util"; import { errorText } from "./format.js"; import { RpcPeer, type RpcMessage } from "./jsonrpc.js"; import type { BackendEvent, MoltTool, ToolRunner } from "./backend.js"; -import { GROK_BUILD_URL } from "./endpoint.js"; +import { GROK_BUILD_URL, OPENCODE_URL } from "./endpoint.js"; +import { OPENCODE_CONFIG } from "./opencode.js"; import { estTokens } from "./types.js"; const exec = promisify(execFile); @@ -103,6 +104,8 @@ export type AcpAgentSpec = { readonly url: string; readonly bin: string; readonly args: readonly string[]; + /** Extra environment for the child, on top of the process's own. */ + readonly env?: Readonly>; /** Aliases the CLI resolves itself against whatever the account can reach. */ readonly models: readonly string[]; readonly installHint: string; @@ -122,6 +125,13 @@ export type AcpAgentSpec = { * rather than being optional, so the call site has no branch. */ readonly sessionMeta: (o: { systemPrompt: string }) => Record; + /** + * The session meta for a pre-turn question (acpAsk), where no MCP server is + * offered. Defaults to `sessionMeta`; Grok needs its own, because its + * session meta tells it every tool lives on an MCP server `molt` that a + * question never has, and the model went looking for it instead of answering. + */ + readonly askMeta?: (o: { systemPrompt: string }) => Record; }; /** @@ -129,6 +139,55 @@ export type AcpAgentSpec = { * the documented way to say "these builtins and no others". Empty means none: * everything the model can do arrives over molt's MCP server. */ +/** + * Grok 1.0.46 does not list MCP tools to the model at all. Every MCP tool is + * reached through two of its own meta-tools: `search_tool` (discovery, never + * asks permission) and `use_tool` with `{tool_name: "molt__grep", tool_input}`. + * Measured against the real CLI over ACP: the `tool_call` announcement is + * titled `use_tool`, and the permission request carries + * `rawInput: {variant: "UseTool", tool_name: "molt__grep", ...}` with + * `_meta["x.ai/tool"].name === "use_tool"` and no `toolName`. Judging by the + * title therefore saw "use_tool" for every Maat call and refused all of them, + * so the model never got a single tool and the turn ended with nothing done. + * + * The real identity of such a call is the tool it targets. A `use_tool` whose + * target is not a fully-qualified Maat tool (another MCP server, a target + * hidden in an arguments file, a bare name that could belong to anyone) keeps + * the name `use_tool` and is refused like any other builtin. + */ +export function useToolTarget(raw: unknown, title?: string, metaName?: string): string | undefined { + const o = raw && typeof raw === "object" ? (raw as Record) : {}; + const wrapped = title === "use_tool" || metaName === "use_tool" || o.variant === "UseTool"; + if (!wrapped) return undefined; + const target = o.tool_name; + return typeof target === "string" && target ? target : "use_tool"; +} + +/** Grok's discovery meta-tool: lists tool names and schemas, changes nothing. */ +const GROK_DISCOVERY_TOOL = "search_tool"; + +/** + * Said to Grok because it cannot be inferred from its tool list: that list has + * no Maat tools in it (see `useToolTarget`), so a model told only "use your + * tools" reaches for `run_terminal_command`, is refused, and stops. + */ +export const GROK_TOOL_ROUTE = [ + "TOOL ROUTING (mandatory). Every tool you may use is served by the MCP server `molt`.", + "They are not in your direct tool list. Call `search_tool` with query `molt` to list them with their schemas,", + "then call each one through `use_tool` with `tool_name` set to the fully-qualified name `molt__`", + "(for example `molt__grep`) and `tool_input` set to its arguments.", + "Never call run_terminal_command, read_file, search_replace, write, list_dir, grep or any other built-in tool directly: they are refused.", +].join(" "); + +/** + * A pre-turn question has no tools at all. Without this Grok, still believing + * the routing text above, spent 65 s "looking through the project" and answered + * in prose, so the drafted checks never arrived (bench grok1, 29 of 29 runs). + */ +export const GROK_ASK_NO_TOOLS = + "You have NO tools in this conversation: do not search for tools, do not read files, do not run anything. " + + "Everything you need is in the message. Answer immediately, in text, in exactly the format the message asks for."; + const GROK_MOLT_PROFILE = { name: "molt", description: "Maat drives every tool", tools: "" }; export const ACP_AGENTS: readonly AcpAgentSpec[] = [ @@ -149,10 +208,30 @@ export const ACP_AGENTS: readonly AcpAgentSpec[] = [ credentialPath: ".grok/auth.json", mcpTransport: "http", sessionMeta: ({ systemPrompt }) => ({ - systemPromptOverride: systemPrompt, + systemPromptOverride: `${systemPrompt}\n\n${GROK_TOOL_ROUTE}`, + agentProfile: GROK_MOLT_PROFILE, + }), + askMeta: ({ systemPrompt }) => ({ + systemPromptOverride: `${systemPrompt}\n\n${GROK_ASK_NO_TOOLS}`, agentProfile: GROK_MOLT_PROFILE, }), }, + { + name: "opencode", + label: "OpenCode", + url: OPENCODE_URL, + bin: "opencode", + args: ["acp"], + // Every permission is "ask": a deny breaks the free tier (see opencode.ts), and an ask + // reaches `session/request_permission`, where Maat refuses all but its own tools. + env: { OPENCODE_CONFIG_CONTENT: OPENCODE_CONFIG }, + models: ["opencode/big-pickle"], + installHint: "npm install -g opencode-ai", + loginHint: "opencode auth login", + credentialPath: ".local/share/opencode/auth.json", + mcpTransport: "http", + sessionMeta: () => ({}), + }, ]; /** Is this endpoint one of the ACP CLIs rather than an HTTP API? */ @@ -381,7 +460,7 @@ export class AcpConnection { // empty PATH and it cannot find its own helpers. // `electron/login-path.ts` has already repaired process.env.PATH by the // time anything gets here. - env: { ...process.env }, + env: { ...process.env, ...this.spec.env, ...(this.spec.env ? { PWD: this.opts.cwd ?? process.cwd() } : {}) }, }); this.child = child; // A write to a child that has died raises EPIPE as an 'error' event on its @@ -863,6 +942,15 @@ export class AcpSession { } } + /** + * OpenCode names an MCP tool `_` (`molt_write_file`) where the others say + * `molt__write_file`. Its own builtins have no `molt_` prefix, so the rewrite cannot + * promote one of them into a Maat tool. + */ + private canon(name: string): string { + return this.opts.spec.name === "opencode" ? name.replace(/^molt_(?=[a-z])/u, "molt__") : name; + } + /** Notifications: the streamed reply, thoughts, tool calls, plans. */ private onNotify(method: string, params: unknown): void { if (method !== "session/update" && method !== "x.ai/session/update") return; @@ -881,7 +969,7 @@ export class AcpSession { const name = String(u.title ?? (u as { toolCallId?: string }).toolCallId ?? ""); const raw = (u.rawInput ?? {}) as Record; const id = String((u as { toolCallId?: string }).toolCallId ?? `acp_${Date.now().toString(36)}`); - const called = String((u as { toolName?: string }).toolName ?? name); + const called = this.canon(useToolTarget(raw, name) ?? String((u as { toolName?: string }).toolName ?? name)); this.toolCallNames.set(id, called); if (McpToolServer.isMoltTool(called)) { // The transcript records the call Maat is about to run — the handler @@ -892,6 +980,9 @@ export class AcpSession { text: "", toolCalls: [{ id, name: McpToolServer.bareName(called), args: raw }], }); + } else if (called === GROK_DISCOVERY_TOOL) { + // Discovery of Maat's own catalogue: nothing to account for. + return; } else { /** * A builtin, announced. Nothing is concluded yet. @@ -941,24 +1032,33 @@ export class AcpSession { throw new Error(`maat does not implement ${method}`); } const p = (params ?? {}) as { - toolCall?: { toolCallId?: string; title?: string; toolName?: string }; + toolCall?: { + toolCallId?: string; + title?: string; + toolName?: string; + rawInput?: unknown; + _meta?: Record; + }; options?: { optionId?: string; kind?: string }[]; }; - const called = String( - p.toolCall?.toolName ?? - this.toolCallNames.get(String(p.toolCall?.toolCallId ?? "")) ?? - p.toolCall?.title ?? - "", + const called = this.canon( + String( + useToolTarget(p.toolCall?.rawInput, p.toolCall?.title, p.toolCall?._meta?.["x.ai/tool"]?.name) ?? + p.toolCall?.toolName ?? + this.toolCallNames.get(String(p.toolCall?.toolCallId ?? "")) ?? + p.toolCall?.title ?? + "", + ), ); const options = p.options ?? []; - if (McpToolServer.isMoltTool(called)) { + if (McpToolServer.isMoltTool(called) || called === GROK_DISCOVERY_TOOL) { const allow = options.find((o) => o.kind === "allow_always") ?? options.find((o) => o.kind === "allow_once"); if (allow?.optionId) return { outcome: { outcome: "selected", optionId: allow.optionId } }; } const reject = options.find((o) => o.kind === "reject_always") ?? options.find((o) => o.kind === "reject_once"); - if (!McpToolServer.isMoltTool(called)) { + if (!McpToolServer.isMoltTool(called) && called !== GROK_DISCOVERY_TOOL) { for (const [id, name] of this.inFlight) if (name === called) this.inFlight.delete(id); if (!this.refused.has(called)) { this.refused.add(called); @@ -1126,6 +1226,7 @@ export async function acpAsk( onRequest: async () => ({ outcome: { outcome: "cancelled" } }), }); let answer = ""; + const askMeta = opts.spec.askMeta ?? opts.spec.sessionMeta; const limit = opts.timeoutMs ?? 0; let timer: ReturnType | undefined; const expired = @@ -1148,7 +1249,7 @@ export async function acpAsk( const res = (await conn.request("session/new", { cwd: opts.cwd ?? process.cwd(), mcpServers: [], - _meta: opts.spec.sessionMeta({ systemPrompt: opts.systemPrompt }), + _meta: askMeta({ systemPrompt: opts.systemPrompt }), })) as { sessionId?: string; models?: AcpModelState }; if (!res?.sessionId) return { ok: false, error: `${opts.spec.bin} opened no session` }; // The same choice the session makes, for the same reason: a draft written @@ -1165,7 +1266,7 @@ export async function acpAsk( if (offered.length) throw e; }); } - const prompt = ("systemPromptOverride" in opts.spec.sessionMeta({ systemPrompt: opts.systemPrompt }) + const prompt = ("systemPromptOverride" in askMeta({ systemPrompt: opts.systemPrompt }) ? opts.prompt : `${opts.systemPrompt}\n\n${opts.prompt}`); await conn.request("session/prompt", { diff --git a/src/ask.ts b/src/ask.ts index 67d17f4..5d98748 100644 --- a/src/ask.ts +++ b/src/ask.ts @@ -13,10 +13,21 @@ * person or a gate will read, never work. */ import { acpAgentFor, acpAsk } from "./acp.js"; +import { isOpencode, opencodeAsk } from "./opencode.js"; import { removedSubscriptionProblem } from "./endpoint.js"; import { errorText } from "./format.js"; -import { authHeaders } from "./providers.js"; -import { askError, askTimeoutMs, probeSignal } from "./watchdog.js"; +import { authHeaders, isSelfHosted, selfHostedThinking, openRouterProvider } from "./providers.js"; +import { LONG_RATE_LIMIT_MS, longQuotaText, providerErrorText, rateLimitResetAt, readStream, transientProviderError, untilText, type ProviderError } from "./stream.js"; +import { askError, askTimeoutMs, localSpeed, probeSignal } from "./watchdog.js"; +import { takeTurn } from "./localgate.js"; + +/** + * Pauses before asking again when the provider said it is overloaded or + * rate-limited — inside a 200 (`{"error":{"code":503,...}}`) or as a 429/503. + * Read as a reply, that error was an empty answer: on a free model the critic + * "could not review" two drafts in five and the reviewer abstained. + */ +export const ASK_OVERLOAD_BACKOFF_MS = [2_000, 5_000, 10_000, 20_000, 30_000]; export type AskOptions = { baseUrl: string; @@ -34,10 +45,23 @@ export type AskOptions = { what?: string; fetchFn?: typeof fetch; acpSpawn?: typeof import("node:child_process").spawn; + /** The CLI runner for the opencode ask path. Tests only. */ + cliRun?: (cmd: string, args: string[], opts: object) => Promise<{ stdout: string }>; timeoutMs?: number; + /** Pauses between asks after an overloaded provider (ASK_OVERLOAD_BACKOFF_MS). Tests only. */ + overloadBackoffMs?: number[]; + /** + * The session's latency learner (Engine.askLatency). A completed ask to a + * cloud endpoint records how long it took, so the first stall of the engine's + * own requests is judged against what this provider has shown the drafter + * and the critic, not the full fixed allowance: three samples are needed + * before the learner tightens anything, and the drafter supplies two or + * three before the first step. Never recorded for a self-hosted server. + */ + latency?: { record(w: { firstProgressMs: number | undefined; maxGapMs: number }): void }; }; -export type Asked = { ok: true; text: string; cutOff: boolean } | { ok: false; error: string }; +export type Asked = { ok: true; text: string; cutOff: boolean } | { ok: false; error: string; transient?: true }; /** * The output ceiling for one ask, unless the caller says otherwise. @@ -54,6 +78,16 @@ export const ASK_MAX_TOKENS = 2_000; export const ASK_RETRY_FACTOR = 4; export async function askModel(opts: AskOptions): Promise { + const pauses = opts.overloadBackoffMs ?? ASK_OVERLOAD_BACKOFF_MS; + let asked = await askSized(opts); + for (let i = 0; i < pauses.length && !asked.ok && asked.transient; i++) { + await new Promise((r) => setTimeout(r, pauses[i])); + asked = await askSized(opts); + } + return asked; +} + +async function askSized(opts: AskOptions): Promise { const first = await askOnce(opts, opts.maxTokens ?? ASK_MAX_TOKENS); // Empty AND cut off means the model spent the whole ceiling reasoning and // never started the answer. Once, with room: a model that empties a four @@ -77,11 +111,29 @@ export async function askModel(opts: AskOptions): Promise { } async function askOnce(opts: AskOptions, maxTokens: number): Promise { - const limitMs = askTimeoutMs(maxTokens, opts.timeoutMs); + // Sized from this question's own prompt, and at local-hardware rates for a + // self-hosted server (watchdog.ts localSpeed): the NUC's drafter timed out + // on a fixed 2,000-token guess and a laptop's rates, and sealed nothing. + const limitMs = askTimeoutMs(maxTokens, opts.timeoutMs, { + promptTokens: Math.ceil((opts.system.length + opts.prompt.length) / 4), + ...(isSelfHosted(opts.baseUrl) ? { speed: localSpeed() } : {}), + }); const removed = removedSubscriptionProblem(opts.baseUrl); if (removed) return { ok: false, error: removed }; + if (isOpencode(opts.baseUrl)) { + const asked = await opencodeAsk({ + timeoutMs: limitMs, + model: opts.model, + systemPrompt: opts.system, + prompt: opts.prompt, + ...(opts.cliRun ? { run: opts.cliRun } : {}), + }); + if (asked.ok) return { ok: true, text: asked.text, cutOff: false }; + return { ok: false, error: longQuotaText(asked.error) ?? asked.error, ...(asked.transient ? { transient: true as const } : {}) }; + } + const acp = acpAgentFor(opts.baseUrl); if (acp) { const asked = await acpAsk({ @@ -93,11 +145,24 @@ async function askOnce(opts: AskOptions, maxTokens: number): Promise { cwd: opts.cwd, ...(opts.acpSpawn ? { spawnFn: opts.acpSpawn } : {}), }); - return asked.ok ? { ok: true, text: asked.text, cutOff: false } : asked; + if (asked.ok) return { ok: true, text: asked.text, cutOff: false }; + return { ok: false, error: longQuotaText(asked.error) ?? asked.error }; } const f = opts.fetchFn ?? fetch; const base = opts.baseUrl.replace(/\/$/, ""); + // One request at a time to a self-hosted server, and the timeout below + // starts only once this one has its turn (src/localgate.ts). + const release = await takeTurn(base); + // A self-hosted server is asked with a streamed request. llama.cpp keeps + // generating a non-streamed request after the client has gone: on the NUC + // every ask Maat timed out or cancelled went on holding the only slot for + // minutes, with nobody to read the answer. It stops a streamed request when + // the connection closes. The reply is the same text either way; reasoning + // deltas are read and dropped. The timeout is unchanged: still one bound on + // the whole answer. + const stream = isSelfHosted(base); + const sentAt = Date.now(); try { const res = await f(`${base}/chat/completions`, { method: "POST", @@ -112,19 +177,58 @@ async function askOnce(opts: AskOptions, maxTokens: number): Promise { max_tokens: maxTokens, temperature: 0, ...(opts.reasoningEffort ? { reasoning: { effort: opts.reasoningEffort } } : {}), + ...selfHostedThinking(base, opts.reasoningEffort), + ...openRouterProvider(base, opts.model), + ...(stream ? { stream: true } : {}), }), }); - if (!res.ok) return { ok: false, error: `HTTP ${res.status}${opts.what ? ` ${opts.what}` : ""}` }; - const json = (await res.json()) as { + if (!res.ok) { + const body = res.status === 429 ? await res.text().catch(() => "") : ""; + const resetAt = rateLimitResetAt(body); + if (resetAt !== undefined && resetAt - Date.now() > LONG_RATE_LIMIT_MS) { + return { ok: false, error: `the provider's rate limit is reached ${untilText(resetAt)}${opts.what ? ` (${opts.what})` : ""}` }; + } + return { ok: false, error: `HTTP ${res.status}${opts.what ? ` ${opts.what}` : ""}`, ...(res.status === 429 || res.status === 503 ? { transient: true } : {}) }; + } + type Reply = { choices?: { message?: { content?: string | null }; finish_reason?: string | null }[]; + error?: ProviderError; }; - return { - ok: true, - text: json.choices?.[0]?.message?.content ?? "", - cutOff: json.choices?.[0]?.finish_reason === "length", - }; + let content: string; + let finish: string | null | undefined; + let error: ProviderError | undefined; + // A server that was sent `stream: true` and answers with plain JSON (a + // proxy that buffers, or an error body) is read as JSON, as it always was. + if (stream && res.body && /text\/event-stream/i.test(res.headers.get("content-type") ?? "")) { + const done = await readStream(res.body, () => {}); + content = done.message.content ?? ""; + finish = done.finishReason; + error = done.error; + } else { + const json = (await res.json()) as Reply; + content = json.choices?.[0]?.message?.content ?? ""; + finish = json.choices?.[0]?.finish_reason; + error = json.error && typeof json.error === "object" ? json.error : undefined; + } + if (error) { + const resetAt = rateLimitResetAt({ error }); + if (resetAt !== undefined && resetAt - Date.now() > LONG_RATE_LIMIT_MS) { + return { ok: false, error: `the provider's rate limit is reached ${untilText(resetAt)}${opts.what ? ` (${opts.what})` : ""}` }; + } + return { ok: false, error: `${providerErrorText(error)}${opts.what ? ` (${opts.what})` : ""}`, ...(transientProviderError(error) ? { transient: true } : {}) }; + } + // The answer is small and read whole, so the time it took is both the wait + // for the first byte and the longest silence: the slow end of what this + // provider does, which is what the learner wants to know. + if (!stream && opts.latency) { + const took = Date.now() - sentAt; + opts.latency.record({ firstProgressMs: took, maxGapMs: took }); + } + return { ok: true, text: content, cutOff: finish === "length" }; } catch (e) { return { ok: false, error: askError(e, limitMs, errorText) }; + } finally { + release(); } } diff --git a/src/autonomy.ts b/src/autonomy.ts index 56bdc99..10f7b4b 100644 --- a/src/autonomy.ts +++ b/src/autonomy.ts @@ -318,6 +318,58 @@ export function deletesOnlyCreated( return true; } +/** `tee [-flags] file...` with plain-word targets; the rest of the line is not matched. */ +const TEE_TARGETS = + /\btee((?:[ \t]+-[a-zA-Z]+)*)((?:[ \t]+(?:"[^"$`\\\s]+"|'[^'\s]+'|[^\s;&|<>$`*?{}()'"\\-][^\s;&|<>$`*?{}()'"\\]*))+)/g; + +/** + * The command with the body of every `cat`/`tee` heredoc taken out. + * + * `cat << 'EOF' > report.csv … EOF` is a file being written from text, and the + * text is not run. But the same body fed to `bash` or `python` is a program, + * so only a heredoc line that starts with cat or tee, has no second command + * on it, and (when its delimiter is unquoted, so the shell expands it) holds + * no `$(` or backtick is stripped. Anything else — an unterminated body, a + * different consumer — is returned untouched, and the ordinary rules ask. + */ +function withoutHeredocData(command: string): string { + const lines = command.split("\n"); + const out: string[] = []; + for (let i = 0; i < lines.length; i++) { + const line = lines[i]!; + const m = /<<(-?)[ \t]*(['"]?)([A-Za-z_]\w*)\2/.exec(line); + if (!m) { + out.push(line); + continue; + } + let end = -1; + for (let j = i + 1; j < lines.length; j++) { + if ((m[1] ? lines[j]!.trim() : lines[j]) === m[3]) { + end = j; + break; + } + } + const body = end < 0 ? [] : lines.slice(i + 1, end); + const readable = + end >= 0 && + /^\s*(cat|tee)\b/.test(line) && + !/;|&&|\|\|/.test(line) && + (m[2] !== "" || !body.some((b) => /\$\(|`/.test(b))); + if (!readable) return command; + out.push(line.replace(m[0], " ")); + i = end; + } + return out.join("\n"); +} + +const ECHO_ARG = `(?:'[^']*'|"(?:[^"$\`\\\\]|\\\\[nt\\\\"])*"|[^\\s;&|<>'"\`$()\\\\])+`; +const ECHO_DATA = new RegExp(`(^|[;&|\\n]\\s*)(?:echo|printf)(?:[ \\t]+${ECHO_ARG})*`, "g"); + +/** `echo`/`printf` and their literal arguments reduced to `echo`: text is not a command. */ +function withoutEchoData(command: string): string { + return command.replace(ECHO_DATA, (_m, lead: string) => `${lead}echo`); +} + /** Would this command do something no later step could undo? */ /** * Does this command's only irreversible act write files that did not exist? @@ -343,14 +395,35 @@ export function overwritesOnlyNew( cwd: string, tmp: string = tmpdir(), ): boolean { - const bare = command.replace(HARMLESS_REDIRECT, " "); - // Every `> target` and `N> target`. `>>` never reached the list, and `>&2` + // Heredoc bodies and echo/printf arguments are text being written, not + // commands: a script whose body mentions `rm` is not an rm. Both strips + // are all-or-nothing and leave anything they cannot read exactly as it was. + const prepared = withoutEchoData(withoutHeredocData(command)); + // A heredoc that was not stripped is one whose body could not be read. + if (/< { + for (const w of list.trim().split(/[ \t]+/)) teed.push(w); + return " "; + }); + const bare = text.replace(HARMLESS_REDIRECT, " "); + // Every `> target` and `N> target`. `>>` never reached the list, and `>&2` // style duplications are not files. const re = /(?&])\d?>(?![>&])\s*(\S+)/g; const targets: string[] = []; // Quotes off, and the shell punctuation a target can run into (`> out;`). for (const m of bare.matchAll(re)) targets.push(m[1].replace(/[;&|)]+$/, "").replace(/^['"]|['"]$/g, "")); - if (!targets.length) return false; + for (const w of teed) targets.push(w.replace(/^['"]|['"]$/g, "")); + // `>> file` loses nothing, but it is what `echo 'rm -- "$f"' >> rotate.sh` + // is written with, so the stripped text above has to be allowed to stand on + // its own. Only a plain path inside the project counts. + const appended = [...bare.matchAll(/>>\s*(\S+)/g)].map((m) => m[1].replace(/[;&|)]+$/, "").replace(/^['"]|['"]$/g, "")); + if (!targets.length && !appended.length) return false; + for (const t of appended) { + if (/[*?$`{}]/.test(t) || !insideProject(cwd, t)) return false; + const abs = resolve(cwd, t); + if (existsSync(abs) && statSync(abs).isDirectory()) return false; + } // What is left must be reversible on its own: `rm a > log` is still an rm. if (isIrreversible(bare.replace(re, " "))) return false; for (const t of targets) { @@ -494,11 +567,17 @@ export function insideProject(cwd: string, p: unknown): boolean { const READING_TOOLS = new Set(["read_file", "list_dir", "grep", "inspect"]); /** - * Tools that touch nothing at all. `plan` writes a note into the model's own + * Tools that touch nothing at all. `act` is here for the case where it reaches + * the gate as itself: a readable batch is expanded into its inner actions + * (each gated on its own) before it gets here, so the one that arrives whole + * is the one whose action list could not be parsed, and the dispatcher only + * answers it with the shape error. Refusing it instead told the model "User + * denied this action" (39/72/31 times per bench arm) for a call that ran + * nothing, and it never learned the real problem. `plan` writes a note into the model's own * conversation and nowhere else; there is no disk, no process and no network * on its code path, so no autonomy level has anything to say about it. */ -const INERT_TOOLS = new Set(["plan"]); +const INERT_TOOLS = new Set(["plan", "act"]); /** Tools that write, and are gated exactly like write_file. */ const WRITING_TOOLS = new Set(["write_file", "edit_file"]); diff --git a/src/background.ts b/src/background.ts index 7348a71..9c41389 100644 --- a/src/background.ts +++ b/src/background.ts @@ -7,7 +7,7 @@ * timeout kills the server, or the model backgrounds it with `&` and the * shell's stdout pipe holds the tool open anyway. Watching a model try * `nohup`, `setsid`, `disown` and `sleep 5 &&` in four consecutive steps is - * what makes the case. Factory's Terminal-Bench notes name "a controlled + * what makes the case. A competitor's Terminal-Bench notes name "a controlled * background-execution primitive" as one of their harness changes for the same * reason. * diff --git a/src/bar.ts b/src/bar.ts index 2b834d8..716f016 100644 --- a/src/bar.ts +++ b/src/bar.ts @@ -11,7 +11,7 @@ import { DISPUTE_HINT } from "./dispute.js"; import { judgePass } from "./evidence.js"; import { testsRealFor } from "./tests-real.js"; -import { runCommand } from "./run.js"; +import { runCommand, draftedShell } from "./run.js"; import { parseLcov, coverageFor, coverageCouldSpeak, unprovenIn, type Unproven } from "./coverage.js"; import { planMutations, applyMutation, negateComparison, type Mutation } from "./mutate.js"; import { proposeBar, type Detected } from "./detect.js"; @@ -2176,12 +2176,14 @@ export async function runCheck(check: Check, ctx: BarContext): Promise boolean; + /** `python3 -c "import name"` */ + hasPyModule: (name: string) => boolean; + /** Does this awk understand `{n}` interval expressions? (mawk 1.3.4-2020 does not.) */ + awkIntervals: () => boolean; +}; + +export type LintCtx = { + /** The untouched project. Without one, the rules that read the tree are skipped. */ + cwd?: string; + /** `readTree(cwd)` taken before the work began; read fresh from `cwd` when absent. */ + tree?: Tree; + task: string; + /** The shell the check will run under. Under bash, `kill %1` and `[[` are fine. */ + shell: "bash" | "sh"; + probes?: Partial; +}; + +const cache = new Map(); +function probeOnce(key: string, f: () => boolean): boolean { + const hit = cache.get(key); + if (hit !== undefined) return hit; + let v = false; + try { + v = f(); + } catch { + v = false; + } + cache.set(key, v); + return v; +} + +export const realProbes: Probes = { + hasCommand: (n) => /^[\w.+-]+$/.test(n) && probeOnce(`cmd:${n}`, () => (execFileSync("sh", ["-c", `command -v ${n}`], { stdio: "ignore", timeout: 2_000 }), true)), + hasPyModule: (m) => /^\w+$/.test(m) && probeOnce(`py:${m}`, () => (execFileSync("python3", ["-c", `import ${m}`], { stdio: "ignore", timeout: 4_000 }), true)), + awkIntervals: () => + probeOnce("awk{n}", () => execFileSync("awk", ["BEGIN{print match(\"1.00\",/[0-9]{2}$/)}"], { timeout: 2_000 }).toString().trim() !== "0"), +}; + +// ---------------------------------------------------------------- the untouched tree + +type Sig = { file: string; req: number; max: number; text: string }; +export type Tree = { files: Set; text: string; defs: Map; hasGit: boolean }; + +const TREE_MAX_FILES = 400; +const TREE_MAX_BYTES = 200_000; +const SKIP_DIRS = new Set([".git", ".maat", "node_modules", "__pycache__", ".venv", "venv", "dist", "build", "target"]); + +function splitTop(s: string): string[] { + const out: string[] = []; + let depth = 0; + let cur = ""; + let quote = ""; + for (let i = 0; i < s.length; i++) { + const ch = s[i]!; + if (quote) { + cur += ch; + if (ch === "\\") cur += s[++i] ?? ""; + else if (ch === quote) quote = ""; + continue; + } + if (ch === '"' || ch === "'") quote = ch; + if ("([{".includes(ch)) depth++; + if (")]}".includes(ch)) depth--; + if (ch === "," && depth === 0) { + out.push(cur); + cur = ""; + } else cur += ch; + } + if (cur.trim()) out.push(cur); + return out.map((x) => x.trim()).filter(Boolean); +} + +/** The module-level and nested `def`s of a python file with their positional arity. Methods are skipped. */ +export function pythonDefs(src: string, file: string): Map { + const out = new Map(); + for (const m of src.matchAll(/^[ \t]*(?:async\s+)?def\s+(\w+)\s*\(/gm)) { + let i = (m.index ?? 0) + m[0].length; + let depth = 1; + let quote = ""; + const start = i; + for (; i < src.length && depth > 0; i++) { + const ch = src[i]!; + if (quote) { + if (ch === "\\") i++; + else if (ch === quote) quote = ""; + } else if (ch === '"' || ch === "'") quote = ch; + else if (ch === "(" ) depth++; + else if (ch === ")") depth--; + } + if (depth !== 0) continue; + const params = splitTop(src.slice(start, i - 1)); + if (/^(self|cls)\b/.test(params[0] ?? "")) continue; + let req = 0; + let max = 0; + let star = false; + let kwOnly = false; + for (const p of params) { + if (p === "/") continue; + if (p === "*") { + kwOnly = true; + continue; + } + if (p.startsWith("**")) { + max = 99; + continue; + } + if (p.startsWith("*")) { + star = true; + kwOnly = true; + continue; + } + max += 1; + if (!kwOnly && !p.includes("=")) req += 1; + } + if (star) max = 99; + const sig: Sig = { file, req, max, text: `${m[1]}(${params.join(", ")})` }; + out.set(m[1]!, [...(out.get(m[1]!) ?? []), sig]); + } + return out; +} + +export function readTree(cwd: string): Tree { + const t: Tree = { files: new Set(), text: "", defs: new Map(), hasGit: existsSync(join(cwd, ".git")) }; + const bufs: string[] = []; + let n = 0; + const walk = (dir: string, rel: string, depth: number): void => { + if (depth > 6 || n >= TREE_MAX_FILES) return; + let names: string[]; + try { + names = readdirSync(dir); + } catch { + return; + } + for (const name of names) { + if (n >= TREE_MAX_FILES) return; + if (SKIP_DIRS.has(name)) continue; + const abs = join(dir, name); + const r = rel ? `${rel}/${name}` : name; + let st; + try { + st = statSync(abs); + } catch { + continue; + } + if (st.isDirectory()) { + walk(abs, r, depth + 1); + continue; + } + n++; + t.files.add(r); + t.files.add(name); + if (st.size > TREE_MAX_BYTES) continue; + let body = ""; + try { + body = readFileSync(abs, "utf8"); + } catch { + continue; + } + if (body.includes("\0")) continue; + bufs.push(body); + if (name.endsWith(".py")) for (const [k, v] of pythonDefs(body, r)) t.defs.set(k, [...(t.defs.get(k) ?? []), ...v]); + } + }; + walk(cwd, "", 0); + t.text = bufs.join("\n"); + return t; +} + +// ---------------------------------------------------------------- helpers + +/** Positional arguments in the text just after a call's `(`: top-level commas up to the closing paren. */ +export function countArgs(call: string): number { + let depth = 0; + let n = 0; + let seen = false; + let quote = ""; + for (let i = 0; i < call.length; i++) { + const ch = call[i]!; + if (quote) { + if (ch === "\\") { + i++; + continue; + } + if (ch === quote) quote = ""; + seen = true; + } else if (ch === '"' || ch === "'") quote = ch; + else if ("([{".includes(ch)) depth++; + else if (")]}".includes(ch)) { + if (depth === 0) break; + depth--; + } else if (ch === "," && depth === 0) n++; + else if (!/\s/.test(ch)) seen = true; + } + return seen ? n + 1 : 0; +} + +const MISSABLE = ["python", "pip", "pip3", "pytest", "rg", "fd", "bat", "tree", "lsof", "netstat", "column", "perl", "ruby", "nc", "bc", "ss"]; +const PY_MODULES = ["pytest", "numpy", "pandas", "requests", "yaml", "hypothesis", "flask"]; + +const plural = (n: number, w: string) => `${n} ${w}${n === 1 ? "" : "s"}`; + +/** + * Every rule the command breaks. Pure apart from the cheap probes in `ctx.probes`. + */ +export function lintAll(run: string, ctx: LintCtx): LintHit[] { + const hits: LintHit[] = []; + const add = (rule: string, why: string) => hits.push({ rule, why }); + const probes: Probes = { ...realProbes, ...ctx.probes }; + const cmd = run; + const low = ctx.task.toLowerCase(); + const tree = ctx.tree ?? (ctx.cwd ? readTree(ctx.cwd) : undefined); + const bash = ctx.shell === "bash"; + + // L1 tool or module not installed + if (/python3?\s+-m\s+pytest|\bpytest\b/.test(cmd) && !probes.hasPyModule("pytest") && !probes.hasCommand("pytest")) { + add("L1-pytest", "pytest is not installed here (use `python3 -m unittest`, or run the script directly)"); + } + const seenTools = new Set(); + for (const m of cmd.matchAll(/(?:^|[;&|(]\s*)(?:time\s+)?([A-Za-z_][\w.-]*)(?=\s|$)/g)) { + const w = m[1]!; + if (seenTools.has(w) || !MISSABLE.includes(w) || w === "pytest") continue; + seenTools.add(w); + if (tree?.files.has(w)) continue; + if (!probes.hasCommand(w)) add(`L1-${w}`, `\`${w}\` is not installed here`); + } + for (const mod of PY_MODULES) { + if (mod === "pytest") continue; + if (new RegExp(`(?:^|[^\\w.])(?:import|from)\\s+${mod}\\b|python3?\\s+-m\\s+${mod}\\b`).test(cmd) && !probes.hasPyModule(mod)) { + add(`L1-module:${mod}`, `the python module ${mod} is not installed here`); + } + } + // L2 git in a project with no repository (and a task that never mentions git) + if (tree && !tree.hasGit && /(?:^|[;&|(]\s*|\s)git\s/.test(cmd) && !/\bgit\b|\brepo\b|\bcommit\b/.test(low)) { + add("L2-git-norepo", "it runs git in a project that is not a git repository"); + } + // L3 job control under sh; pkill -f matching the check's own command line + if (!bash && /\bkill\s+%/.test(cmd)) add("L3-jobcontrol", "`kill %N` needs job control, which sh does not have"); + if (/(?:^|[;&|(]\s*)(?:fg|bg|jobs)\b/.test(cmd)) add("L3-jobcontrol", "`fg`/`bg`/`jobs` need an interactive shell"); + const pk = /pkill\s+-f\s+(['"]?)([^'"]+)\1/.exec(cmd); + if (pk && pk[2]!.trim() && cmd.replace(pk[0], "").includes(pk[2]!.trim())) { + add("L3-pkill-self", `\`pkill -f '${pk[2]!.trim()}'\` also matches the check's own shell and kills it`); + } + // L4 single-quoted substitution + if (/'[^']*\$[({][^']*'/.test(cmd) && !/awk|sed|perl|python3? -c '/.test(cmd)) { + add("L4-quoted-subst", "`$(…)` or `${…}` inside single quotes never expands"); + } + // L5 grep with an unquoted multi-word pattern + if (/grep(?:\s+-\S+)*\s+[A-Za-z]+(?:\s+[a-z]+){2,}\s*(?:$|[|;&)])/.test(cmd)) { + add("L5-grep-unquoted", "a multi-word grep pattern is not quoted, so the extra words are taken as file names"); + } + // L6 test -z on a quoted literal + if (/test\s+-[zn]\s+"[a-z]+ [^$"]*"/.test(cmd)) add("L6-test-literal", "`test -z \"command words\"` tests the literal text, not the command's output"); + // L7 regex dialects + if (/\bawk\b[^|;]*\\d/.test(cmd) || /\bgrep\s+(?:-[a-zA-Z]*\s+)*(?:-e\s+)?['"][^'"]*\\d/.test(cmd) || /\bgrep\s+-[a-zA-Z]*E[a-zA-Z]*[^|;]*\\d/.test(cmd)) { + add("L7-backslash-d", "`\\d` is not a POSIX regex class in grep/awk (use [0-9])"); + } + if (/\bawk\b[^|;]*\{\d+(?:,\d*)?\}/.test(cmd) && !probes.awkIntervals()) { + add("L7-mawk-interval", "this awk (mawk) does not support {n} interval expressions"); + } + for (const gm of cmd.matchAll(/\bgrep\s+((?:-\S+\s+)*)(['"])(.*?)\2/g)) { + const flags = gm[1]!; + const pat = gm[3]!; + if (!/-\w*[EP]/.test(flags) && /(?<=[\w\])*.])(?(); + for (const [name, sigs] of tree.defs) { + for (const cm of cmd.matchAll(new RegExp(String.raw`(? n >= s.req && n <= s.max)) continue; + if (reported.has(name)) break; + reported.add(name); + const s = sigs[0]!; + add(`L12-arity:${name}(${n})`, `it calls ${name} with ${plural(n, "argument")}; ${s.file} defines ${s.text}`); + break; + } + } + // L13 a string literal passed to a project function that is nowhere in the task or the tree + if (!/Error|raise|except|-q 'Traceback/.test(cmd)) { + outer: for (const name of tree.defs.keys()) { + for (const cm of cmd.matchAll(new RegExp(String.raw`(? draft the hidden checks and review the claim on a different + model than the worker (same endpoint unless --judge-url). + A separate judge does not share the worker's misreadings; + it raised verified-on-correct-work on every worker tested + with no wrong verifieds. Same as MAAT_JUDGE_MODEL. + --judge-url where the judge runs, e.g. grok-build://subscription, + opencode://subscription, or an OpenAI-style URL. --reasoning-checks effort for drafting checks and planning a mission only (single calls outside the work loop). Defaults to --reasoning. --reasoning-retry effort for every step after the checks refuse a claim. @@ -235,6 +262,10 @@ type Args = { maxTokens?: number; /** `--reasoning low`: a reasoning model's effort, sent only when set. */ reasoning?: string; + /** `--judge `: the model that drafts checks and reviews (MAAT_JUDGE_MODEL). */ + judge?: string; + /** `--judge-url `: where the judge runs (MAAT_JUDGE_URL). */ + judgeUrl?: string; /** `--reasoning-checks high`: effort for drafting checks and planning only. */ reasoningChecks?: string; /** `--reasoning-retry high`: effort after the checks refuse a claim. */ @@ -243,8 +274,17 @@ type Args = { steps?: number; /** `--batch`: the model's only tool is act — a plan and a list of actions per reply. */ batch?: boolean; - /** `--reveal-stuck`: see EngineConfig.revealOnStuck. Experimental. */ + /** `--reveal-stuck` (opt-in) / `--no-reveal`: see EngineConfig.revealOnStuck. */ revealStuck?: boolean; + /** `--review-advisory`: see EngineConfig.reviewAdvisory. */ + reviewAdvisory?: boolean; + /** `--signout`: see EngineConfig.signOut. */ + signout?: boolean; + /** `--arbiter-model` / `--dispute-votes`: see EngineConfig.dispute. */ + arbiterModel?: string; + disputeVotes?: number; + /** `--reference`: an independent reference check (src/reference.ts). */ + reference?: boolean; /** `--review [n]`: independent review of a verified claim, n votes (default 3). A label, not a gate. */ review?: number; attempts?: number; @@ -458,6 +498,24 @@ export function parseArgs(argv: string[], stored: StoredEndpoint = {}): Args { case "--reveal-stuck": out.revealStuck = true; break; + case "--no-reveal": + out.revealStuck = false; + break; + case "--review-advisory": + out.reviewAdvisory = true; + break; + case "--signout": + out.signout = true; + break; + case "--arbiter-model": + out.arbiterModel = next(); + break; + case "--dispute-votes": + out.disputeVotes = positiveInt("--dispute-votes", next()); + break; + case "--reference": + out.reference = true; + break; case "--review": { // Optional count: `--review` alone is three votes. const peek = argv[i + 1]; @@ -476,6 +534,18 @@ export function parseArgs(argv: string[], stored: StoredEndpoint = {}): Args { out.steps = n; break; } + case "--judge": + out.judge = next(); + break; + case "--judge-url": { + // Same words as --url (`grok`, `opencode`), and refused here rather + // than when the first check is drafted. + const judgeUrl = expandEndpointShorthand(next()); + const wrong = endpointProblem(judgeUrl); + if (wrong) throw new Error(`--judge-url: ${wrong}`); + out.judgeUrl = judgeUrl; + break; + } case "--reasoning-checks": case "--reasoning-retry": { const effort = next(); @@ -719,7 +789,10 @@ function engineFor(args: Args, session = false, extra: { files?: FileAccess } = retryReasoningEffort: args.reasoningRetry, maxSteps: args.steps, batch: args.batch === true, - ...(args.revealStuck ? { revealOnStuck: true } : {}), + ...(args.revealStuck === true ? { revealOnStuck: true } : args.revealStuck === false ? { revealOnStuck: false } : {}), + ...(args.reviewAdvisory || env("REVIEW_ADVISORY") === "1" ? { reviewAdvisory: true } : {}), + ...(args.signout ? { signOut: true } : {}), + ...(args.arbiterModel || args.disputeVotes ? { dispute: { model: args.arbiterModel, votes: args.disputeVotes } } : {}), ...(args.review ? { review: { votes: args.review, reasoningEffort: args.reasoningChecks ?? args.reasoning } } : {}), // MAAT_JUDGMENT=0: no judgment cases (benchmarks, where no person will ever rule). ...(env("JUDGMENT") === "0" ? { judgment: false } : {}), @@ -1100,45 +1173,118 @@ async function cmdMission(args: Args): Promise { async function autoDraft( engine: Engine, args: Args, - soFar?: { draft?: Draft }, + soFar?: { draft?: Draft; sealed?: boolean; late?: boolean }, ): Promise> { const none: ReturnType = { taskChecks: [], taskNotes: [] }; // Drafted, then read cold by a critic against the task text: a check that // invents or guesses is dropped (with a task quote), and a draft where // nothing runs the deliverable is asked for once more. See criteria.ts. - const r = await draftCriteriaCritiqued({ - commands: commandsHere(args.cwd), - lessons: new Judgments(args.cwd).lessons(), + const draftOnce = () => + draftCriteriaCritiqued({ + commands: commandsHere(args.cwd), + lessons: new Judgments(args.cwd).lessons(), + task: args.task ?? "", + scripts: projectScripts(args.cwd), + barChecks: (engine.cfg.bar?.checks ?? []).map((c) => c.name), + ...judgeTarget({ baseUrl: args.url, apiKey: args.key, model: args.model }), + cwd: args.cwd, + reasoningEffort: judgeEffort(args.reasoningChecks ?? args.reasoning), + latency: engine.askLatency, + // What is ready when a time budget stops the wait (RunOptions.criteriaSoFar). + onProgress: (d) => { + if (soFar) soFar.draft = d; + }, + }); + let r = await draftOnce(); + // One unlucky reply (an overloaded provider, JSON that does not parse) leaves + // a run with no checks, and so no verdict worth the name. Asked + // once more; a second failure is journalled as `no-checks`, not left silent. + if (!r.ok && !soFar?.sealed) { + process.stderr.write(`maat: criteria not drafted — ${r.error}; asking once more\n`); + engine.cfg.journal?.append("note", { kind: "draft-failed", text: `criteria draft failed once: ${r.error}` }); + r = await draftOnce(); + } + if (!r.ok) { + process.stderr.write(`maat: criteria not drafted — ${r.error}; running against the project bar only\n`); + engine.cfg.journal?.append("note", { kind: "no-checks", text: `no-checks: criteria could not be drafted after two tries — ${r.error}` }); + return none; + } + for (const line of r.critique) process.stderr.write(`maat: criteria review — ${line}\n`); + // A check the seal-time lint retired is never sealed; the record says which and why. + for (const d of r.lint ?? []) { + engine.cfg.journal?.append("note", { + kind: "check-lint-dropped", + text: `drafted check ${d.name} dropped by lint ${d.rule}: ${d.why}`, + name: d.name, + run: d.run, + rule: d.rule, + redraft: d.redraft, + }); + } + // A time budget already sealed what was ready (criteriaSoFar), and the work + // has begun in this folder. Sealing again would run the preflight a second + // time — and its cleanup removes every file that appeared while it ran, + // which by now includes the model's. + if (soFar?.sealed) return none; + return sealDraft(r.draft, args, soFar?.late); +} + +/** + * Snapshot the project now, before the first step, and write a reference + * check from it in the background (src/reference.ts). Null — said once — when + * there is no python3, the project is too large to copy, or no reference + * applies. + */ +function startReference(args: Args): Promise<{ check: Check; note: Record } | null> | undefined { + const here = commandsHere(args.cwd); + if (!here.present.includes("python3")) { + process.stderr.write("maat: no reference check — python3 is not installed here\n"); + return undefined; + } + const snapshot = snapshotProject(args.cwd); + if (!snapshot) { + process.stderr.write("maat: no reference check — the project is too large to snapshot\n"); + return undefined; + } + return draftReference({ task: args.task ?? "", - scripts: projectScripts(args.cwd), - barChecks: (engine.cfg.bar?.checks ?? []).map((c) => c.name), + snapshot, baseUrl: args.url, apiKey: args.key, model: args.model, cwd: args.cwd, reasoningEffort: args.reasoningChecks ?? args.reasoning, - // What is ready when a time budget stops the wait (RunOptions.criteriaSoFar). - onProgress: (d) => { - if (soFar) soFar.draft = d; - }, + }).then((r) => { + if (!r.ok) { + process.stderr.write(`maat: no reference check — ${r.why}\n`); + return null; + } + process.stderr.write(`maat: reference check ready — ${r.reason}\n`); + return { + check: r.check, + note: { snapshot: r.snapshot.hash, files: r.snapshot.files, reason: r.reason, source: r.source }, + }; + }).catch((e: unknown) => { + // A throw here (a file gone mid-copy) was an unhandled rejection whenever + // no claim was waiting on the reference — enough to end the process. + process.stderr.write(`maat: no reference check — ${e instanceof Error ? e.message : String(e)}\n`); + return null; }); - if (!r.ok) { - process.stderr.write(`maat: criteria not drafted — ${r.error}; running against the project bar only\n`); - return none; - } - for (const line of r.critique) process.stderr.write(`maat: criteria review — ${line}\n`); - return sealDraft(r.draft, args); } /** A draft as sealed, hidden checks, with the ones that break before any work dropped. */ -async function sealDraft(draft: Draft, args: Args): Promise> { +async function sealDraft(draft: Draft, args: Args, late = false): Promise> { // Hidden: the model wrote these, and a model shown its own exam makes the // work equal the check. It gets the names, and the output on failure. const sealed = taskChecksFrom(draft, { hidden: true }); // Headless, the checks' own side effects are cleaned up (src/leftovers.ts). - const beforeTry = listProject(args.cwd); - const broken = await preflightCriteria(sealed.taskChecks, { cwd: args.cwd }); - removeNew(args.cwd, beforeTry); + // A draft that joins at a claim (RunOptions.pendingCriteria, cut with no + // checks ready) lands after the work began: trying it in the live folder + // would be meaningless, and the cleanup after the try removes every file + // that appeared meanwhile, the model's included. It joins untried. + const beforeTry = late ? null : listProject(args.cwd); + const broken = late ? [] : await preflightCriteria(sealed.taskChecks, { cwd: args.cwd, stray: { task: args.task ?? "" } }); + if (!late) removeNew(args.cwd, beforeTry); const drop = new Set(broken.map((b) => b.name)); for (const b of broken) { process.stderr.write(`maat: dropped drafted criterion ${b.name} (${b.run}) — ${b.why}\n`); @@ -1148,7 +1294,7 @@ async function sealDraft(draft: Draft, args: Args): Promise { @@ -1240,10 +1386,16 @@ async function cmdRun(args: Args, ask = false): Promise { const sp = ev.spend; const cached = sp.cachedTokens > 0 ? ` (${sp.cachedTokens} cached)` : ""; const said = - ev.outcome === "verified" && ev.review && !ev.review.confirmed + ev.tier === "passed-checks" + ? passedChecksWords(ev.tierReason) + : ev.outcome === "verified" && ev.revealed?.length + ? "verified (checks shown after a repeat failure)" + : ev.outcome === "verified" && ev.review && !ev.review.confirmed ? "passed its checks, unconfirmed" : ev.outcome === "verified" && ev.review?.confirmed ? "verified, independently reviewed" + : ev.outcome === "verified" && ev.unreviewed + ? "passed its checks, unreviewed" : ev.outcome === "verified" && ev.selfChecked ? "passed its own checks" : ev.outcome === "unverified" && ev.checksDisagree?.length @@ -1334,15 +1486,25 @@ async function cmdRun(args: Args, ask = false): Promise { const { taskChecks, taskNotes } = criteriaFromArgs(args); // Drafted while the model starts reading: the engine seals them before the // first change or claim, so they still predate the work (see RunOptions). - const soFar: { draft?: Draft } = {}; + const soFar: { draft?: Draft; sealed?: boolean; late?: boolean } = {}; const pendingCriteria = args.autoCriteria && !ask ? autoDraft(engine, args, soFar) : undefined; const criteriaSoFar = pendingCriteria - ? () => (soFar.draft ? sealDraft(soFar.draft, args) : Promise.resolve({ taskChecks: [], taskNotes: [] })) + ? () => { + // Nothing reviewed yet: leave the draft running to join at the claim + // (it is then sealed without the preflight, see sealDraft). + if (soFar.draft?.checks.length) soFar.sealed = true; + else soFar.late = true; + return soFar.draft?.checks.length ? sealDraft(soFar.draft, args) : Promise.resolve({ taskChecks: [], taskNotes: [] }); + } : undefined; + const referenceCheck = args.reference && !ask ? startReference(args) : undefined; let undetermined = false; /** The turn's own verdict, from job_end. */ let outcome: string | undefined; - for await (const ev of engine.run(args.task, confirm, { ask, taskChecks, taskNotes, pendingCriteria, criteriaSoFar })) { + // How long a timed turn waits for its drafted checks; for tests and debugging. + const waitEnv = Number(env("CRITERIA_WAIT_MS")); + const criteriaWait = Number.isFinite(waitEnv) && waitEnv > 0 ? { criteriaWaitMs: waitEnv } : {}; + for await (const ev of engine.run(args.task, confirm, { ask, taskChecks, taskNotes, pendingCriteria, criteriaSoFar, referenceCheck, ...criteriaWait })) { emit(ev); if (ev.kind === "proof_exhausted" && ev.result.undetermined?.length) undetermined = true; // The sentence that explains an undetermined bar arrives as an error, so @@ -1976,6 +2138,9 @@ export async function main(argv = process.argv.slice(2)): Promise { process.stdout.write(USAGE + "\n"); return 0; } + // The judge is read where checks are drafted and claims reviewed (judge.ts). + if (args.judge) process.env.MAAT_JUDGE_MODEL = args.judge; + if (args.judgeUrl) process.env.MAAT_JUDGE_URL = args.judgeUrl; if (args.version) { process.stdout.write(VERSION + "\n"); return 0; diff --git a/src/criteria.ts b/src/criteria.ts index 4a0099b..e532ad6 100644 --- a/src/criteria.ts +++ b/src/criteria.ts @@ -22,9 +22,17 @@ * A sentence dressed as a check is worse than no check. */ import { execFileSync } from "node:child_process"; -import { askModel } from "./ask.js"; -import { runCommand } from "./run.js"; +import { openSync, readSync, closeSync, realpathSync, statSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { isAbsolute, relative } from "node:path"; +import { topLevel } from "./brief.js"; +import { namedInputs, profileLine } from "./inspect.js"; +import { askModel, type AskOptions } from "./ask.js"; +import { runCommand, draftedShell, bashPath } from "./run.js"; +import { lintAll, readTree, type LintCtx } from "./checklint.js"; import { diagnoseFailure } from "./bar.js"; +import { normalizeRequirements } from "./signout.js"; +import { evidenceTags } from "./tiers.js"; /** * `surface`: the critic read it as only looking (a file exists, a word @@ -32,7 +40,16 @@ import { diagnoseFailure } from "./bar.js"; * check alone cannot carry a "verified" once the ones that ran it are retired. */ export type DraftedCheck = { name: string; run: string; surface?: true }; -export type Draft = { checks: DraftedCheck[]; notes: string[] }; +export type Draft = { + checks: DraftedCheck[]; + notes: string[]; + /** + * The task's own stated requirements, verbatim quotes (src/signout.ts). Not + * checks: they are kept with the sealed criteria so an unattended turn can + * sign each one out before its claim is judged. + */ + requirements?: string[]; +}; /** Same bounds the drafter uses — applied again at `session:run`. */ export const CRITERIA_MAX_CHECKS = 4; @@ -60,7 +77,7 @@ export const CRITERIA_MAX_NOTE = 200; */ export function sanitizeCriteria(raw: unknown): Draft { if (!raw || typeof raw !== "object") return { checks: [], notes: [] }; - const o = raw as { checks?: unknown; notes?: unknown }; + const o = raw as { checks?: unknown; notes?: unknown; requirements?: unknown }; const checks: DraftedCheck[] = Array.isArray(o.checks) ? o.checks .filter( @@ -86,7 +103,8 @@ export function sanitizeCriteria(raw: unknown): Draft { .slice(0, CRITERIA_MAX_NOTES) .map((n) => n.trim().slice(0, CRITERIA_MAX_NOTE)) : []; - return { checks, notes }; + const requirements = normalizeRequirements(Array.isArray(o.requirements) ? o.requirements : undefined); + return { checks, notes, ...(requirements.length ? { requirements } : {}) }; } /** @@ -107,6 +125,7 @@ export function taskChecksFrom( hidden?: boolean; }[]; taskNotes: string[]; + requirements?: string[]; } { const drafted = sanitizeCriteria(raw); return { @@ -116,10 +135,12 @@ export function taskChecksFrom( run: c.run, timeoutMs: 120_000, expectExit: 0, - tags: c.surface ? ["task", "surface"] : ["task"], + // surface from the critic, value from the command itself (evidence.ts). + tags: evidenceTags(c.run, c.surface), ...(opts.hidden ? { hidden: true } : {}), })), taskNotes: drafted.notes, + ...(drafted.requirements ? { requirements: drafted.requirements } : {}), }; } @@ -174,16 +195,82 @@ export function checkSelfError(output: string): string | null { [/unknown primary or operator/i, "a tool rejected the check's options"], [/^usage: git /im, "git rejected the check's command"], [/not a git repository/i, "the check runs git outside a git repository"], + // A check that compares against `git show HEAD:` needs a commit that + // holds the file. Where the file is untracked, or the repository has no + // commit, git answers fatal and the check says nothing about the work: on + // the local suite 5 of the passing runs molt called unverified were + // refused by exactly this, in different checks. Only HEAD (or an empty + // revision left by a failed substitution) counts — a branch the work was + // meant to create and did not is the work's failure, not the check's. + [/fatal: path '[^']*' (?:does not exist|exists on disk, but not) in 'HEAD'/i, "the check reads a file from HEAD that no commit holds"], + [/fatal: (?:ambiguous argument '(?:HEAD)?': unknown revision|bad revision 'HEAD'|invalid object name 'HEAD')/i, "the check names a revision that does not exist"], [/\b(illegal|invalid|unrecognized) option\b/i, "a tool rejected the check's options"], [/illegal time specification|out of range or illegal time/i, "a tool rejected the check's date format"], [/syntax error near unexpected token|unexpected EOF while looking for/i, "the check's shell syntax is broken"], ]; for (const [re, why] of tool) if (re.test(output)) return why; + // Compile errors of the check's own program. `python3 -c` with a syntax + // error prints `File "", line 10` and `SyntaxError:` with no + // "Traceback" line at all, so the traceback rule below never saw it: a + // redact-secrets check with an unterminated f-string refused five correct + // solutions. jq and sqlite3 name their own compile errors. + // Only when the innermost frame is the inline program: `-c "import x"` + // over a deliverable with a syntax error names x.py last, and that is the + // work's fault. + const lastFrame = [...output.matchAll(/^\s*File "([^"]+)", line \d+/gm)].map((m) => m[1]).at(-1); + if ((lastFrame === "" || lastFrame === "") && /^(?:SyntaxError|IndentationError|TabError)\b/m.test(output)) return "the check's own program has a bug"; + if (/^jq: error: syntax error|^jq: \d+ compile errors?/m.test(output)) return "the check's jq program does not compile"; + if (/^(?:Parse error|Error): .*\n?.*syntax error/im.test(output) || /near ".*": syntax error/.test(output)) return "the check's SQL does not parse"; if (/Traceback \(most recent call last\)/.test(output)) { const frames = [...output.matchAll(/File "([^"]+)", line \d+/g)].map((m) => m[1]); const inline = frames.length > 0 && (frames.at(-1) === "" || frames.at(-1) === ""); if (inline && /^(NameError|SyntaxError|IndentationError|UnboundLocalError)\b/m.test(output)) return "the check's own program has a bug"; if (/got an unexpected keyword argument|no such group|invalid group reference/.test(output)) return "the check's own program misuses a library call"; + // Syntax-tree nodes belong to the standard library, not the deliverable: an + // attribute they lack (`FunctionDef.docstring`) is the check's mistake. + if (/^AttributeError: '(?:Module|FunctionDef|AsyncFunctionDef|ClassDef)' object has no attribute/m.test(output)) return "the check's own program misuses a library call"; + } + return null; +} + +/** Absolute prefixes of the system itself: tools, libraries, devices. Never the project's own files. */ +const SYSTEM_PATHS = ["/usr", "/bin", "/sbin", "/opt", "/lib", "/lib32", "/lib64", "/etc", "/dev", "/proc", "/sys", "/System", "/Library", "/Applications"]; +const TEMP_PATHS = ["/tmp", "/private/tmp", "/var/tmp", "/var/folders", "/private/var/folders"]; + +function under(p: string, root: string): boolean { + return p === root || p.startsWith(root.endsWith("/") ? root : `${root}/`); +} + +/** + * The first absolute path in a drafted command that points outside the + * project, or null. Allowed: anything under the working directory, the system + * temp directory (what `mktemp -d` returns), system locations (/usr/bin/env, + * /dev/null, /opt/homebrew/...), and a path the task text itself states. + * + * Seen in real runs: `python3 /wc.py` and `/home/user/data.csv` in checks + * drafted without any view of the project. They fail for ever, whatever the + * work does, and refuse correct work. Paths with a trailing slash are skipped + * (`awk '/x/'`, `sed 's/a/b/'` delimiters), as is anything glued to a word, + * `$VAR`, `./`, or `://` — those are relative paths, expansions and URLs — + * and so are closing tags (``) and cron steps. A deliberately missing path + * (`/nonexistent`) tests error handling and is allowed. + */ +export function strayPath(run: string, opts: { cwd: string; task?: string }): string | null { + let real = opts.cwd; + try { + real = realpathSync(opts.cwd); + } catch { + /* a cwd that is not there: compare as given */ + } + const roots = [opts.cwd, real, ...TEMP_PATHS, tmpdir(), ...SYSTEM_PATHS]; + for (const m of run.matchAll(/(?()*?\[\]{}$,=\\]*)/g)) { + const p = m[1]!; + if (p.endsWith("/")) continue; + // A path built to not exist is an error-handling test, not an invention. + if (/nonexist|no[-_]such|does[-_]?not[-_]?exist|missing/i.test(p)) continue; + if (roots.some((r) => under(p, r))) continue; + if (opts.task && opts.task.includes(p)) continue; + return p; } return null; } @@ -202,15 +289,28 @@ export async function preflightCriteria( * evidence the task was done. The engine uses this to say so. */ passed?: string[]; + /** + * Set for DRAFTED checks: a command that reaches for an absolute path + * outside the project (see strayPath) is reported broken without being + * run. The task text is where a path the person stated is allowed from. + */ + stray?: { task: string }; }, ): Promise { const broken: BrokenCriterion[] = []; for (const c of checks) { if (c.kind && c.kind !== "command") continue; if (!c.run) continue; + const stray = opts.stray ? strayPath(c.run, { cwd: opts.cwd, task: opts.stray.task }) : null; + if (stray) { + broken.push({ name: c.name, run: c.run, why: `it uses ${stray}, an absolute path outside the project (paths must be relative to the working directory)` }); + continue; + } try { const r = await runCommand(c.run, { cwd: opts.cwd, + // Drafted checks (stray set) run under the shell the bar will use. + shell: draftedShell({ hidden: !!opts.stray, tags: ["task"] }), timeoutMs: opts.timeoutMs ?? 5_000, maxBuffer: 1024 * 1024, signal: opts.signal, @@ -237,6 +337,59 @@ export async function preflightCriteria( return broken; } +/** Caps for the project view the drafter and critic are shown. */ +export const VIEW_MAX_ENTRIES = 40; +export const VIEW_MAX_LISTING = 2_000; +export const VIEW_MAX_PROFILE = 1_500; + +function firstTextLine(abs: string): string { + try { + const fd = openSync(abs, "r"); + try { + const buf = Buffer.alloc(400); + const n = readSync(fd, buf, 0, 400, 0); + const head = buf.subarray(0, n); + if (head.includes(0)) return ""; + return (head.toString("utf8").split("\n")[0] ?? "").trim().slice(0, 100); + } finally { + closeSync(fd); + } + } catch { + return ""; + } +} + +/** + * What the worker is shown and the drafter used not to be: where the work + * happens, what is in it, and what the files the task names look like. Without + * it a drafter invents `python3 /wc.py` and hand-computes expected values it + * could have read from the input. Bounded (entries, characters) because the + * drafter often runs on a slow model. Empty when there is no directory. + */ +export function projectView(task: string, cwd?: string): string { + if (!cwd) return ""; + const lines = [`Working directory: ${cwd}`]; + let listing = topLevel(cwd, VIEW_MAX_ENTRIES + 1).filter((n) => n !== ".maat/" && n !== ".maat").slice(0, VIEW_MAX_ENTRIES).join(" "); + if (listing.length > VIEW_MAX_LISTING) listing = `${listing.slice(0, VIEW_MAX_LISTING)}…`; + lines.push(`Project files (top level): ${listing || "(empty)"}`); + let profile = ""; + for (const abs of namedInputs(task, cwd, 6)) { + try { + if (!statSync(abs).isFile()) continue; + const rel = relative(cwd, abs); + let entry = profileLine(abs, isAbsolute(rel) || rel.startsWith("..") ? abs : rel).slice(0, 240); + const first = firstTextLine(abs); + if (first) entry += `\n first line: ${first}`; + if (profile.length + entry.length > VIEW_MAX_PROFILE) break; + profile += `${profile ? "\n" : ""} ${entry}`; + } catch { + /* unreadable: leave it out */ + } + } + if (profile) lines.push(`Input files the task names:`, profile); + return lines.join("\n"); +} + const SYSTEM = [ "You draft acceptance criteria for one coding task. You are not doing the task", "and you will not judge whether it was done — a person approves what you write", @@ -267,8 +420,16 @@ const SYSTEM = [ "provides instead: the stated threshold on the data that IS here (accuracy >= 0.62", "on the provided test split), the fit's error against the measured points, the new", "query's time against the original's. Never hard-code an answer you had to guess.", + "Never write an expected value you worked out yourself — a date, a total, a count,", + "an output line the task does not state. Your own arithmetic is where checks go", + "wrong. Expected values come only from the task text; otherwise test what the task", + "states as a property: the stated number of lines, each strictly after the start,", + "the stated format, totals that add up, the same answer from two equivalent inputs.", "Checks must leave nothing behind: build and write into $(mktemp -d), never into", "a directory the task names.", + "Every path is relative to the working directory shown below. Never write an", + "absolute path outside it (no /wc.py, no /home/..) except $(mktemp -d) and system", + "tools; a check with one is dropped. Read expected values from the input files.", "", "notes are for anything a command cannot decide — how something looks, reads, or", "feels. They are recorded on the receipt as stated intent and never reported as", @@ -347,6 +508,60 @@ function drafted( }; } +const EDGE_SYSTEM = [ + "You review a task given to a coding agent BEFORE any work starts. You see only the task text and the files that exist.", + "Find the places where two reasonable readings of the task would make a correct-looking deliverable produce DIFFERENT OUTPUT on the same input.", + "List up to 6 probes. A probe is one concrete literal input (a value, a line, a date, a number), the two readings, and what each outputs for it.", + "Prefer boundary cases: off-by-one in what counts, inclusive/exclusive ranges, rounding and formatting, whole token vs substring, how two rules combine,", + "empty or malformed input, ordering and ties, what counts as a category. Only real divergence; no generic engineering concerns.", + "Quote the task words each reading rests on. Fewer is better than padding; an empty list is fine.", + 'Reply with JSON only: {"probes":[{"input":"...","readingA":"...","outputA":"...","readingB":"...","outputB":"...","quote":"..."}]}', +].join("\n"); + +/** + * Edge cases where the task reads two ways (MAAT_EDGE_CHECKS=1), asked of the + * judge before the draft. They are handed to the drafter as candidates, not + * asked of anyone: a check is written only where the task text settles the + * reading. Asked as questions they were 90% noise (Bet 3); as check targets + * they cost one check each and found every trap the models kept failing. + */ +async function edgeProbes(opts: Parameters[0], view: string | undefined): Promise { + const asked = await askModel({ + baseUrl: opts.baseUrl, + apiKey: opts.apiKey, + model: opts.model, + system: EDGE_SYSTEM, + prompt: [`Task: ${opts.task}`, ...(view ? ["", view] : [])].join("\n"), + cwd: opts.cwd, + what: "probing edge cases", + fetchFn: opts.fetchFn, + acpSpawn: opts.acpSpawn, + cliRun: opts.cliRun, + timeoutMs: opts.timeoutMs, + reasoningEffort: opts.reasoningEffort, + latency: opts.latency, + }).catch(() => null); + if (!asked?.ok) return ""; + let probes: { input?: unknown; readingA?: unknown; outputA?: unknown; readingB?: unknown; outputB?: unknown; quote?: unknown }[] = []; + try { + const m = asked.text.match(/\{[\s\S]*\}/); + const parsed = m ? (JSON.parse(m[0]) as { probes?: unknown }) : {}; + probes = Array.isArray(parsed.probes) ? (parsed.probes as typeof probes).slice(0, 6) : []; + } catch { + return ""; + } + const lines = probes + .filter((p) => typeof p.input === "string" && p.input) + .map((p, i) => `${i + 1}. input ${JSON.stringify(p.input)} — reading A (${String(p.readingA ?? "")}) gives ${String(p.outputA ?? "")}; reading B (${String(p.readingB ?? "")}) gives ${String(p.outputB ?? "")}. Task words: "${String(p.quote ?? "")}"`); + if (!lines.length) return ""; + return [ + "Edge cases where the task could be read two ways (from a review of the task text):", + ...lines, + "For each one the task text SETTLES (quote it), add a check that feeds that literal input and asserts the settled output.", + "Skip any the task text does not settle: a check must never pick a side the task leaves open.", + ].join("\n"); +} + export async function draftCriteria(opts: { task: string; scripts: string[]; @@ -359,11 +574,15 @@ export async function draftCriteria(opts: { fetchFn?: typeof fetch; /** How an ACP agent is spawned. Tests only; see `EngineConfig.acpSpawn`. */ acpSpawn?: typeof import("node:child_process").spawn; + /** How a subscription CLI is run for a pre-turn question (opencode). Tests only. */ + cliRun?: (cmd: string, args: string[], opts: object) => Promise<{ stdout: string }>; /** How long the HTTP question may wait for its answer; see askTimeoutMs. Tests only. */ timeoutMs?: number; /** Pause before re-asking after empty replies (EMPTY_DRAFT_DELAY_MS). Tests only. */ emptyRetryDelayMs?: number; reasoningEffort?: string; + /** Learns from completed asks (AskOptions.latency). */ + latency?: AskOptions["latency"]; /** What is installed, so a check never calls a command that is not (see commandsHere). */ commands?: { present: string[]; missing: string[] }; /** @@ -372,8 +591,10 @@ export async function draftCriteria(opts: { */ lessons?: string[]; }): Promise<{ ok: true; draft: Draft } | { ok: false; error: string }> { + const view = projectView(opts.task, opts.cwd); const context = [ `Task: ${opts.task}`, + ...(view ? ["", view] : []), "", `Scripts available: ${opts.scripts.length ? opts.scripts.join(", ") : "(none found)"}`, `The project already checks: ${opts.barChecks.length ? opts.barChecks.join(", ") : "(nothing)"}`, @@ -390,18 +611,22 @@ export async function draftCriteria(opts: { "this task.", ].join("\n"); + const edges = process.env.MAAT_EDGE_CHECKS === "1" ? await edgeProbes(opts, view) : ""; + const prompted = edges ? `${context}\n\n${edges}` : context; const asked = await askModel({ baseUrl: opts.baseUrl, apiKey: opts.apiKey, model: opts.model, system: SYSTEM, - prompt: context, + prompt: prompted, cwd: opts.cwd, what: "drafting criteria", fetchFn: opts.fetchFn, acpSpawn: opts.acpSpawn, + cliRun: opts.cliRun, timeoutMs: opts.timeoutMs, reasoningEffort: opts.reasoningEffort, + latency: opts.latency, }); if (!asked.ok) return asked; const first = drafted(asked.text, asked.cutOff); @@ -413,13 +638,15 @@ export async function draftCriteria(opts: { apiKey: opts.apiKey, model: opts.model, system: SYSTEM, - prompt: `${context}\n\nYour last reply could not be parsed as JSON. Reply with the JSON object only.`, + prompt: `${prompted}\n\nYour last reply could not be parsed as JSON. Reply with the JSON object only.`, cwd: opts.cwd, what: "drafting criteria", fetchFn: opts.fetchFn, acpSpawn: opts.acpSpawn, + cliRun: opts.cliRun, timeoutMs: opts.timeoutMs, reasoningEffort: opts.reasoningEffort, + latency: opts.latency, }); if (!again.ok) return first; const second = drafted(again.text, again.cutOff); @@ -440,8 +667,10 @@ export async function draftCriteria(opts: { what: "drafting criteria", fetchFn: opts.fetchFn, acpSpawn: opts.acpSpawn, + cliRun: opts.cliRun, timeoutMs: opts.timeoutMs, reasoningEffort: opts.reasoningEffort, + latency: opts.latency, }); if (!more.ok) break; const d = drafted(more.text, more.cutOff); @@ -495,9 +724,12 @@ export const CRITIC_SYSTEM = [ "that state it, verbatim, at most two. Usually the list is empty: list one only when", "you are sure a wrong result would pass every check.", "", + "Finally list the task's stated requirements, each a short verbatim quote (at most ten):", + "every concrete thing the finished work must do or contain, in the task's own words.", + "", "Reply with JSON only:", '{"checks":[{"name":"...","verdict":"invents|guesses|surface|runs","quote":"verbatim task words, or empty"}],', - ' "uncovered":["verbatim task words"]}', + ' "uncovered":["verbatim task words"], "requirements":["verbatim task words"]}', ].join("\n"); export type Critique = { @@ -511,6 +743,12 @@ export type Critique = { * hold a requirement nobody stated. */ uncovered: string[]; + /** + * Every stated requirement the critic quoted, plus the quotes of its drops + * and uncovered findings — all string-matched against the task, deduplicated, + * at most REQUIREMENTS_MAX (src/signout.ts). + */ + requirements: string[]; }; function norm(s: string): string { @@ -553,7 +791,11 @@ export function applyCritique(draft: Draft, reply: string, task: string): Critiq .map((q) => String(q ?? "").trim()) .filter((q) => norm(q).length >= 8 && t.includes(norm(q))) .slice(0, CRITIC_MAX_UNCOVERED); - return { kept, dropped, surfaceOnly: runs === 0 && kept.length > 0, uncovered }; + const stated = (Array.isArray((raw as { requirements?: unknown }).requirements) ? (raw as { requirements: unknown[] }).requirements : []) + .map((q) => String(q ?? "").trim()) + .filter((q) => norm(q).length >= 8 && t.includes(norm(q))); + const requirements = normalizeRequirements([...stated, ...uncovered, ...dropped.map((d) => d.quote).filter((q) => norm(q).length >= 8)]); + return { kept, dropped, surfaceOnly: runs === 0 && kept.length > 0, uncovered, requirements }; } /** How long the uncovered-requirement step may add to drafting. */ @@ -574,32 +816,110 @@ export async function draftCriteriaCritiqued( /** Bound on the step that adds checks for uncovered requirements (COVER_MAX_MS). */ coverMaxMs?: number; }, +): Promise<{ ok: true; draft: Draft; critique: string[]; lint?: LintDrop[] } | { ok: false; error: string }> { + // The critic's list of stated requirements rides on whatever draft comes + // back, however many ways the function below returns. + const sink: Sink = { requirements: [], lint: [] }; + const withReqs = (d: Draft): Draft => (sink.requirements.length ? { ...d, requirements: sink.requirements } : d); + const r = await critiqued({ ...opts, onProgress: opts.onProgress && ((d) => opts.onProgress!(withReqs(d))) }, sink); + return r.ok ? { ...r, draft: withReqs(r.draft), ...(sink.lint.length ? { lint: sink.lint } : {}) } : r; +} + +/** A drafted check the seal-time lint retired (src/checklint.ts), for the journal. */ +export type LintDrop = { name: string; run: string; rule: string; why: string; redraft: boolean }; +type Sink = { requirements: string[]; lint: LintDrop[] }; + +async function critiqued( + opts: Parameters[0] & { + /** Called with the best draft so far — the reviewed checks once there are any. */ + onProgress?: (d: Draft) => void; + /** Bound on the step that adds checks for uncovered requirements (COVER_MAX_MS). */ + coverMaxMs?: number; + }, + sink: Sink, ): Promise<{ ok: true; draft: Draft; critique: string[] } | { ok: false; error: string }> { const fix = (d: Draft): Draft => opts.commands ? { ...d, checks: d.checks.map((c) => ({ ...c, run: fixInterpreters(c.run, opts.commands!) })) } : d; + const view = projectView(opts.task, opts.cwd); + // The project as it is now, before the model has changed anything. + const lintCtx: LintCtx = { + task: opts.task, + shell: bashPath() ? "bash" : "sh", + ...(opts.cwd ? { cwd: opts.cwd, tree: readTree(opts.cwd) } : {}), + }; + const said: string[] = []; + /** Split a draft's checks into the ones that pass the lint and the ones that do not. Drops are recorded, never silent. */ + const lintSplit = (checks: DraftedCheck[], redraft: boolean): { ok: DraftedCheck[]; bad: LintDrop[] } => { + const ok: DraftedCheck[] = []; + const bad: LintDrop[] = []; + for (const c of checks) { + // Off unless MAAT_CHECK_LINT=1: in the v13 paired run the lint left fewer, shallower + // checks sealed (the redraft lands after the seal), which cost passes and let wrong work through. + const hit = process.env.MAAT_CHECK_LINT === "1" ? lintAll(c.run, lintCtx)[0] : undefined; + if (!hit) ok.push(c); + else bad.push({ name: c.name, run: c.run, rule: hit.rule, why: hit.why, redraft }); + } + return { ok, bad }; + }; + const drop = (bad: LintDrop[], where: string) => { + for (const b of bad) { + sink.lint.push(b); + said.push(`dropped ${b.name} (${b.run}) ${where}: ${b.why} [${b.rule}]`); + } + }; const drafted1 = await draftCriteria(opts); - const first = drafted1.ok ? { ...drafted1, draft: fix(drafted1.draft) } : drafted1; - if (!first.ok || !first.draft.checks.length) return first.ok ? { ...first, critique: [] } : first; + let first = drafted1.ok ? { ...drafted1, draft: fix(drafted1.draft) } : drafted1; + if (first.ok && first.draft.checks.length) { + const l1 = lintSplit(first.draft.checks, false); + if (l1.bad.length) { + // Sent back once with the reasons. What passes the lint is already ready + // to seal while that second ask runs. + first = { ...first, draft: { ...first.draft, checks: l1.ok } }; + if (l1.ok.length) opts.onProgress?.(first.draft); + const findings = l1.bad.map((b) => `"${b.name}" was dropped: ${b.why}.`).join(" "); + const again = await draftCriteria({ + ...opts, + task: `${opts.task}\n\n(Review of an earlier draft of the checks: ${findings} Draft replacement checks that avoid this. Checks run under bash.)`, + }); + const have = new Set(l1.ok.map((c) => c.name)); + const ran = new Set(l1.ok.map((c) => c.run)); + const l2 = again.ok ? lintSplit(fix(again.draft).checks, true) : { ok: [] as DraftedCheck[], bad: [] as LintDrop[] }; + drop(l1.bad, "before sealing"); + drop(l2.bad, "from the redraft"); + const added = l2.ok.filter((c) => !have.has(c.name) && !ran.has(c.run)).slice(0, Math.max(0, CRITERIA_MAX_CHECKS - l1.ok.length)); + if (added.length) said.push(`redrafted for the dropped checks: ${added.map((c) => c.name).join(", ")}`); + first = { ...first, draft: { ...first.draft, checks: [...l1.ok, ...added], notes: first.draft.notes.length ? first.draft.notes : again.ok ? again.draft.notes : [] } }; + } + } + if (!first.ok || !first.draft.checks.length) return first.ok ? { ...first, critique: said } : first; + const lintCover = (checks: DraftedCheck[]): DraftedCheck[] => { + const l = lintSplit(checks, false); + drop(l.bad, "from the cover draft"); + return l.ok; + }; const critic = async (d: Draft) => { const asked = await askModel({ baseUrl: opts.baseUrl, apiKey: opts.apiKey, model: opts.model, system: CRITIC_SYSTEM, - prompt: `TASK TEXT:\n${opts.task}\n\nCHECKS:\n${d.checks.map((c) => `- ${c.name}: ${c.run}`).join("\n")}`, + prompt: `TASK TEXT:\n${opts.task}\n\n${view ? `${view}\n\n` : ""}CHECKS:\n${d.checks.map((c) => `- ${c.name}: ${c.run}`).join("\n")}`, cwd: opts.cwd, what: "reviewing the drafted checks", fetchFn: opts.fetchFn, acpSpawn: opts.acpSpawn, + cliRun: opts.cliRun, timeoutMs: opts.timeoutMs, reasoningEffort: opts.reasoningEffort, + latency: opts.latency, }); return asked.ok ? applyCritique(d, asked.text, opts.task) : null; }; - const said: string[] = []; + opts.onProgress?.(first.draft); const c1 = await critic(first.draft); - if (!c1) return { ...first, critique: ["the checks could not be reviewed; sealed as drafted"] }; + if (!c1) return { ...first, critique: [...said, "the checks could not be reviewed; sealed as drafted"] }; + sink.requirements = c1.requirements; if (c1.kept.length) opts.onProgress?.({ ...first.draft, checks: c1.kept }); const runOf = (name: string) => first.draft.checks.find((c) => c.name === name)?.run ?? ""; for (const d of c1.dropped) said.push(`dropped ${d.name} (${runOf(d.name)}): it ${d.verdict === "invents" ? "demands what the task does not state" : "guesses an answer the task does not give"} — "${d.quote}"`); @@ -619,7 +939,7 @@ export async function draftCriteriaCritiqued( `Nothing tests these stated requirements yet: ${c1.uncovered.map((q) => `"${q}"`).join("; ")}. ` + `Draft checks for these requirements only.)`, }); - const extra = cover.ok ? fix(cover.draft).checks : []; + const extra = cover.ok ? lintCover(fix(cover.draft).checks) : []; return extra.length ? await critic({ checks: extra, notes: [] }) : null; }; let coverTimer: NodeJS.Timeout | undefined; @@ -646,7 +966,12 @@ export async function draftCriteriaCritiqued( c1.surfaceOnly || !c1.kept.length ? "No check runs the deliverable on an input and tests what it produces. Add one." : "", ].filter(Boolean).join(" "); const drafted2 = await draftCriteria({ ...opts, task: `${opts.task}\n\n(Review of an earlier draft of the checks: ${findings})` }); - const second = drafted2.ok ? { ...drafted2, draft: fix(drafted2.draft) } : drafted2; + let second = drafted2.ok ? { ...drafted2, draft: fix(drafted2.draft) } : drafted2; + if (second.ok) { + const l = lintSplit(second.draft.checks, false); + drop(l.bad, "from the redraft"); + second = { ...second, draft: { ...second.draft, checks: l.ok } }; + } if (second.ok && second.draft.checks.length) { const c2 = await critic(second.draft); const kept = c2 ? c2.kept : second.draft.checks; diff --git a/src/dispute.ts b/src/dispute.ts index 7169767..6b2db37 100644 --- a/src/dispute.ts +++ b/src/dispute.ts @@ -74,7 +74,7 @@ export const ARBITER_SYSTEM = [ ].join("\n"); /** - * Rule on one dispute with `votes` independent asks. Upheld only when more + * Rule on one dispute with `votes` independent asks (one by default). Upheld only when more * than half say the check contradicts the task with a quote that really is in * the task text. Null when no ask could be made at all. */ @@ -86,7 +86,7 @@ export async function arbitrate(opts: { votes?: number; ask: Omit; }): Promise { - const votes = Math.max(1, opts.votes ?? 3); + const votes = Math.max(1, opts.votes ?? 1); const prompt = `TASK TEXT:\n${opts.task}\n\nCHECK "${opts.check.name}":\n${opts.check.run}\n\n` + `CHECK OUTPUT (it failed):\n${opts.output.slice(0, 4000)}\n\n` + diff --git a/src/endpoint.ts b/src/endpoint.ts index ff2b31f..efbfb93 100644 --- a/src/endpoint.ts +++ b/src/endpoint.ts @@ -31,10 +31,13 @@ const REMOVED_SHORTHANDS = [ "gemini-cli", ] as const; -/** The one subscription CLI molt still spawns: Grok Build over ACP. */ +/** A subscription CLI molt still spawns: Grok Build over ACP. */ const GROK_BUILD_SCHEME = "grok-build://"; +/** OpenCode's own subscription (ACP), the other subscription CLI molt spawns. */ +const OPENCODE_SCHEME = "opencode://"; export const GROK_BUILD_URL = `${GROK_BUILD_SCHEME}subscription`; +export const OPENCODE_URL = `${OPENCODE_SCHEME}subscription`; /** * What someone types, and the sentinel it stands for. @@ -46,6 +49,7 @@ export const GROK_BUILD_URL = `${GROK_BUILD_SCHEME}subscription`; const SHORTHAND: Readonly> = { "grok-build": GROK_BUILD_URL, grok: GROK_BUILD_URL, + opencode: OPENCODE_URL, }; /** Why a removed subscription backend cannot be used. */ @@ -107,7 +111,7 @@ export function endpointProblem(baseUrl: string): string | null { `/login, or 'grok-build' to run your own logged-in Grok Build CLI.` ); } - const allowed = ["http:", "https:", GROK_BUILD_SCHEME.replace(/\/\/$/u, "")]; + const allowed = ["http:", "https:", ...[GROK_BUILD_SCHEME, OPENCODE_SCHEME].map((s) => s.replace(/\/\/$/u, ""))]; if (!allowed.includes(parsed.protocol)) { return ( `'${url}' uses the scheme '${parsed.protocol.replace(":", "")}', which Maat cannot ` + diff --git a/src/engine.ts b/src/engine.ts index 3f6cd9a..8f982bd 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -21,10 +21,13 @@ import { runCommand } from "./run.js"; import { describeStart, listBackground, startBackground, stopBackground } from "./background.js"; import { reviewClaim, type Review } from "./review.js"; +import { judgeEffort, judgeTarget } from "./judge.js"; +import { tierOf } from "./tiers.js"; import { arbitrate, parseDisputes, type Ruling } from "./dispute.js"; import { createHash } from "node:crypto"; +import { execFileSync } from "node:child_process"; import { mkdirSync, readFileSync, writeFileSync, existsSync, statSync } from "node:fs"; -import { excludeMoltFromGit, listProject, removeNew, type ProjectListing } from "./leftovers.js"; +import { excludeMoltFromGit, listProject, removeNew, unnamedNewFiles, type ProjectListing } from "./leftovers.js"; import { inspectDir, inspectFile, namedInputs, profileLine } from "./inspect.js"; import { basename, dirname, resolve, relative, isAbsolute, join } from "node:path"; import type { ArchiveLike } from "./archive.js"; @@ -38,7 +41,7 @@ import { type BarContext, barPath, } from "./bar.js"; -import { preflightCriteria } from "./criteria.js"; +import { checkSelfError, preflightCriteria } from "./criteria.js"; import { AUTONOMY_SUMMARY, DEFAULT_AUTONOMY, @@ -48,9 +51,9 @@ import { isReadOnlyCommand, } from "./autonomy.js"; import { errorText } from "./format.js"; -import { narratedCallIn, narratedCallNudge } from "./narrated.js"; +import { MOLT_TOOL_NAMES, narratedCallIn, narratedCallNudge } from "./narrated.js"; import { redact } from "./redact.js"; -import { Watchdog, envFirstByteMs, firstByteMs, probeError, probeSignal, requestIdleMs, waited } from "./watchdog.js"; +import { Watchdog, envFirstByteMs, firstByteMs, probeError, probeSignal, requestIdleMs, waited, localSpeed, LatencyLearner } from "./watchdog.js"; import { SKIP_DIRS, WALK_DEADLINE_MS, @@ -97,9 +100,16 @@ import { Integrity } from "./integrity.js"; import { authHeaders, isSelfHosted, + selfHostedThinking, openRouterProvider, } from "./providers.js"; import { Receipts } from "./receipts.js"; -import { readStream, type StreamAccumulator, type Usage } from "./stream.js"; +import { normalizeRequirements, signOut, signOutMessage, type SignOut } from "./signout.js"; +import { gitPathsStaged, touches } from "./gitbend.js"; +import { REFERENCE_CHECK_NAME, REFERENCE_SELF_ERROR } from "./reference.js"; +import { argumentProblems } from "./toolargs.js"; +import { takeTurn } from "./localgate.js"; +import { parseLenient } from "./lenient-json.js"; +import { LONG_RATE_LIMIT_MS, longQuotaText, rateLimitResetAt, untilText, providerErrorText, normalizeMessage, readStream, transientProviderError, type ProviderError, type StreamAccumulator, type Usage } from "./stream.js"; import { Fragments, SafeStream } from "./live.js"; import { Transcript, toolDetail } from "./transcript.js"; import { acpAgentFor, acpHealth, acpModels, AcpSession, isAcp } from "./acp.js"; @@ -477,6 +487,12 @@ export function resultBudgetBytes(window: number, scale: number, cap = READ_MAX_ } export const MAX_STEPS = 32; export const MAX_PROOF_ATTEMPTS = 4; +/** + * The longest one sealed check may run once the turn's clock has run out. The + * runner kills the process a minute after `--for`, and the bar is what the + * verdict is made of: a check that needs more is retired as timed out. + */ +export const DEADLINE_CHECK_CAP_MS = 40_000; /** * How many empty assistant turns to ask through before treating one as a * claim of completion. @@ -557,6 +573,19 @@ export const CACHE_LOST_STREAK = 3; export const NETWORK_RETRIES = 3; export const NETWORK_BACKOFF_MS = [500, 2_000, 5_000]; +/** + * Retries for a provider that said it is overloaded or rate-limited. + * + * A separate, longer budget, because the network policy above is sized for a + * blip and an overload is not a blip: on a free OpenRouter model half of all + * requests came back "Upstream error: Service temporarily overloaded", in + * runs that last seconds to a minute. Four tries inside eight seconds lose a + * thirty-step task to the provider about half the time; this waits it out + * for a little over two minutes before giving up. + */ +export const OVERLOAD_RETRIES = 8; +export const OVERLOAD_BACKOFF_MS = [2_000, 4_000, 8_000, 15_000, 20_000, 30_000, 30_000, 30_000]; + /** * How long a provider asked us to wait, from `Retry-After`. * @@ -574,6 +603,29 @@ function retryAfterMs(res: Response): number | undefined { return Math.min(ms, 30_000); } +/** + * A provider error that arrived inside a 200 response (see StreamChunk.error), + * as the request failure it is. Whatever partial message came with it is + * dropped by the caller: the retry replays the request from the start. + */ +function providerFailure(e: ProviderError): { text: string; why: string; retryable: boolean; overload?: boolean } { + const resetAt = rateLimitResetAt({ error: e }); + if (resetAt !== undefined && resetAt - Date.now() > LONG_RATE_LIMIT_MS) { + return { + text: `the provider's rate limit is reached ${untilText(resetAt)} — ${providerErrorText(e)}`, + why: "The provider will not take requests again until its limit resets; waiting inside a turn cannot get past it.", + retryable: false, + }; + } + const transient = transientProviderError(e); + return { + text: providerErrorText(e), + why: "The provider reported an error instead of an answer.", + retryable: transient, + ...(transient ? { overload: true } : {}), + }; +} + /** Wait, unless the turn is cancelled first — then return immediately. */ function sleepUnlessAborted(ms: number, signal: AbortSignal): Promise { return new Promise((resolve) => { @@ -763,7 +815,7 @@ export const MAX_BASH_TIMEOUT_MS = 30 * 60_000; * A plan, as the model reads it back. * * Done steps are crossed off, the current one is marked, the rest are - * pending. The marking exploits the same recency bias Factory's write-up + * pending. The marking exploits the same recency bias a competitor's write-up * describes: the last thing in the result is the next thing to do. */ export function renderPlan(steps: string[], current: number): string { @@ -851,31 +903,66 @@ type ActSub = { id: string; name: string; rawArgs: string; label: string }; * one (a malformed act falls through and is reported as malformed). */ export function expandAct(call: { id: string; function?: { name?: string; arguments?: string } }): - | { analysis: string; subs: ActSub[] } + | { analysis: string; subs: ActSub[]; unusable: number } | null { if (call.function?.name !== "act") return null; let a: { analysis?: unknown; plan?: unknown; actions?: unknown }; try { - a = JSON.parse(call.function.arguments || "{}"); + a = parseLenient(call.function.arguments || "{}") as typeof a; } catch { return null; } + if (!a || typeof a !== "object") return null; const analysis = [a.analysis, a.plan].filter((x) => typeof x === "string" && x.trim()).join("\n\n"); - const list = Array.isArray(a.actions) ? a.actions : []; + // Models drift from the schema in a few common, unambiguous ways, and on a + // long write the drift cost the write itself: Nemotron sent an act whose + // actions were not the {tool, args} array, nothing in it was read as an + // action, and an act with no actions is the "finished" signal — so a reply + // writing clean.csv became a claim with nothing on disk. Read the shapes + // whose meaning is not in doubt: actions as a JSON string, one action + // object instead of a list, name/function/tool_name for tool, and + // arguments/input/parameters (object or JSON string) for args. + let raw: unknown = a.actions; + if (typeof raw === "string") { + try { + raw = parseLenient(raw); + } catch { + /* left as is: counted as unusable below */ + } + } + const list = Array.isArray(raw) ? raw : raw && typeof raw === "object" ? [raw] : []; const subs: ActSub[] = []; + let unusable = typeof raw === "string" && raw.trim() !== "" && raw.trim() !== "[]" ? 1 : 0; list.forEach((x, k) => { - if (!x || typeof x !== "object") return; - const o = x as { tool?: unknown; args?: unknown }; - if (typeof o.tool !== "string") return; - const args = o.args && typeof o.args === "object" ? (o.args as Record) : {}; + if (!x || typeof x !== "object") { + unusable += 1; + return; + } + const o = x as Record; + const fn = o.function && typeof o.function === "object" ? (o.function as Record) : undefined; + const tool = [o.tool, o.name, o.tool_name, fn?.name, typeof o.function === "string" ? o.function : undefined].find((t) => typeof t === "string" && t.trim()) as string | undefined; + if (!tool) { + unusable += 1; + return; + } + let given: unknown = [o.args, o.arguments, o.input, o.parameters, fn?.arguments].find((v) => v !== undefined); + if (typeof given === "string") { + try { + given = parseLenient(given); + } catch { + unusable += 1; + return; + } + } + const args = given && typeof given === "object" && !Array.isArray(given) ? (given as Record) : {}; subs.push({ id: `${call.id}~${k}`, - name: o.tool, + name: tool, rawArgs: JSON.stringify(args), - label: `${o.tool} ${toolDetail(o.tool, args)}`.slice(0, 160), + label: `${tool} ${toolDetail(tool, args)}`.slice(0, 160), }); }); - return { analysis, subs }; + return { analysis, subs, unusable }; } const SECRET_ENV = [ @@ -929,15 +1016,26 @@ export type RunOptions = { * drafter sees only the task text, never the transcript. Drafting took a * median 17 s of every local task, all of it before the first step. */ - pendingCriteria?: Promise<{ taskChecks: Check[]; taskNotes: string[] }>; + pendingCriteria?: Promise<{ taskChecks: Check[]; taskNotes: string[]; requirements?: string[] }>; /** * What of `pendingCriteria` is ready now — the checks that passed review so * far. With a time budget the wait for the full draft is bounded * (criteriaWaitMs); when it runs out these are sealed instead. */ - criteriaSoFar?: () => Promise<{ taskChecks: Check[]; taskNotes: string[] }>; + criteriaSoFar?: () => Promise<{ taskChecks: Check[]; taskNotes: string[]; requirements?: string[] }>; /** Overrides criteriaWaitMs(budget). Tests only. */ criteriaWaitMs?: number; + /** + * An independent reference check, still being written (src/reference.ts). + * Built from the task text and a snapshot of the project taken before the + * first step, so it may join the sealed checks at a claim without the work + * having had any say in it. Null when none applies. + */ + referenceCheck?: Promise<{ check: Check; note: Record } | null>; + /** Overrides referenceWaitMs. Tests only. */ + referenceWaitMs?: number; + /** Overrides how long a claim waits for checks still drafting after the time budget's cut. Tests only. */ + lateCriteriaWaitMs?: number; /** * Criteria stated in words rather than as commands. * @@ -948,6 +1046,13 @@ export type RunOptions = { * cannot make a turn succeed or fail. */ taskNotes?: string[]; + /** + * The task's stated requirements, verbatim (src/signout.ts), kept with the + * sealed criteria. Unattended, the first claim is held once while each is + * paired with the commands the model ran for it. Empty or absent: nothing + * happens. + */ + requirements?: string[]; }; /** @@ -1046,6 +1151,42 @@ export function criteriaWaitMs(budgetMs: number): number | undefined { return Math.min(120_000, Math.max(45_000, Math.round(budgetMs / 10))); } +/** + * How long a claim waits for a reference check still being written: up to five + * minutes with no time budget, else a quarter of what is left, at most three. + * Not ready by then, the claim is judged without it, and the next claim looks + * again. + */ +export function referenceWaitMs(timeLeftMs: number | undefined): number { + if (timeLeftMs === undefined) return 300_000; + return Math.max(0, Math.min(180_000, Math.round(timeLeftMs / 4))); +} + +/** + * How long a claim waits for drafted checks that were cut with none ready: + * they are the only thing that can make the claim verifiable, so they get most + * of what is left (60%, leaving the rest for running the bar), at most two + * minutes; with no time budget, two minutes. + */ +export function lateCriteriaWaitMs(timeLeftMs: number | undefined): number { + if (timeLeftMs === undefined) return 120_000; + return Math.max(0, Math.min(120_000, Math.round(timeLeftMs * 0.6))); +} + +/** + * A hash of everything the drafter reads: the task text and the project's file + * listing before the first step. Journalled at turn start and again when late + * checks join, so "written without sight of the work" is two equal lines in + * the record rather than a promise. + */ +export function draftInputsHash(task: string, listing: ProjectListing | null): string { + const h = createHash("sha256"); + h.update(task, "utf8"); + h.update("\0files\0" + [...(listing?.files ?? [])].sort().join("\n")); + h.update("\0dirs\0" + [...(listing?.dirs ?? [])].sort().join("\n")); + return h.digest("hex").slice(0, 16); +} + export function withTaskChecks(bar: Bar | null | undefined, task: Check[]): Bar | null { if (!task.length) return bar ?? null; const base = bar ?? { version: 1 as const, checks: [] }; @@ -1153,10 +1294,38 @@ export type EngineConfig = { * When the turn's own drafted (hidden) checks fail the same way twice, show * the model those checks' commands once and let it go on — fix the work or * dispute the check — instead of stopping. On Terminal-Bench that stop came - * with right work 34 times and wrong work 33: a coin flip, a fifth of tasks. - * Off by default until measured. + * with right work 34 times and wrong work 33: a coin flip, a fifth of tasks, + * and locally it ended runs whose check was right and whose work the model + * could still have fixed. OFF unless `true` (`--reveal-stuck`): as the default + * it cost 7 tasks of 60 on Mercury by letting the model bend correct work to + * wrong checks (see the proof loop). A claim that + * then passes carries `revealed`, and a later write to a path the task names + * as an input, or to one DISPUTE_HINT forbids, refuses it (bentAfterReveal). */ revealOnStuck?: boolean; + /** + * Experimental (`--review-advisory`, MAAT_REVIEW_ADVISORY=1): the independent + * review is recorded and shown but gates nothing — no nudge, no demotion — + * and "verified" instead needs a passing drafted runs+value check that + * failed before the work (tiers.ts `reviewAdvisory`). Off by default. + */ + reviewAdvisory?: boolean; + /** + * Requirement sign-out (src/signout.ts): one round per unattended turn that + * lists each stated requirement beside the commands run for it. Off unless + * `true` (`--signout`): it fired 60 times in one measured set of runs and + * rescued no task, while doubling steps and pushing input tokens to 97-116k. + */ + signOut?: boolean; + /** + * Who rules on a DISPUTE line (src/dispute.ts), and how many asks. One ask + * unless `votes` says more (`--dispute-votes`). `model` / `baseUrl` name an + * arbiter other than the worker (`--arbiter-model`). With none, or the same + * baseUrl and model as the worker, the dispute is rejected without asking: + * 0 of 20 were upheld, ten of those trees passed the grader, and a model + * ruling on its own misreading repeats it. + */ + dispute?: { votes?: number; model?: string; baseUrl?: string; apiKey?: string }; /** * Review a verified claim independently and label it (src/review.ts): * `votes` reviews of the task text and the receipt; a majority-backed, @@ -1697,6 +1866,13 @@ export class Engine { private ccForwarded = new WeakSet(); /** The turn's tool context, read by the MCP handlers as each call arrives. */ private ccCtx?: ToolContext; + /** + * The seal gate for a subprocess backend. The agent there makes its own tool + * calls inside one step, so the check the native loop does between planning + * and running (seal the drafted checks before the first change) has to run + * here, per call, before the tool does. Set by `run()` for the turn. + */ + private ccGate?: (name: string, args: Record) => AsyncGenerator; /** * Tools are shut for the rest of this turn. * @@ -1842,6 +2018,8 @@ export class Engine { * nowhere in the document. */ private did: string[] = []; + /** This turn's requirement sign-out, for the receipt (src/signout.ts). */ + private turnSignOut: SignOut | undefined; /** * Tool calls made this turn, by id. The session ledger is keyed by call id, * so this is what tells a write this turn made from one an earlier turn did @@ -1932,13 +2110,47 @@ export class Engine { private createdThisSession(): ReadonlySet { const out = new Set(); for (const e of this.sessionLedger()) if (e.before === null) out.add(e.path); + for (const f of this.bashCreated) out.add(f); return out; } + /** + * Project-relative files that first appeared while a bash call ran. + * + * The ledger only sees file-tool writes, so `echo x > t1.txt; ...; rm t1.txt` + * was a delete of a file molt "did not create": refused headless, left + * behind, and the grader failed the task for the leftover. A file absent + * before a call and present after it is molt's own doing exactly as a + * write_file of a new path is. Files only: a folder created by bash may + * later hold things that were moved into it. + */ + private bashCreated = new Set(); + /** Files this TURN wrote, with what was there before — what a revert acts on. */ private turnWrites: LedgerLike[] = []; /** Every check was retired by an upheld dispute this turn (judgment.ts). */ private turnAllRetired = false; + /** This session's provider latency, learned from completed requests (watchdog.ts). */ + private readonly latency = new LatencyLearner(); + /** + * The learner for the drafter's and critic's asks (AskOptions.latency): + * what they take teaches the engine's own requests what normal is. Not + * offered for a self-hosted server, whose allowances come from its hardware. + */ + get askLatency(): LatencyLearner | undefined { + return isSelfHosted(this.cfg.baseUrl) ? undefined : this.latency; + } + /** Why the independent review was skipped this turn (see reviewSkipReason), if it was. */ + private reviewSkipped: string | undefined; + /** Hidden checks whose commands were shown to the model this turn. */ + private turnRevealed: string[] = []; + /** + * Set when the model was stopped (clock, provider) and the sealed bar was run + * on the tree as it stood. Six of twenty local runs ended with no verdict at + * all because the clock or a failed request stopped the turn before any check + * ran; two of them had passing work on disk. + */ + private turnEndedBy: "deadline" | "provider" | undefined; /** `--revert` put this turn's work back, so a person cannot accept it as it stands. */ private turnRestored = false; /** The pre-turn working tree, as a commit object nothing else can see. */ @@ -2292,6 +2504,27 @@ export class Engine { return ms > 0 && this.turnStartedAt > 0 ? this.turnStartedAt + ms : undefined; } + /** + * Whether the independent review would outlast the turn, and so is not run. + * Six of eight no-verdict runs on Mercury 2.5 passed their bar and then died + * inside the review's three asks, past the runner's grace; one stalled ask is + * 123 s. With under three first-byte allowances left (a fixed 60 s until the + * provider has shown what its normal is) the claim ends as it stands, said + * to be unreviewed. + */ + private reviewSkipReason(): string | undefined { + const left = this.timeLeftMs(); + if (left === undefined) return undefined; + // The review runs on the judge. A separate judge model is timed at the default, not + // at the worker's learned latency (a slow worker made a fast judge's review look like 9 min). + const judged = judgeTarget({ baseUrl: this.cfg.baseUrl, model: this.cfg.model }).model !== this.cfg.model; + const need = 3 * ((judged ? undefined : this.latency.learnedFirstByte()) ?? 20_000); + if (left > need) return undefined; + return left <= 0 + ? "the time budget had run out" + : `${Math.round(left / 1000)}s of the time budget were left, under the ${Math.round(need / 1000)}s a review can take`; + } + /** Milliseconds left before this turn's deadline, or undefined for none. */ private timeLeftMs(): number | undefined { const at = this.deadlineAt(); @@ -3002,6 +3235,9 @@ export class Engine { const left = this.timeLeftMs(); const timeoutMs = left !== undefined ? Math.max(5_000, Math.min(wanted, left - 10_000)) : wanted; const t0 = Date.now(); + // Only a command that could write needs the before/after listing; a + // listing that was cut short (null) records nothing rather than guess. + const listed = isReadOnlyCommand(command) ? null : listProject(this.cwd); const r = await runCommand(command, { cwd: this.cwd, timeoutMs, @@ -3013,6 +3249,10 @@ export class Engine { signal: this.running?.signal, }); const took = Date.now() - t0; + if (listed) { + const now = listProject(this.cwd); + if (now) for (const f of now.files) if (!listed.files.has(f)) this.bashCreated.add(f); + } // Same shape execSync produced: bare stdout when it worked, and a // tagged dump of both streams when it did not — plus how long it // took, once it took long enough to matter. A model that knows the @@ -3055,6 +3295,18 @@ export class Engine { return renderPlan(steps, Number(args.current ?? 0)); } + case "act": + // Reached only when an act's actions could not be read (expandAct): + // a readable one never arrives here as itself. + return ( + "act: nothing ran, because at least one of its actions could not be read. The actions go in order, " + + "so the readable ones were not run without it either. This is not a refusal — nothing was " + + "denied; the action list itself could not be parsed. `actions` must be a JSON array (not a " + + "string holding one) of " + + '{"tool": "", "args": {...}} objects, e.g. ' + + '[{"tool": "write_file", "args": {"path": "out.txt", "content": "..."}}]. Send them again.' + ); + default: return `unknown tool: ${name}`; } @@ -3195,6 +3447,11 @@ export class Engine { return this.markGuards(await runBar(bar, this.barContext(claim))); } + /** tierOf's advisory-review arguments: empty unless `reviewAdvisory`. */ + private advisoryTier(): { reviewAdvisory?: true; guards?: ReadonlySet } { + return this.cfg.reviewAdvisory === true ? { reviewAdvisory: true, guards: this.passedBeforeWork } : {}; + } + /** See `passedBeforeWork`. Only a passing command criterion is relabelled. */ private markGuards(result: BarResult): BarResult { if (this.passedBeforeWork.size === 0) return result; @@ -3278,7 +3535,7 @@ export class Engine { promptTokens: Math.round(this.bom().requestTotalEst * this.tokenScale), maxTokens: this.maxTokensFor(), stream: false, - }), + }, isSelfHosted(this.cfg.baseUrl) ? localSpeed() : undefined), idleMs: idle, }); try { @@ -3495,6 +3752,7 @@ export class Engine { result: BarResult, attempts: number, log?: Journal, + endedBy?: "deadline" | "provider", ): AsyncGenerator { const onlyWrites = failedOnlyWriteChecks(result); if (this.cfg.receipts) { @@ -3507,6 +3765,7 @@ export class Engine { provider: this.provider, sessionTokens: this.sessionTokens, shedBatches: this.transcript.shedCount, + ...(endedBy ? { endedBy } : {}), }); log?.append("receipt", { verdict: "exhausted", file: receipt.path, attempt: attempts }); this.bindReceipt(receipt.path, "exhausted"); @@ -3782,9 +4041,12 @@ export class Engine { this.turnStartedAt = startedAt; this.turnWrites = []; this.turnAllRetired = false; + this.turnRevealed = []; + this.turnEndedBy = undefined; this.turnRestored = false; this.refusedThisTurn = false; this.turnReview = undefined; + this.reviewSkipped = undefined; this.sealedChecks = []; this.turnCalls = new Set(); // molt's records stay out of the project's `git status` (leftovers.ts). @@ -3792,6 +4054,7 @@ export class Engine { // Per turn, or receipt five lists what turn one ran. It did: receipts // 0043–0047 of this project each open with the same three reads. this.did = []; + this.turnSignOut = undefined; await this.captureTree(); this.turnTree = snapshotTree(this.cwd); let steps = 0; @@ -3806,6 +4069,7 @@ export class Engine { let receiptPath: string | undefined; let exhaustedResult: BarResult | undefined; + let lastProof: BarResult | undefined; for await (const ev of this.runTurn(userText, confirm, job, opts)) { switch (ev.kind) { case "receipt": @@ -3826,6 +4090,7 @@ export class Engine { break; case "proof_result": proven = true; + lastProof = ev.result; break; case "assistant_text": answered = true; @@ -3841,16 +4106,32 @@ export class Engine { ? "cancelled" : exhausted ? "not proven" - : errored + : errored && !(this.turnEndedBy && proven) ? "error" : proven ? opts.ask && this.turnWrites.length === 0 ? "answered" : "verified" - : answered + : answered || this.turnEndedBy === "deadline" ? "unverified" : "stopped"; + // Checks that passed on a tree the model was STOPPED on — by the clock or + // a failed provider — never make "verified": the model never said the work + // was done. On Mercury 2.5 that is exactly where the false verifieds came + // from: log-summary, sql-top-customers and cron-next each ran out the + // clock half-finished, the drafted checks passed on what was there, and + // the grader failed all three. The pass is kept on the record + // (passedAtEnd) — it is information — but the outcome is unverified. + const passedAtEnd = outcome === "verified" && this.turnEndedBy !== undefined; + if (passedAtEnd) { + outcome = "unverified"; + yield { + kind: "info", + text: `the checks passed on the work as it stood when the ${this.turnEndedBy === "deadline" ? "clock" : "provider"} stopped the model, but it never said it was done — unverified.`, + }; + } + // Refused only by checks the model drafted for itself. On the local suite // such a refusal was right about one time in five: 19 of 23 "not proven" // runs had passed the grader, the checks having miscounted, misread a @@ -3891,20 +4172,24 @@ export class Engine { if (review && !review.confirmed) { yield { kind: "info", text: `passed its checks, unconfirmed: the reviewer (${review.votes}) found ` + review.violations.map((v) => `"${v.quote}" — ${v.evidence}`).join("; ") }; } - } else if (outcome === "verified" && this.cfg.review && receiptPath && existsSync(receiptPath)) { + } else if (outcome === "verified" && this.cfg.review && !this.turnEndedBy && this.reviewSkipReason()) { + this.reviewSkipped = this.reviewSkipReason(); + this.cfg.journal?.append("review", { ran: false, skipped: this.reviewSkipped }); + yield { kind: "info", text: `skipping the independent review: ${this.reviewSkipped}. The claim stands as checked, unreviewed.` }; + } else if (outcome === "verified" && this.cfg.review && !this.turnEndedBy && receiptPath && existsSync(receiptPath)) { yield { kind: "info", text: "reviewing the claim independently against the task text" }; review = await reviewClaim({ task: userText, receipt: readFileSync(receiptPath, "utf8"), votes: this.cfg.review.votes, ask: { - baseUrl: this.cfg.baseUrl, - apiKey: this.cfg.apiKey, - model: this.cfg.model, + ...judgeTarget({ baseUrl: this.cfg.baseUrl, apiKey: this.cfg.apiKey, model: this.cfg.model }), cwd: this.cwd, - reasoningEffort: this.cfg.review.reasoningEffort ?? this.cfg.reasoningEffort, + reasoningEffort: judgeEffort(this.cfg.review.reasoningEffort ?? this.cfg.reasoningEffort), fetchFn: this.cfg.fetchFn, acpSpawn: this.cfg.acpSpawn, + // Never longer than the turn has left: a review that outlives the clock ends it without a verdict. + ...(this.timeLeftMs() !== undefined ? { timeoutMs: Math.max(1_000, this.timeLeftMs()!) } : {}), }, }).catch(() => null); this.cfg.journal?.append("review", review ? { confirmed: review.confirmed, votes: review.votes, violations: review.violations.length } : { ran: false }); @@ -3951,6 +4236,29 @@ export class Engine { // The record of a case is never worth failing the turn over. } } + // The word "verified" is earned by evidence, the same on every model: a + // passing check that ran the deliverable and asserted a value (or one a + // person wrote), and no reviewer contradiction. Measured, that class was + // right 7/7 where everything else carrying the word was right 28/50. What + // passed without earning it is reported as what it is. After the judgment + // case above, which is opened on what the reviewers said. + let tier: "verified" | "passed-checks" | undefined; + let tierReason: string | undefined; + if (outcome === "verified" && lastProof) { + // Review was asked for and did not run (skipped near the deadline, or a + // nudge cleared it and no re-review followed): no "verified". On v11, 6 + // of the 7 "verified" claims were unreviewed this way and 4 were wrong. + const unreviewed = this.cfg.review !== undefined && !review; + const t = tierOf({ results: lastProof.results, review, unreviewed, ...this.advisoryTier() }); + tier = t.tier; + if (t.tier === "passed-checks") { + outcome = "unverified"; + tierReason = t.reason; + yield { kind: "info", text: `passed its checks (not verified: ${t.reason}).` }; + } + if (receiptPath) this.cfg.receipts?.amendTier(basename(receiptPath), t); + this.cfg.journal?.append("note", { text: `tier: ${t.tier}`, evidence: t.evidence, ...(t.reason ? { reason: t.reason } : {}), ...(t.reviewNote ? { review: t.reviewNote } : {}) }); + } yield { kind: "job_end", job, @@ -3958,15 +4266,111 @@ export class Engine { spend: this.spendSince(before), durationMs: Date.now() - startedAt, outcome, + ...(tier ? { tier, ...(tierReason ? { tierReason } : {}) } : {}), ...(selfChecked ? { selfChecked: true } : {}), + ...(this.reviewSkipped ? { unreviewed: true } : {}), + ...(this.turnEndedBy ? { endedBy: this.turnEndedBy, ...(this.turnEndedBy === "deadline" ? { deadline: true } : {}) } : {}), + ...(passedAtEnd ? { passedAtEnd: true } : {}), ...(checksDisagree ? { checksDisagree: failing.map((r) => r.name) } : {}), + ...(outcome === "verified" && this.turnRevealed.length ? { revealed: [...this.turnRevealed] } : {}), ...(review ? { review: { confirmed: review.confirmed, votes: review.votes, violations: review.violations } } : {}), ...(opened !== undefined ? { case: opened } : {}), }; } /** - * The live ACP (Grok Build) session, started when it is first needed. + * The git-tracked project files the revealed checks' commands name: fixtures + * the check reads, which the model did not create. Committing one after the + * reveal is bentAfterReveal's git case. Untracked names are the model's own + * output and stay free. + */ + private revealedTargets(commands: string[]): string[] { + let tracked: Set; + try { + tracked = new Set( + execFileSync("git", ["ls-files"], { cwd: this.cwd, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }) + .split("\n") + .filter(Boolean), + ); + } catch { + return []; + } + const out = new Set(); + for (const cmd of commands) { + for (const w of cmd.match(/[\w./-]+/g) ?? []) { + const rel = relative(this.cwd, resolve(this.cwd, w)); + if (tracked.has(rel)) out.add(rel); + } + } + return [...out]; + } + + /** + * Why a claim that passed after a reveal is refused, or undefined. + * + * Once the model has read a hidden check's command, the cheap way to meet it + * is to change what the check reads: the task's input data, the shell, an + * installed package, the check's own files. 10 of the 29 right-looking + * Terminal-Bench passes after such a reveal were bent that way. Only writes + * made after the reveal count, and only to a file that existed (an input the + * model created is an output). Inputs are the paths the task text names + * itself, not namedInputs' guess at the project's data files. + */ + private bentAfterReveal( + seen: Set, + task: string, + revealed?: { didAt: number; targets: string[] }, + ): { reason: string; hard: boolean } | undefined { + const later = this.turnLedger().filter((e) => !seen.has(e.callId)); + const inputs = new Set( + namedInputs(task, this.cwd) + .filter((p) => task.includes(relative(this.cwd, p)) || task.includes(basename(p))) + .map((p) => resolve(p)), + ); + // A `git add` / `git commit` through bash after the reveal makes a change + // to the check's target durable without the ledger seeing a write. Same + // split as the writes below: environment or a test file the task does not + // ask for is hard; a named input, or a tracked file the revealed check + // reads, ends the claim unverified. + if (revealed) { + // What the model created this turn is an output, however the task names it. + const made = new Set(this.turnLedger().filter((e) => e.before === null).map((e) => resolve(this.cwd, e.path))); + for (const rel of gitPathsStaged(this.did.slice(revealed.didAt), this.cwd)) { + const abs = resolve(this.cwd, rel); + if (/^\/(usr\/)?(local\/)?s?bin\//.test(abs) || /\/(site|dist)-packages\//.test(abs) || /\/\.maat\//.test(abs)) { + return { reason: `after the check was shown, the model committed ${rel} with git, part of the environment, not the work`, hard: true }; + } + if (isTestPath(rel) && !task.includes(basename(rel))) { + return { reason: `after the check was shown, the model committed ${rel} with git, a test or check file the task does not ask for`, hard: true }; + } + if (made.has(abs)) continue; + if ([...inputs].some((i) => touches(rel, relative(this.cwd, i))) || revealed.targets.some((t) => touches(rel, t))) { + return { reason: `after the check was shown, the model committed ${rel} with git, a file the check reads or the task names as an input`, hard: false }; + } + } + } + if (later.length === 0) return undefined; + for (const e of later) { + const abs = resolve(this.cwd, e.path); + if (/^\/(usr\/)?(local\/)?s?bin\//.test(abs) || /\/(site|dist)-packages\//.test(abs) || /\/\.maat\//.test(abs)) { + return { reason: `after the check was shown, the model wrote ${e.path}, part of the environment, not the work`, hard: true }; + } + if (e.before !== null && inputs.has(abs)) { + // Not a refusal: in a task that transforms its inputs in place + // (config-migrate, redact-secrets) the named file IS the deliverable, + // and refusing would fail correct work. It cannot earn "verified" + // after the check was shown, so the claim ends unverified, said why. + return { reason: `after the check was shown, the model changed ${e.path}, an input the task names`, hard: false }; + } + if (isTestPath(e.path) && !task.includes(basename(e.path))) { + return { reason: `after the check was shown, the model changed ${e.path}, a test or check file the task does not ask for`, hard: true }; + } + } + return undefined; + } + + /** + * The live ACP session (Grok Build, OpenCode), started when it is first needed. * * Rebuilt whenever molt's system prompt changes — `/map`, `/read` and a * repo-map refresh all rewrite it — because a session carrying the old one @@ -3995,6 +4399,14 @@ export class Engine { if (this.ccNoTools) { return "[molt: the turn is over. No more tools — answer with what you already have.]"; } + if (this.ccGate) { + const gate = this.ccGate(name, args); + for (;;) { + const g = await gate.next(); + if (g.done) break; + emit(g.value); + } + } const calls = this.invokeTool({ id: callId, name, rawArgs: JSON.stringify(args) }, ctx); for (;;) { const next = await calls.next(); @@ -4165,7 +4577,15 @@ export class Engine { return null; } if (done.error) { - ctx.log?.append("error", { text: done.error }); + const wall = longQuotaText(done.error); + ctx.log?.append("error", { text: wall ?? done.error, ...(wall ? { providerCapped: true } : {}) }); + if (wall) { + // A quota that lifts hours from now is not this task's failure, and + // nothing inside a turn gets past it. Fail fast, say when it lifts. + yield { kind: "error", text: `${wall}. Nothing was verified; the work above still happened.` }; + await this.dropAcpSession(); + return null; + } yield { kind: "error", text: @@ -4257,9 +4677,14 @@ export class Engine { const { id: callId, name } = call; let args: Record = {}; let malformed = false; + /** What the arguments get wrong against the tool's schema (src/toolargs.ts). */ + let violations: string[] = []; try { - args = JSON.parse(call.rawArgs || "{}") as Record; + args = parseLenient(call.rawArgs || "{}") as Record; + if (!args || typeof args !== "object" || Array.isArray(args)) throw new Error("not an object"); if (name === "bash") foldCommands(args, this.cwd); + violations = argumentProblems(TOOLS.find((t) => t.function.name === name)?.function.parameters as never, args); + if (violations.length) malformed = true; } catch { // Running with empty arguments produced a misleading error — a // malformed read_file became "EISDIR: illegal operation on a @@ -4316,7 +4741,7 @@ export class Engine { kind: "tool", name, id: callId, - detail: "malformed arguments", + detail: violations.length ? "arguments do not fit the schema" : "malformed arguments", note: "malformed", args: raw, bytes: 0, @@ -4336,8 +4761,11 @@ export class Engine { `off part-way through the arguments for ${name}, so nothing ran. Sending the same call ` + `again will be cut off in the same place. Write less in one go — a smaller file, or one ` + `part of it at a time — or ask for the ceiling to be raised with --max-tokens.]` - : `[molt: the arguments for ${name} were not valid JSON, so nothing ran. ` + - `Send them again as a JSON object. What arrived was: ${raw}]`; + : violations.length + ? `[molt: the arguments for ${name} do not fit its schema, so nothing ran: ` + + `${violations.join("; ")}. Send the call again with them fixed.]` + : `[molt: the arguments for ${name} were not valid JSON, so nothing ran. ` + + `Send them again as a JSON object. What arrived was: ${raw}]`; this.transcript.push({ role: "tool", tool_call_id: callId, content: complaint }); return { name, result: complaint, auto: true, repeated: false }; } @@ -4523,6 +4951,16 @@ export class Engine { ): AsyncGenerator { // Remember where this turn began so a cancellation can leave no trace. const turnStart = this.transcript.length; + /** + * The project as the turn found it, headless only: at the first claim, + * files that appeared since and that the task never names are put to the + * model once — the helper script or copy it left beside the deliverable. + */ + const turnListing = this.cfg.unattended ? listProject(this.cwd) : null; + let leftoversAsked = false; + /** Requirement sign-out: the stated requirements, and whether they were put once (src/signout.ts). */ + let requirements = normalizeRequirements(opts.requirements); + let signedOut = false; this.passedBeforeWork = new Set(); const log = this.cfg.journal; /** @@ -4545,6 +4983,16 @@ export class Engine { */ let pendingCriteria = opts.pendingCriteria; const criteriaSince = Date.now(); + /** + * The draft, when the time budget's cut found it with no checks: kept alive + * to join at the first claim instead of sealing nothing (joinLateCriteria). + */ + let lateCriteria: typeof pendingCriteria; + let draftInputs = ""; + if (pendingCriteria && !opts.ask) { + draftInputs = draftInputsHash(userText, listProject(this.cwd)); + log?.append("note", { kind: "draft-inputs", text: "drafter inputs fixed at turn start", inputsSha: draftInputs }); + } /** Sealed mid-step: tell the model once this step's tool results are in. */ let announceAfterTools = false; const self = this; @@ -4665,6 +5113,11 @@ export class Engine { /** The previous attempt's failures, to notice a bar going nowhere. */ let lastFailure = ""; let revealedThisTurn = false; + let revealMark: { didAt: number; targets: string[] } | undefined; + /** Calls already ledgered when the reveal happened: what is new came after the model read the check. */ + let revealSeen = new Set(); + /** Set when a write after the reveal bent the work to the check; ends the turn refused. */ + let bentReason: string | undefined; let lastReceipt: string | undefined; let lastReceiptPath: string | undefined; /** The reviewers' findings have been put to the model once this turn. */ @@ -4722,10 +5175,10 @@ export class Engine { pendingCriteria = undefined; const ready = await Promise.race([p.then(() => true, () => true), new Promise((r) => setTimeout(() => r(false), 50))]); if (!ready) yield { kind: "info", text: "waiting for this task's checks to be sealed before the first change" }; - const none = { taskChecks: [] as Check[], taskNotes: [] as string[] }; + const none: { taskChecks: Check[]; taskNotes: string[]; requirements?: string[] } = { taskChecks: [], taskNotes: [] }; // Bounded under a time budget: query-optimize spent 262 s of 705 here. const cap = opts.criteriaWaitMs ?? criteriaWaitMs(self.turnDeadlineMs); - let got: { taskChecks: Check[]; taskNotes: string[] }; + let got: { taskChecks: Check[]; taskNotes: string[]; requirements?: string[] }; if (cap !== undefined && opts.criteriaSoFar) { const left = Math.max(0, cap - (Date.now() - criteriaSince)); let timer: NodeJS.Timeout | undefined; @@ -4737,14 +5190,28 @@ export class Engine { if (out) got = out; else { got = await opts.criteriaSoFar().catch(() => none); - yield { - kind: "info", - text: `checks were still being drafted after ${Math.round(cap / 1000)}s of the time budget; sealing the ${got.taskChecks.length} reviewed so far`, - }; + if (got.taskChecks.length === 0 && !opts.ask) { + // Sealing nothing would leave every correct result unverifiable + // (14 of 60 runs on Mercury 2.5, when the drafter's request + // stalled). The draft keeps going and joins at the first claim. + lateCriteria = p; + yield { + kind: "info", + text: + `checks were still being drafted after ${Math.round(cap / 1000)}s of the time budget and none was ready; ` + + `not sealing an empty set — the first claim waits up to ${Math.round((opts.lateCriteriaWaitMs ?? lateCriteriaWaitMs(self.timeLeftMs())) / 1000)}s for them and judges with them if they arrive`, + }; + } else { + yield { + kind: "info", + text: `checks were still being drafted after ${Math.round(cap / 1000)}s of the time budget; sealing the ${got.taskChecks.length} reviewed so far`, + }; + } } } else { got = await p.catch(() => none); } + requirements = normalizeRequirements([...requirements, ...(got.requirements ?? [])]); yield* sealCriteria([...(opts.taskChecks ?? []), ...got.taskChecks], [...(opts.taskNotes ?? []), ...got.taskNotes]); // Mid-step, the announcement waits for this step's tool results: a // message between a tool call and its result is a malformed @@ -4755,6 +5222,107 @@ export class Engine { ? withoutWriteChecks(withTaskChecks(self.cfg.bar, taskChecks)) : withTaskChecks(self.cfg.bar, taskChecks); } + // The same gate for a backend that runs its own tool calls (ACP, Claude + // Code): sealed before the first call that changes anything. + this.ccGate = async function* (name, args) { + if (pendingCriteria && changesSomething(name, JSON.stringify(args))) yield* settleCriteria(false); + }; + /** The reference check, until it joins or proves not to apply. */ + let referencePending = opts.ask ? undefined : opts.referenceCheck; + async function* joinReference(): AsyncGenerator { + const p = referencePending; + if (!p) return; + const wait = opts.referenceWaitMs ?? referenceWaitMs(self.timeLeftMs()); + let timer: NodeJS.Timeout | undefined; + const late = Symbol("late"); + const got = await Promise.race([ + p.catch(() => null), + new Promise((r) => { timer = setTimeout(() => r(late), wait); }), + ]); + clearTimeout(timer); + if (got === late) { + yield { kind: "info", text: "the reference check is still being written; judging this claim without it" }; + return; + } + referencePending = undefined; + if (!got) return; + const check = Object.freeze({ ...got.check, name: got.check.name.startsWith("task:") ? got.check.name : `task:${got.check.name}` }); + if (taskChecks.some((c) => c.name === check.name)) return; + taskChecks = Object.freeze([...taskChecks, check]) as Check[]; + self.sealedChecks = taskChecks; + bar = withTaskChecks(self.cfg.bar, taskChecks); + log?.append("note", { text: "reference check joined the sealed checks", ...got.note }); + yield { + kind: "info", + text: "an independent reference check joined this turn's checks — written from the task text and the project as it was before the work", + }; + self.transcript.push({ + role: "user", + content: + "An independent reference check has joined the acceptance criteria. It was written from the task text " + + "and the project as it was before you began, by a reviewer who never saw your work: it compares your " + + "deliverable with a reference on the task's examples, the edge cases its words imply, and random inputs. " + + "The command is withheld; on a failure you will see the input, the expected and the actual result.", + }); + } + /** + * Checks that were still drafting when the time budget's cut found none + * ready. Like the reference check they were written from the task text and + * the project before the first step, so they may join at a claim; the + * inputs' hash journalled at turn start is journalled again here. Not + * ready by the claim, it is judged without them and the next claim looks + * again. + */ + async function* joinLateCriteria(): AsyncGenerator { + const p = lateCriteria; + if (!p) return; + const wait = opts.lateCriteriaWaitMs ?? lateCriteriaWaitMs(self.timeLeftMs()); + let timer: NodeJS.Timeout | undefined; + const late = Symbol("late"); + const got = await Promise.race([ + p.catch(() => null), + new Promise((r) => { timer = setTimeout(() => r(late), wait); }), + ]); + clearTimeout(timer); + if (got === late) { + yield { kind: "info", text: `the drafted checks did not arrive within ${Math.round(wait / 1000)}s; this claim is judged without them, so it cannot be verified by task checks` }; + return; + } + lateCriteria = undefined; + const have = new Set(taskChecks.map((c) => c.name)); + const added = (got?.taskChecks ?? []) + .map((c) => Object.freeze({ ...c, name: c.name.startsWith("task:") ? c.name : `task:${c.name}` })) + .filter((c) => !have.has(c.name)); + if (!added.length) { + log?.append("note", { kind: "late-checks", text: "the drafter finished with no checks to join", inputsSha: draftInputs }); + return; + } + taskChecks = Object.freeze([...taskChecks, ...added]) as Check[]; + taskNotes = Object.freeze([...taskNotes, ...(got?.taskNotes ?? [])]) as string[]; + taskSeal = sealOf(taskChecks, taskNotes); + requirements = normalizeRequirements([...requirements, ...(got?.requirements ?? [])]); + self.sealedChecks = taskChecks; + bar = withTaskChecks(self.cfg.bar, taskChecks); + log?.append("note", { + kind: "late-checks", + text: `late drafted checks joined the sealed checks: ${added.length} check(s)`, + inputsSha: draftInputs, + seal: taskSeal, + checks: added.map((c) => c.name), + }); + yield { + kind: "info", + text: `${added.length} drafted check(s) joined this turn's checks — written from the task text and the project as it was before the work`, + }; + self.transcript.push({ + role: "user", + content: + "Drafted acceptance criteria have joined this turn's checks. They were written from the task text and " + + "the project as it was before you began, by a drafter that never saw your work: " + + added.map((c) => c.name).join(", ") + + ". The commands are withheld; on a failure you will see the output.", + }); + } // Drafted checks retired by an upheld dispute this turn (src/dispute.ts), // and every check already disputed once, by bare name. const retired = new Map(); @@ -4765,6 +5333,136 @@ export class Engine { opts.ask ? asQuestion(withTaskChecks(this.cfg.bar, liveTaskChecks()), this.turnLedger().length === 0) : withTaskChecks(this.cfg.bar, liveTaskChecks()); + /** + * Sealed checks that failed in their own code or ran out their own time. + * + * A drafted check that errs says nothing about the work. One that hangs + * says nothing either, and a bar that waits on it ends the run at the + * runner's kill with no verdict: a check that never finished is retired + * for this run, and the others judge. + */ + const brokenOwn = (res: BarResult): CheckResult[] => + res.cancelled + ? [] + : res.results.filter((r) => { + if (r.ok || r.advisory || retired.has(bare(r.name))) return false; + if (bare(r.name) === REFERENCE_CHECK_NAME) return r.exitCode === REFERENCE_SELF_ERROR || r.timedOut === true; + const sealed = taskChecks.some((c) => bare(c.name) === bare(r.name)); + return sealed && (r.timedOut === true || (r.hidden === true && checkSelfError(r.output) !== null)); + }); + async function* retireBroken(list: CheckResult[]): AsyncGenerator { + for (const r of list) { + const timeout = r.timedOut === true; + const why = timeout + ? "timeout" + : bare(r.name) === REFERENCE_CHECK_NAME + ? "the reference check failed in its own code" + : (checkSelfError(r.output) ?? ""); + retired.set(bare(r.name), { name: bare(r.name), upheld: true, votes: timeout ? "timeout" : "own error", reason: why }); + log?.append("note", { text: `check ${bare(r.name)} retired: ${why}`, output: r.output.slice(-500) }); + yield { + kind: "info", + text: timeout + ? `check ${bare(r.name)} timed out, which says nothing about the work; retired, and the claim judged without it` + : `check ${bare(r.name)} failed in its own code, not on the work (${why}); retired, and the claim judged without it`, + }; + } + } + /** + * The model is stopped; the work on disk is not left unjudged. + * + * The clock and a failed provider both used to end a turn with no check + * run: 6 of 20 local runs had no verdict, two with passing work in place. + * The sealed bar runs once on the tree as it stands, and the turn's outcome + * is what it says: verified only if it passed, not proven if it did not, + * unverified when nothing was sealed. Returns whether a bar ran. + */ + async function* judgeOnDisk(why: "deadline" | "provider", claim: string): AsyncGenerator { + self.turnEndedBy = why; + if (opts.ask) return false; + let barThis = barNow(); + if (!barThis || barThis.checks.length === 0) return false; + // Whatever the runner allows after the deadline is short: a check may + // not take the rest of it. Past the cap it is retired as timed out. + const capMs = why === "deadline" ? DEADLINE_CHECK_CAP_MS : undefined; + const capped = (b: Bar): Bar => + capMs === undefined + ? b + : { ...b, checks: b.checks.map((c) => (c.kind === "command" ? { ...c, timeoutMs: Math.min(c.timeoutMs, capMs) } : c)) }; + proofAttempts += 1; + yield { kind: "proof_start", checks: barThis.checks.length, names: barThis.checks.map((c) => c.name) }; + let result = await self.runBarGuarded(claim, capped(barThis)); + const broken = brokenOwn(result); + if (broken.length) { + yield* retireBroken(broken); + barThis = barNow(); + if (!barThis || barThis.checks.length === 0) { + self.turnAllRetired = true; + yield { kind: "info", text: "no other check is left to judge the work — this claim is unverified." }; + log?.append("session_end", { reason: `unverified: every check retired (${why})` }); + return true; + } + result = await self.runBarGuarded(claim, capped(barThis)); + } + if (result.cancelled) return true; + lastResult = result; + self.actsSinceBar = 0; + log?.append("bar_run", { + attempt: proofAttempts, + ok: result.ok, + total: result.results.length, + passed: result.results.filter((r) => r.ok).length, + failed: result.results.filter((r) => !r.ok).map((r) => r.name).join(", "), + ms: result.durationMs, + endedBy: why, + checks: result.results.map((r) => ({ + name: r.name, kind: r.kind, detail: r.detail, ok: r.ok, exitCode: r.exitCode ?? null, ms: r.durationMs, cached: r.cached === true, + })), + }); + const undetermined = + !result.ok && (result.undetermined?.length ?? 0) > 0 && result.results.every((r) => r.ok || r.advisory || r.skipped); + if (result.ok || undetermined) { + const verdict = result.ok ? "accepted" : "undetermined"; + if (self.cfg.receipts) { + const head = await treeState(self.cwd).catch(() => null); + const receipt = self.cfg.receipts.write({ + claim, result, attempt: proofAttempts, verdict, head, + model: self.modelOfRecord(), provider: self.provider, sessionTokens: self.sessionTokens, + session: self.cfg.journal?.sessionId, costUsd: self.costUsd(), costEstimated: self.costEstimated, + shedBatches: self.transcript.shedCount, endedBy: why, + changed: self.turnLedger().map((e) => ({ + path: e.path, before: e.before, after: e.after, + ...(e.changedLines?.length ? { lines: e.changedLines } : {}), + })), + cwd: self.cwd, did: [...self.did], + task: taskSeal + ? { + seal: sealOf(taskChecks, taskNotes), + checks: taskChecks.map((c) => (c.kind === "command" ? `${c.name}: ${c.run}` : `${c.name}: builtin ${c.builtin}`)), + notes: [...taskNotes], + } + : undefined, + ...(verdict === "accepted" ? { tier: tierOf({ results: result.results, ...self.advisoryTier() }) } : {}), + }); + log?.append("receipt", { verdict, file: receipt.path, attempt: proofAttempts, endedBy: why }); + self.bindReceipt(receipt.path, verdict); + self.capture(receipt.path, verdict, userText, result, claim); + lastReceipt = basename(receipt.path); + lastReceiptPath = receipt.path; + yield { kind: "receipt", path: receipt.path }; + } + if (result.ok) { + log?.append("session_end", { reason: "bar met", attempts: proofAttempts, endedBy: why }); + yield { kind: "proof_result", result, attempt: proofAttempts }; + yield* self.settlePassed(userText, proofAttempts, result.results.map((r) => r.name), lastReceipt, log); + } else { + log?.append("session_end", { reason: "undetermined", attempts: proofAttempts, endedBy: why }); + } + return true; + } + yield* self.finishUnproven(claim, result, proofAttempts, log, why); + return true; + } if (opts.ask) { const dropped = (this.cfg.bar?.checks.length ?? 0) - (bar?.checks.length ?? 0); log?.append("note", { text: `ask turn — ${dropped} write-dependent check(s) not run` }); @@ -4834,7 +5532,10 @@ export class Engine { `tool calls. The bar still decides what the work was worth: nothing is committed ` + `that it did not pass.`, }; - yield* this.salvage("Your time budget for this turn is up.", fetchFn, log); + // The work on disk is judged before anything else is asked of the + // provider: a closing summary can take the minutes the verdict needs. + const judged = yield* judgeOnDisk("deadline", ""); + if (!judged) yield* this.salvage("Your time budget for this turn is up.", fetchFn, log); return; } @@ -5107,6 +5808,10 @@ export class Engine { tools: this.offeredTools, tool_choice: "auto", ...(this.effortNow ? { reasoning: { effort: this.effortNow } } : {}), + // Thinking off on a self-hosted server, where `reasoning` is + // ignored (see selfHostedThinking). Nothing on a cloud endpoint. + ...selfHostedThinking(this.cfg.baseUrl, this.effortNow), + ...openRouterProvider(this.cfg.baseUrl, this.cfg.model), ...(stream ? { stream: true } : {}), ...(withUsage ? { stream_options: { include_usage: true } } : {}), }; @@ -5140,7 +5845,7 @@ export class Engine { // the credentials being wrong does not improve by asking again, and a // second identical refusal is exactly the spending this avoids. let failure: - | { text: string; why: string; retryable: boolean; retryAfterMs?: number } + | { text: string; why: string; retryable: boolean; retryAfterMs?: number; overload?: boolean } | undefined; /** * Shed-and-retry rounds used on this step. @@ -5164,10 +5869,21 @@ export class Engine { /** The deadline ended this step's request; the step loop closes the turn. */ let deadlineHit = false; + /** This request's turn at a self-hosted endpoint (src/localgate.ts); a no-op otherwise. */ + let releaseTurn: () => void = () => {}; for (let attempt = 0; ; attempt++) { failure = undefined; msg = undefined; watch?.dispose(); + // An attempt that left through a `continue` or a `break` inside the + // try (a shed-and-retry, a refusal that stops the retries) never + // reached the release below; its turn is given back here and after + // the loop, or the next request to a one-slot server waits for it + // for ever. Safe to call twice. + releaseTurn(); + // Taken before the watchdog starts: waiting behind Maat's own other + // requests to a one-slot local server is not the server being hung. + releaseTurn = await takeTurn(this.cfg.baseUrl, controller.signal).catch(() => () => {}); watch = new Watchdog(controller.signal, { firstByteMs: this.cfg.requestFirstByteMs ?? @@ -5176,10 +5892,10 @@ export class Engine { promptTokens: Math.round(requestEst * this.tokenScale), maxTokens: this.maxTokensFor(), stream, - }), + }, isSelfHosted(this.cfg.baseUrl) ? localSpeed() : undefined), idleMs: idle, deadlineAt: this.deadlineAt(), - }); + }, isSelfHosted(this.cfg.baseUrl) ? undefined : this.latency); let res: Response | undefined; try { res = await send(askForUsage); @@ -5370,10 +6086,20 @@ export class Engine { break; } const transient = res.status === 408 || res.status === 429 || res.status >= 500; + const resetAt = res.status === 429 ? rateLimitResetAt(body) : undefined; + if (resetAt !== undefined && resetAt - Date.now() > LONG_RATE_LIMIT_MS) { + failure = { + text: `the provider's rate limit is reached ${untilText(resetAt)} — ${body.slice(0, 200)}`, + why: "The provider will not take requests again until its limit resets; waiting inside a turn cannot get past it.", + retryable: false, + }; + break; + } failure = { text: `HTTP ${res.status}: ${body.slice(0, 300)}`, why: `The provider refused the request with HTTP ${res.status}.`, retryable: transient, + ...(res.status === 429 || res.status === 503 || res.status === 529 ? { overload: true } : {}), }; // A rate limit usually says when to come back. Believe it over a // fixed backoff — guessing shorter earns a second refusal, and @@ -5461,6 +6187,7 @@ export class Engine { yield { kind: "delta", text: tail }; } const result = await reading; + if (result.error) failure = providerFailure(result.error); msg = result.message; finishReason = result.finishReason; usage = { @@ -5478,6 +6205,7 @@ export class Engine { type Payload = { choices?: { message?: Msg; finish_reason?: string | null }[]; usage?: Usage; + error?: ProviderError; }; let json: Payload | undefined; try { @@ -5502,6 +6230,7 @@ export class Engine { finishReason = finishReasonFor(native.stop_reason); usage = usageFor(native.usage); } else if (json) { + if (json.error && typeof json.error === "object") failure = providerFailure(json.error); msg = json.choices?.[0]?.message; finishReason = json.choices?.[0]?.finish_reason ?? undefined; usage = json.usage; @@ -5517,9 +6246,11 @@ export class Engine { retryable: true, }; } + if (msg) msg = normalizeMessage(msg); } } catch (e) { if (controller.signal.aborted) { + releaseTurn(); watch.dispose(); this.inFlight = undefined; this.transcript.rollbackTo(turnStart); @@ -5551,6 +6282,7 @@ export class Engine { retryable: true, }; } + releaseTurn(); // The watchdog, whichever layer noticed it. A body read cut off by // it can surface as "not JSON" or as a network error, and neither is // what happened. @@ -5571,8 +6303,14 @@ export class Engine { }; } + if (failure) msg = undefined; + // A request that completed teaches the session what this provider's + // normal looks like (watchdog.ts LatencyLearner). Local servers keep + // their hardware-based allowances. + if (!failure && msg && !isSelfHosted(this.cfg.baseUrl)) this.latency.record(watch); if (!failure) break; - if (!failure.retryable || attempt >= NETWORK_RETRIES) break; + const retries = failure.overload ? OVERLOAD_RETRIES : NETWORK_RETRIES; + if (!failure.retryable || attempt >= retries) break; // About to replay this message from the beginning. Anything already on // screen belongs to an attempt that is being abandoned, and leaving it // there would show the reader the same sentence twice with no way to @@ -5581,14 +6319,14 @@ export class Engine { shownThisAttempt = false; yield { kind: "stream_reset", why: failure.text }; } - const backoff = this.cfg.retryBackoffMs ?? NETWORK_BACKOFF_MS; + const backoff = this.cfg.retryBackoffMs ?? (failure.overload ? OVERLOAD_BACKOFF_MS : NETWORK_BACKOFF_MS); const wait = failure.retryAfterMs ?? backoff[attempt] ?? backoff.at(-1) ?? 4_000; log?.append("note", { text: `${failure.text} — retrying in ${wait}ms` }); yield { kind: "info", text: `${failure.text} — retrying in ${Math.round(wait / 100) / 10}s, ` + - `attempt ${attempt + 2} of ${NETWORK_RETRIES + 1}`, + `attempt ${attempt + 2} of ${retries + 1}`, }; await sleepUnlessAborted(wait, controller.signal); if (controller.signal.aborted) { @@ -5602,6 +6340,7 @@ export class Engine { } watch?.dispose(); + releaseTurn(); this.inFlight = undefined; if (deadlineHit) { @@ -5628,7 +6367,9 @@ export class Engine { // is the conversation, the model id, or the credentials being wrong, // and a salvage would be refused in exactly the same way — paying // twice to be told the same thing is the spending this avoids. - if (failure?.retryable !== false) { + // Work that happened is judged, whatever the provider did after it. + const judged = this.turnWrites.length > 0 ? yield* judgeOnDisk("provider", "") : false; + if (!judged && failure?.retryable !== false) { yield* this.salvage(failure?.why ?? "The provider returned nothing usable.", fetchFn, log); } return; @@ -5810,7 +6551,10 @@ export class Engine { // with text and no tool call — so the claim path is the ordinary one. if (this.cfg.batch && msg.tool_calls?.length) { const acts = msg.tool_calls.map((c) => expandAct(c)); - if (acts.every((a) => a !== null && a.subs.length === 0)) { + // Finished only when every act really is empty. One that carried + // actions Maat could not read is not "done": it goes to the tool path + // below, which answers it with an error naming the expected shape. + if (acts.every((a) => a !== null && a.subs.length === 0 && a.unusable === 0)) { const said = acts.map((a) => a!.analysis).filter(Boolean).join("\n\n"); msg = { ...msg, content: [msg.content, said].filter((x) => x && x.trim()).join("\n\n") }; delete (msg as { tool_calls?: unknown }).tool_calls; @@ -5883,7 +6627,15 @@ export class Engine { for (const [i, call] of msg.tool_calls.entries()) { const last = finishReason === "length" && i === msg.tool_calls.length - 1; const act = this.cfg.batch ? expandAct(call) : null; - if (act && act.subs.length) { + if (act && act.unusable > 0) { + log?.append("note", { text: `act with ${act.unusable} unreadable action(s)`, args: (call.function?.arguments ?? "").slice(0, 20_000) }); + } + // An act with any action Maat could not read runs none of them: the + // others may depend on the one that was dropped (a write, then the + // command that reads it), and the model, shown results for fewer + // actions than it sent, would not know which one vanished. It goes + // the unreadable way below and is answered with the expected shape. + if (act && act.subs.length && act.unusable === 0) { act.subs.forEach((sub, k) => planned.push({ id: sub.id, name: sub.name, rawArgs: sub.rawArgs, truncated: last && k === act.subs.length - 1 }), ); @@ -6006,7 +6758,7 @@ export class Engine { // step that did call something is left alone: its text is a report. const narrated = msg.content && !(this.subprocess && this.turnCalls.size > callsBeforeStep) - ? narratedCallIn(msg.content) + ? narratedCallIn(msg.content, this.cfg.batch ? [...MOLT_TOOL_NAMES, "act"] : undefined) : null; if (narrated) { narratedTurns += 1; @@ -6085,6 +6837,52 @@ export class Engine { // ---- The model believes it is finished. That is a claim. ---- const claim = msg.content ?? ""; if (pendingCriteria) yield* settleCriteria(); + if (referencePending) yield* joinReference(); + if (lateCriteria) yield* joinLateCriteria(); + + // Once per turn, unattended, before anything is judged: each stated + // requirement beside the commands the model ran for it. The checks are + // hidden, so nothing else sends the model back down the task's list. + // Exactly one round; the hidden checks never appear in it. + if (this.cfg.unattended && this.cfg.signOut === true && !signedOut && !opts.ask && requirements.length) { + signedOut = true; + const so = signOut(requirements, this.did); + this.turnSignOut = so; + log?.append("note", { + text: "requirements put to the model for sign-out before judging", + signout: so, + }); + yield { + kind: "info", + text: + `before checking: signing out ${requirements.length} stated requirement(s) — ` + + `${so.unexercised.length} not yet run`, + }; + this.transcript.push({ role: "user", content: signOutMessage(so), molt: { nudge: true } }); + continue; + } + + // Once per turn, before anything is judged: what the work left behind + // that the task did not ask for. A nudge, never a deletion — the model + // knows which of them the task needs, and says so or removes the rest. + if (turnListing && !leftoversAsked && !opts.ask) { + leftoversAsked = true; + const extra = unnamedNewFiles(turnListing, listProject(this.cwd), userText); + if (extra.length) { + const shown = extra.slice(0, 12).join(", ") + (extra.length > 12 ? `, and ${extra.length - 12} more` : ""); + log?.append("note", { text: "files the task does not name, put to the model before judging", files: extra.slice(0, 50) }); + yield { kind: "info", text: `before checking: asking about files the task does not name — ${shown}` }; + this.transcript.push({ + role: "user", + content: + `[Maat: before this is checked — you created files the task does not name: ${shown}. ` + + "If any is a helper script, a copy, a test or scratch file you made for yourself, delete it now: " + + "the task's directory should hold only what the task asks for. If the task needs one, say which in a " + + "line. Then give your final answer again.]", + }); + continue; + } + } if (!bar || bar.checks.length === 0) { yield { @@ -6129,17 +6927,31 @@ export class Engine { continue; } disputed.add(d.name); + const arb = this.cfg.dispute; + const arbModel = arb?.model || this.cfg.model; + const arbUrl = arb?.baseUrl || this.cfg.baseUrl; + if (arbModel === this.cfg.model && arbUrl === this.cfg.baseUrl) { + rejectedNow += 1; + const why = "rejected: no independent arbiter (the worker model would rule on its own misreading; set --arbiter-model)"; + log?.append("dispute", { check: d.name, quote: d.quote, why: d.why, ran: false, skipped: "same model" }); + yield { kind: "info", text: `the dispute of ${d.name} was ${why}; the check stands` }; + this.transcript.push({ + role: "user", + content: `[molt] Your dispute of ${d.name} was ${why}. The check stands. Change the work so it passes.`, + }); + continue; + } yield { kind: "info", text: `reviewing the dispute of ${d.name} against the task text` }; const ruling = await arbitrate({ task: userText, check: { name: d.name, run: check.run }, output: failed.output, dispute: d, - votes: 3, + votes: arb?.votes ?? 1, ask: { - baseUrl: this.cfg.baseUrl, - apiKey: this.cfg.apiKey, - model: this.cfg.model, + baseUrl: arbUrl, + apiKey: arb?.apiKey ?? this.cfg.apiKey, + model: arbModel, cwd: this.cwd, reasoningEffort: this.cfg.review?.reasoningEffort ?? this.cfg.reasoningEffort, fetchFn: this.cfg.fetchFn, @@ -6236,7 +7048,30 @@ export class Engine { checks: barThis.checks.length, names: barThis.checks.map((c) => c.name), }; - const result = await this.runBarGuarded(claim, barThis); + let result = await this.runBarGuarded(claim, barThis); + // A check that failed in its own code is no verdict on the work. The + // reference check says so with exit 3 (src/reference.ts); a drafted + // check says so in its output — a compile error in its own program, a + // tool rejecting its options (criteria.ts checkSelfError). Preflight + // catches these before the work only when the check gets that far: a + // node-summarize check piped `node summarize.js` into a jq program that + // was wrong, failed first on the missing script, and then refused six + // correct solutions. Retired, and the claim judged by what remains. + const selfBroken = brokenOwn(result); + if (selfBroken.length) { + yield* retireBroken(selfBroken); + const rest = barNow(); + if (!rest || rest.checks.length === 0) { + // Nothing else judges the claim: unverified, as when every check is + // retired by dispute — never a pass on an empty bar. + this.turnAllRetired = true; + yield { kind: "info", text: "no other check is left to judge the claim — this claim is unverified." }; + log?.append("session_end", { reason: "unverified: every check retired" }); + if (claim) yield { kind: "assistant_text", text: redact(claim, this.secrets()), streamed: streamedContent }; + return; + } + result = await this.runBarGuarded(claim, rest); + } if (this.checkLeftovers.length) { log?.append("note", { text: `removed what the checks created: ${this.checkLeftovers.join(", ")}` }); yield { kind: "info", text: `removed what the checks created in the project: ${this.checkLeftovers.join(", ")}` }; @@ -6283,18 +7118,62 @@ export class Engine { .map((r) => `${r.name}:${r.output.trim()}`) .join("|"); let stuck = !result.ok && signature === lastFailure; - // Experiment (revealOnStuck): once per turn, a repeat failure of hidden - // drafted checks only is answered by showing their commands, not a stop. + // A repeat failure of hidden drafted checks only is not a verdict: the + // work was right 34 times and wrong 33 at this stop, so it decided the + // outcome on a coin flip. The model is shown the commands once and the + // turn goes on, bounded by the attempt limit; the DISPUTE route stays + // open. A check a person wrote still ends the turn here. const stuckHidden = result.results.filter((r) => !r.ok && !r.advisory); - const revealNow = - stuck && - this.cfg.revealOnStuck === true && - !revealedThisTurn && + const hiddenOnly = stuckHidden.length > 0 && stuckHidden.every((r) => (barNow()?.checks ?? []).find((c) => c.name === r.name)?.hidden === true); + // Opt-in again (`--reveal-stuck`). As the default it was measured on + // Mercury 2.5 against the build that stops here: 37/60 vs 44/60. Shown + // a wrong drafted check, the model bent correct work to fit it — six + // runs passed their checks on a second try and failed the grader + // (0 in the stopping build): a report cut to the check's 6 lines, a + // folder deleted to make "exactly 5 files", git reset --hard. One rescue + // in 120 runs, at three times the tokens. Stopping leaves the work on + // disk, reported unverified, and that work passed 24 of 25 times. + const revealNow = stuck && this.cfg.revealOnStuck === true && !revealedThisTurn && hiddenOnly; + if (stuck && this.cfg.revealOnStuck === true && hiddenOnly) stuck = false; if (revealNow) { revealedThisTurn = true; - stuck = false; + revealSeen = new Set(this.turnLedger().map((e) => e.callId)); + this.turnRevealed = stuckHidden.map((r) => bare(r.name)); + revealMark = { + didAt: this.did.length, + targets: this.revealedTargets( + stuckHidden.flatMap((r) => { + const c = (barNow()?.checks ?? []).find((k) => k.name === r.name); + return c && c.kind === "command" ? [c.run] : []; + }), + ), + }; + } + // The model has read the check; work that then edits the task's input or + // the environment to satisfy it is the check's answer, not the task's. + if (result.ok && revealedThisTurn) { + const bent = this.bentAfterReveal(revealSeen, userText, revealMark); + if (bent && !bent.hard) { + log?.append("note", { text: `unverified after a reveal: ${bent.reason}` }); + yield { kind: "info", text: `${bent.reason} — this claim is unverified.` }; + log?.append("session_end", { reason: "unverified: input changed after a reveal" }); + if (claim) yield { kind: "assistant_text", text: redact(claim, this.secrets()), streamed: streamedContent }; + return; + } + bentReason = bent?.reason; + if (bentReason) { + result = { + ...result, + ok: false, + results: [ + ...result.results, + { name: "bent-work", kind: "builtin", detail: "bent-work", ok: false, output: bentReason, durationMs: 0 }, + ], + }; + log?.append("note", { text: `claim refused after a reveal: ${bentReason}` }); + } } const stuckChecks = stuck ? result.results.filter((r) => !r.ok).map((r) => r.name) @@ -6312,7 +7191,7 @@ export class Engine { !result.ok && !result.cancelled && (result.undetermined?.length ?? 0) > 0 && result.results.every((r) => r.ok || r.advisory || r.skipped); const exhausted = - !result.ok && (undetermined || stuck || allBroken || proofAttempts >= maxAttempts); + !result.ok && (undetermined || stuck || allBroken || bentReason !== undefined || proofAttempts >= maxAttempts); const verdict = result.ok ? "accepted" : undetermined @@ -6359,6 +7238,7 @@ export class Engine { })), cwd: this.cwd, did: [...this.did], + signout: this.turnSignOut, task: taskSeal ? { // Recomputed from the frozen arrays, not carried along, so a @@ -6371,6 +7251,8 @@ export class Engine { notes: [...taskNotes], } : undefined, + ...(this.turnRevealed.length ? { revealed: [...this.turnRevealed] } : {}), + ...(verdict === "accepted" ? { tier: tierOf({ results: result.results, ...this.advisoryTier() }) } : {}), }); log?.append("receipt", { verdict, file: receipt.path, attempt: proofAttempts }); this.bindReceipt(receipt.path, verdict); @@ -6387,23 +7269,28 @@ export class Engine { // wrong. Afterwards it was only a label: on the local comparison the // reviewers named the missing user and the uppercase email, and both // tasks were still lost because the turn had already ended. - if (result.ok && this.cfg.review && !reviewNudged && !opts.ask && lastReceiptPath && existsSync(lastReceiptPath)) { + const skipReview = result.ok && this.cfg.review && !reviewNudged && !opts.ask ? this.reviewSkipReason() : undefined; + if (skipReview) { + this.reviewSkipped = skipReview; + this.turnReview = null; + log?.append("review", { ran: false, skipped: skipReview }); + yield { kind: "info", text: `skipping the independent review: ${skipReview}. The claim stands as checked, unreviewed.` }; + } else if (result.ok && this.cfg.review && !reviewNudged && !opts.ask && lastReceiptPath && existsSync(lastReceiptPath)) { yield { kind: "info", text: "reviewing the claim independently against the task text" }; const review = await reviewClaim({ task: userText, receipt: readFileSync(lastReceiptPath, "utf8"), votes: this.cfg.review.votes, ask: { - baseUrl: this.cfg.baseUrl, - apiKey: this.cfg.apiKey, - model: this.cfg.model, + ...judgeTarget({ baseUrl: this.cfg.baseUrl, apiKey: this.cfg.apiKey, model: this.cfg.model }), cwd: this.cwd, - reasoningEffort: this.cfg.review.reasoningEffort ?? this.cfg.reasoningEffort, + reasoningEffort: judgeEffort(this.cfg.review.reasoningEffort ?? this.cfg.reasoningEffort), fetchFn: this.cfg.fetchFn, - acpSpawn: this.cfg.acpSpawn, + acpSpawn: this.cfg.acpSpawn, + ...(this.timeLeftMs() !== undefined ? { timeoutMs: Math.max(1_000, this.timeLeftMs()!) } : {}), }, }).catch(() => null); - if (review && !review.confirmed && review.violations.length) { + if (review && !review.confirmed && review.violations.length && this.cfg.reviewAdvisory !== true) { reviewNudged = true; this.turnReview = undefined; log?.append("review", { confirmed: false, votes: review.votes, violations: review.violations.length, nudged: true }); @@ -6484,6 +7371,17 @@ export class Engine { yield* this.settleFailed(log); return; } + if (bentReason) { + yield { + kind: "error", + text: + `claim refused: ${bentReason}. The checks passed only after the model read a hidden ` + + `check's command, and the work that met it changed what the task or the environment ` + + `supplies. Maat will not call that verified. Undo that change and meet the task as written.`, + }; + yield* this.settleFailed(log); + return; + } if (stuck) { yield { kind: "info", diff --git a/src/gitbend.ts b/src/gitbend.ts new file mode 100644 index 0000000..890e06e --- /dev/null +++ b/src/gitbend.ts @@ -0,0 +1,57 @@ +/** + * Work bent to a revealed check through git. + * + * After a hidden check's command is shown, the ledger catches the model + * rewriting the check's input or the environment (engine.ts bentAfterReveal). + * It does not see `git add -f tests/check.sh && git commit`, which makes the + * same change durable through bash: one Terminal-Bench run committed the + * fixture the revealed check read and passed on it. The per-turn command + * record (`did`) holds those commands; this reads them. + */ +import { isAbsolute, relative, resolve } from "node:path"; + +/** Words of a shell segment, quotes removed; enough for `git add a "b c"`, not a shell. */ +function words(seg: string): string[] { + const out: string[] = []; + for (const m of seg.matchAll(/"([^"]*)"|'([^']*)'|(\S+)/g)) out.push(m[1] ?? m[2] ?? m[3] ?? ""); + return out; +} + +/** + * The paths a `git add` / `git commit` in `commands` names, relative to `cwd`. + * Commands the engine refused are skipped. A bare `.`, `-A` or `-a` names + * nothing here: it stages whatever changed, and a changed input or + * environment file is already in the ledger. + */ +export function gitPathsStaged(commands: string[], cwd: string): string[] { + const out: string[] = []; + for (const entry of commands) { + const m = /^bash\s+(?:&\s+)?(.*)$/s.exec(entry); + if (!m) continue; + for (const seg of m[1]!.split(/&&|\|\||[;|\n]/)) { + const w = words(seg); + if (w[0] !== "git") continue; + let i = 1; + while (w[i] === "-C" || w[i] === "-c") i += 2; + const verb = w[i]; + if (verb !== "add" && verb !== "commit") continue; + for (let j = i + 1; j < w.length; j += 1) { + const t = w[j]!; + if (/^-[a-zA-Z]*[mFCc]$/.test(t) || t === "--message" || t === "--file") { + j += 1; + continue; + } + if (t.startsWith("-") || t === ".") continue; + const abs = resolve(cwd, t); + const rel = relative(cwd, abs); + out.push(!rel.startsWith("..") && !isAbsolute(rel) ? rel : abs); + } + } + } + return out; +} + +/** Does staging `staged` touch `target` (the same file, or a directory holding it)? */ +export function touches(staged: string, target: string): boolean { + return staged === target || target.startsWith(staged.replace(/\/$/, "") + "/"); +} diff --git a/src/judge.ts b/src/judge.ts new file mode 100644 index 0000000..353dabc --- /dev/null +++ b/src/judge.ts @@ -0,0 +1,31 @@ +/** + * The judge: the model that drafts the hidden checks and reviews the claim. + * By default it is the worker's own model. MAAT_JUDGE_MODEL (with optional + * MAAT_JUDGE_URL / MAAT_JUDGE_KEY) moves it to a different model, so the + * checks are not written with the same blind spots as the work they judge. + */ +import { expandEndpointShorthand } from "./endpoint.js"; + +export type Target = { baseUrl: string; apiKey?: string; model: string }; + +export function judgeTarget(worker: T, env: NodeJS.ProcessEnv = process.env): T { + const model = env.MAAT_JUDGE_MODEL?.trim(); + if (!model) return worker; + const baseUrl = (env.MAAT_JUDGE_URL?.trim() && expandEndpointShorthand(env.MAAT_JUDGE_URL)) || worker.baseUrl; + const apiKey = + env.MAAT_JUDGE_KEY?.trim() || + (baseUrl === worker.baseUrl ? worker.apiKey : undefined) || + // A subscription worker (Grok Build, OpenCode) judged on OpenRouter uses the OpenRouter key. + (/^https:\/\/openrouter\.ai\//.test(baseUrl) ? env.OPENROUTER_API_KEY?.trim() || undefined : undefined); + const { apiKey: _workerKey, ...rest } = worker; + return { ...rest, baseUrl, model, ...(apiKey !== undefined ? { apiKey } : {}) } as T; +} + +/** + * The judge's reasoning effort: MAAT_JUDGE_REASONING (none, minimal, low, ...) overrides the + * effort the checks would otherwise be asked at. A reasoning judge bills its hidden thinking + * as output; capped, a cheap judge stays cheap. + */ +export function judgeEffort(effort: string | undefined, env: NodeJS.ProcessEnv = process.env): string | undefined { + return env.MAAT_JUDGE_REASONING?.trim() || effort; +} diff --git a/src/leftovers.ts b/src/leftovers.ts index e576c2d..30af8fd 100644 --- a/src/leftovers.ts +++ b/src/leftovers.ts @@ -84,3 +84,24 @@ export function excludeMoltFromGit(cwd: string): boolean { return false; } } + +/** + * Files that appeared since `before` which the task text never names — by + * path, by file name, or by a folder they sit in ("sorted/" covers + * sorted/a/b.txt). The likely leftovers: a helper script, a copy, a test + * file the worker made for itself. On local redact-secrets the work was right + * and failed only on the redact.py left beside it, against "create no other + * files"; organize-files the same with organize.py. + */ +export function unnamedNewFiles(before: ProjectListing | null, after: ProjectListing | null, task: string): string[] { + if (!before || !after) return []; + const out: string[] = []; + for (const f of after.files) { + if (before.files.has(f)) continue; + const parts = f.split("/"); + const names = [f, parts.at(-1)!, ...parts.slice(0, -1).map((_, i) => parts.slice(0, i + 1).join("/"))]; + if (names.some((n) => n && task.includes(n))) continue; + out.push(f); + } + return out.sort(); +} diff --git a/src/lenient-json.ts b/src/lenient-json.ts new file mode 100644 index 0000000..4b1b8bc --- /dev/null +++ b/src/lenient-json.ts @@ -0,0 +1,64 @@ +/** + * JSON as models actually write it inside tool arguments. + * + * Two kinds of damage are common and unambiguous, and both appear in long + * writes, where they cost the most: a raw newline or tab inside a string + * (the file's own line breaks, unescaped), and a backslash that starts no + * JSON escape (`\.` from a regex, `\d`, a Windows path). Strict JSON.parse + * refuses both, and on 2026-10-05 Nemotron sent thirty acts in a row whose + * nested action list failed that way — every one answered "could not be + * read", none ever ran, and the task ended with nothing written. + * + * Repaired here: control characters inside strings are escaped, and a lone + * backslash is kept as a literal backslash. Nothing else is guessed: a + * truncated object, a missing quote or brace is still a failure, because + * there the intent is no longer certain. + */ +export function parseLenient(text: string): unknown { + try { + return JSON.parse(text); + } catch (first) { + const fixed = repairJson(text); + if (fixed === text) throw first; + return JSON.parse(fixed); + } +} + +/** Escape control characters inside strings and double backslashes that start no valid escape. */ +export function repairJson(text: string): string { + let out = ""; + let inString = false; + for (let i = 0; i < text.length; i++) { + const c = text[i]!; + if (!inString) { + if (c === '"') inString = true; + out += c; + continue; + } + if (c === "\\") { + const next = text[i + 1] ?? ""; + if (next === "u" && /^[0-9a-fA-F]{4}$/.test(text.slice(i + 2, i + 6))) { + out += text.slice(i, i + 6); + i += 5; + } else if (next !== "" && '"\\/bfnrt'.includes(next)) { + out += c + next; + i += 1; + } else { + out += "\\\\"; + } + continue; + } + if (c === '"') { + inString = false; + out += c; + continue; + } + const code = c.charCodeAt(0); + if (code < 0x20) { + out += c === "\n" ? "\\n" : c === "\r" ? "\\r" : c === "\t" ? "\\t" : `\\u${code.toString(16).padStart(4, "0")}`; + continue; + } + out += c; + } + return out; +} diff --git a/src/localgate.ts b/src/localgate.ts new file mode 100644 index 0000000..31436fb --- /dev/null +++ b/src/localgate.ts @@ -0,0 +1,90 @@ +/** + * One request at a time to a self-hosted model server. + * + * A llama.cpp server on a small machine has one slot: requests queue behind + * each other. Maat sends several at once — the turn's step, the drafter and + * its critic, the reference writers — which on a cloud API saves minutes and + * on one slot only makes each of them wait for the others. On the NUC's + * Qwen3.8-27B the turn's first request sat behind a drafter that thought for + * 5,700 tokens at 10 tokens/s, got no byte for five minutes, was declared + * hung, cancelled and sent again to the back of the queue; four times, then + * the task failed with zero steps. Every local task did. + * + * So requests to a self-hosted endpoint take turns here, in order, and the + * caller starts its hang watchdog only once it holds the turn: waiting in + * Maat's own queue is not the server being hung. + */ +import { env } from "./env.js"; +import { isSelfHosted } from "./providers.js"; + +type Waiter = { go: () => void }; + +const queues = new Map(); + +/** + * How many requests the self-hosted server serves at once: `MAAT_LOCAL_SLOTS` + * (llama.cpp's `--parallel`), else 1. + */ +function slots(): number { + const n = Number(env("LOCAL_SLOTS")); + return Number.isInteger(n) && n > 0 ? n : 1; +} + +function keyOf(baseUrl: string): string { + try { + const u = new URL(baseUrl); + return `${u.protocol}//${u.host}`; + } catch { + return baseUrl; + } +} + +/** + * Wait for this endpoint's turn. Returns the release function (safe to call + * more than once). For an endpoint that is not self-hosted it returns at once: + * a cloud API serves requests side by side. + */ +export async function takeTurn(baseUrl: string, signal?: AbortSignal): Promise<() => void> { + if (!isSelfHosted(baseUrl)) return () => {}; + // Already cancelled: no turn to wait for, and an abort event that already + // fired would never wake the waiter below. + if (signal?.aborted) throw signal.reason ?? new Error("aborted"); + const key = keyOf(baseUrl); + let q = queues.get(key); + if (!q) queues.set(key, (q = { busy: 0, waiting: [] })); + const queue = q; + if (queue.busy >= slots()) { + // A released slot is handed straight to the next waiter (see below), so + // a newcomer can never slip in between and put two on one slot. + await new Promise((resolve, reject) => { + const w: Waiter = { go: resolve }; + queue.waiting.push(w); + signal?.addEventListener( + "abort", + () => { + const i = queue.waiting.indexOf(w); + if (i !== -1) { + queue.waiting.splice(i, 1); + reject(signal.reason ?? new Error("aborted")); + } + }, + { once: true }, + ); + }); + } else { + queue.busy += 1; + } + let released = false; + return () => { + if (released) return; + released = true; + const next = queue.waiting.shift(); + if (next) next.go(); + else queue.busy -= 1; + }; +} + +/** How many callers are waiting for this endpoint now. Tests only. */ +export function waitingFor(baseUrl: string): number { + return queues.get(keyOf(baseUrl))?.waiting.length ?? 0; +} diff --git a/src/mission.ts b/src/mission.ts index 1565c8d..abeae4d 100644 --- a/src/mission.ts +++ b/src/mission.ts @@ -9,7 +9,7 @@ * feature in a milestone is done, every assertion in that milestone is run * again, together, and the milestone seals only if all of them still pass. * - * What is deliberately NOT here: an orchestrator model. Factory's missions + * What is deliberately NOT here: an orchestrator model. A competitor's missions * put a frontier model in a loop with seventeen tools to decide what runs * next and whether a worker's handoff was good; it is the most expensive * process in the system and it is doing a job a loop can do. Here the diff --git a/src/opencode.ts b/src/opencode.ts new file mode 100644 index 0000000..08b5b56 --- /dev/null +++ b/src/opencode.ts @@ -0,0 +1,208 @@ +/** + * OpenCode as a question-answerer, so its free "Big Pickle" model (or any + * `opencode/`) can be the judge: the model that drafts the hidden + * checks and reviews the claim. `MAAT_JUDGE_URL=opencode://subscription`. + * + * Only the real `opencode` CLI is driven. OpenCode's free tier refuses direct + * HTTP ("can only be used from within OpenCode"), and nothing here imitates + * the CLI's requests to get around that. + * + * Facts measured on opencode 1.18.33, not read off a doc page: + * + * - `opencode run -m --format json ` prints one JSON + * event per line: `step_start`, `text` (`part.text`), `tool_use`, + * `step_finish` (`part.tokens`), or `error` (`error.data.message`, + * `statusCode`). It must be spawned with stdin ignored; with an inherited + * pipe it never exits. + * - There is no system-prompt flag, so the system text rides in the message. + * - **Denying tools by config breaks the free tier.** `permission: {"*": + * "deny"}` (or denying bash/edit alone) is answered 403 "free tier can only + * be used from within OpenCode": the request no longer carries the tool + * list the server expects. `"ask"` keeps the request intact, and a headless + * `run` auto-rejects every ask (measured: bash, write, webfetch and + * external_directory all rejected, no file created). So tools stay offered + * and none can run. A rejected tool ends the turn, which is reported as such. + * - The project directory comes from `PWD` as well as the cwd, so both are set + * to the empty temp directory, and `--dir` is passed too. + */ +import { spawn } from "node:child_process"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { errorText } from "./format.js"; + +export { OPENCODE_URL } from "./endpoint.js"; + +export function isOpencode(baseUrl: string | undefined): boolean { + return (baseUrl ?? "").trim().toLowerCase().startsWith("opencode://"); +} + +/** Every permission "ask", which a headless run rejects. See the file comment. */ +export const OPENCODE_CONFIG = JSON.stringify({ + permission: { + "*": "ask", + bash: "ask", + edit: "ask", + webfetch: "ask", + websearch: "ask", + task: "ask", + external_directory: "ask", + doom_loop: "ask", + }, +}); + +/** `opencode/big-pickle`, or a bare `big-pickle` meaning the same. */ +export function opencodeModel(model: string): string { + const m = model.trim(); + return m.includes("/") ? m : `opencode/${m}`; +} + +export function opencodeArgs(model: string, message: string, dir: string): string[] { + return ["run", "-m", opencodeModel(model), "--format", "json", "--dir", dir, message]; +} + +export function opencodeEnv(dir: string, base: NodeJS.ProcessEnv = process.env): NodeJS.ProcessEnv { + return { ...base, PWD: dir, OPENCODE_CONFIG_CONTENT: OPENCODE_CONFIG }; +} + +export type OpencodeReply = { ok: true; text: string } | { ok: false; error: string; transient?: true }; + +/** The answer in a `--format json` stream, or why there is none. */ +export function parseOpencodeEvents(stdout: string): OpencodeReply { + const texts: string[] = []; + let error: { message: string; status?: number } | undefined; + let tool: string | undefined; + for (const line of stdout.split("\n")) { + const t = line.trim(); + if (!t.startsWith("{")) continue; + let ev: { + type?: string; + part?: { text?: string; tool?: string }; + error?: { name?: string; message?: string; data?: { message?: string; statusCode?: number } }; + }; + try { + ev = JSON.parse(t); + } catch { + continue; + } + if (ev.type === "text" && typeof ev.part?.text === "string") texts.push(ev.part.text); + else if (ev.type === "tool_use" && ev.part?.tool) tool ??= ev.part.tool; + else if (ev.type === "error" && ev.error) { + error ??= { + message: ev.error.data?.message ?? ev.error.message ?? ev.error.name ?? "unknown error", + ...(ev.error.data?.statusCode !== undefined ? { status: ev.error.data.statusCode } : {}), + }; + } + } + const text = texts.join("").trim(); + if (text) return { ok: true, text }; + if (error) return { ok: false, ...opencodeError(error.message, error.status) }; + if (tool) return { ok: false, error: `OpenCode tried its ${tool} tool instead of answering, and no tool is allowed here` }; + return { ok: false, error: "OpenCode returned nothing" }; +} + +/** + * A provider error as a message a person can act on. Rate and usage limits + * keep the words `rate limit` so `longQuotaText` / harnesses recognise them; + * 429/5xx are marked transient so askModel backs off and asks again. + */ +export function opencodeError(message: string, status?: number): { error: string; transient?: true } { + const m = message.replace(/\s+/g, " ").slice(0, 300); + if (/free tier can only be used/iu.test(m)) { + return { error: `OpenCode refused the request (${m}) — it must run through the opencode CLI unmodified; check \`opencode run -m opencode/big-pickle hi\` by hand` }; + } + if (status === 429 || /rate.?limit|quota|too many requests|usage limit|limit (?:reached|exceeded)/iu.test(m)) { + const long = /resets?\s+in|per day|daily|monthly|usage limit|quota/iu.test(m); + return { + error: `OpenCode rate limit${status ? ` (HTTP ${status})` : ""}: ${m}`, + ...(long ? {} : { transient: true as const }), + }; + } + if (status !== undefined && status >= 500) return { error: `OpenCode provider error (HTTP ${status}): ${m}`, transient: true }; + if (status === 401 || /not logged in|unauthori[sz]ed|credential/iu.test(m)) { + return { error: `OpenCode is not signed in (${m}) — run \`opencode auth login\`` }; + } + return { error: `OpenCode: ${m}` }; +} + +export type OpencodeAskOptions = { + model: string; + systemPrompt: string; + prompt: string; + timeoutMs?: number; + /** Injected in tests. Real callers spawn the CLI. */ + run?: (cmd: string, args: string[], opts: object) => Promise<{ stdout: string }>; +}; + +export const STALL_MS = 90_000; + +const NO_TOOLS = + "Answer in plain text only. Do not use any tools, do not read or write files, and do not run commands: " + + "everything you need is in this message."; + +function runCli(cmd: string, args: string[], opts: { cwd: string; env: NodeJS.ProcessEnv; timeoutMs: number }): Promise<{ stdout: string }> { + return new Promise((resolve, reject) => { + // stdin must be ignored: `opencode run` waits on an open pipe forever. + const child = spawn(cmd, args, { cwd: opts.cwd, env: opts.env, stdio: ["ignore", "pipe", "pipe"] }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (d: Buffer) => (stdout += d)); + child.stderr.on("data", (d: Buffer) => (stderr += d)); + const timer = setTimeout(() => child.kill("SIGKILL"), opts.timeoutMs); + child.on("error", (e) => { + clearTimeout(timer); + reject(e); + }); + child.on("close", (code, signal) => { + clearTimeout(timer); + // A non-zero exit still carries the error event on stdout; the caller reads it. + if (signal) reject(new Error(`opencode did not answer within ${Math.round(opts.timeoutMs / 1000)}s`)); + else resolve({ stdout: stdout || (code ? stderr : "") }); + }); + }); +} + +/** + * One question to OpenCode, in an empty temp directory, every tool refused. + * The ask is retried once when the model reached for a tool instead of + * answering, since that is the only way a free model "fails" a plain question. + */ +export async function opencodeAsk(opts: OpencodeAskOptions): Promise { + const injected = opts.run; + const dir = injected ? tmpdir() : mkdtempSync(join(tmpdir(), "molt-opencode-ask-")); + const message = `${opts.systemPrompt}\n\n${NO_TOOLS}\n\n${opts.prompt}`; + // Big Pickle usually answers in 5-25 s but now and then stalls for minutes (measured: one + // 300 s hang in a draft that otherwise took 2 min). A stalled attempt is cut at 90 s and asked + // again once, rather than spending the whole allowance on one dead request. + const timeoutMs = Math.min(opts.timeoutMs ?? 180_000, STALL_MS); + try { + let last: OpencodeReply = { ok: false, error: "OpenCode returned nothing" }; + for (let attempt = 0; attempt < 2; attempt++) { + let stdout: string; + try { + ({ stdout } = injected + ? await injected("opencode", opencodeArgs(opts.model, message, dir), { cwd: dir, env: opencodeEnv(dir) }) + : await runCli("opencode", opencodeArgs(opts.model, message, dir), { cwd: dir, env: opencodeEnv(dir), timeoutMs })); + } catch (e) { + const em = errorText(e); + if (/ENOENT/u.test(em)) return { ok: false, error: "the opencode CLI is not installed (npm i -g opencode-ai, or brew install sst/tap/opencode)" }; + // execFile-style errors carry the stream; try to read it before giving up. + const out = (e as { stdout?: unknown }).stdout; + if (typeof out === "string" && out) { + const parsed = parseOpencodeEvents(out); + if (!parsed.ok) return parsed; + } + if (/did not answer within/u.test(em) && attempt === 0) { + last = { ok: false, error: em }; + continue; + } + return { ok: false, error: em }; + } + last = parseOpencodeEvents(stdout); + if (last.ok || !/tried its .* tool/u.test(last.error)) return last; + } + return last; + } finally { + if (!injected) rmSync(dir, { recursive: true, force: true }); + } +} diff --git a/src/providers.ts b/src/providers.ts index 474f93f..bd9928f 100644 --- a/src/providers.ts +++ b/src/providers.ts @@ -11,7 +11,9 @@ * tool whose whole pitch is an auditable record never writes a credential * into one. */ +import { env } from "./env.js"; import { ACP_AGENTS, acpAgentFor, isAcp } from "./acp.js"; +import { isOpencode } from "./opencode.js"; import { chmodSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { windowAround } from "./commands.js"; @@ -309,6 +311,7 @@ export function anthropicPricing(model: string): Pricing | null { * invent one, for a run that costs no money at all. */ export function planFor(baseUrl: string): string | undefined { + if (isOpencode(baseUrl)) return "OpenCode"; return acpAgentFor(baseUrl)?.label; } @@ -467,16 +470,27 @@ export function isSelfHosted(baseUrl: string): boolean { * wrong for a frontier model: with the map it won 3 of 3 paired runs and * cost 23% less. Said first, before the address is parsed at all. */ - if (isAcp(baseUrl)) return false; + if (isAcp(baseUrl) || isOpencode(baseUrl)) return false; let host = ""; try { host = new URL(baseUrl).hostname.toLowerCase(); } catch { return false; } + // Said outright, for a private name no rule below can know. + const told = env("SELF_HOSTED"); + if (told === "1" || told === "0") return told === "1"; if (host === "localhost" || host.endsWith(".localhost") || host.endsWith(".local")) return true; + // Private names. host.docker.internal is how a container reaches its own + // Mac: the local benchmark ran Qwen there for a night with every + // self-hosted rule off — thinking on (10,000 thought events, no step taken + // in 14 minutes), no one-at-a-time gate, cloud timeouts. A Tailscale name + // or address is a machine on the owner's own network too. + if (/\.(internal|lan|home\.arpa|ts\.net)$/.test(host) || host === "host.containers.internal") return true; // IPv6 loopback, with or without the brackets a URL puts round it. if (host === "::1" || host === "[::1]") return true; + // IPv6 unique-local (fc00::/7), which Tailscale also hands out. + if (/^\[?f[cd][0-9a-f]{2}:/.test(host)) return true; const v4 = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/.exec(host); if (v4) { const [a, b] = [Number(v4[1]), Number(v4[2])]; @@ -485,6 +499,7 @@ export function isSelfHosted(baseUrl: string): boolean { if (a === 192 && b === 168) return true; // 192.168.0.0/16 if (a === 172 && b >= 16 && b <= 31) return true; // 172.16.0.0/12 if (a === 169 && b === 254) return true; // link-local + if (a === 100 && b >= 64 && b <= 127) return true; // 100.64.0.0/10: Tailscale, carrier-grade NAT return false; } // A bare name with no dots is a LAN hostname, not a public one. @@ -622,3 +637,51 @@ export function windowRows( return windowAround(rows, index, size).map(({ item, i }) => ({ row: item, i })); } + +/** + * The extra request fields that turn a self-hosted model's thinking off, for + * a configured reasoning effort. Empty for every other endpoint and effort. + * + * `reasoning: { effort }` is OpenRouter's shape and llama.cpp ignores it. On + * the NUC's Qwen3.8-27B (about 10 tokens/s) one drafting question thought for + * 5,700 tokens, ten minutes of the server's only slot, whatever `--reasoning + * none` said. llama.cpp's OpenAI endpoint does take `chat_template_kwargs` per + * request, and `{"enable_thinking": false}` is the switch the Qwen3-family + * chat templates read. That is all this sends, and only for `none`: + * + * - `reasoning_effort` is not sent. Whether a given llama.cpp build reads it + * is not established, and a field a server does not know is a risk for no + * measured gain. + * - `low` is not mapped to anything. "Think a little" has no template switch + * here; turning thinking off outright would be stronger than what was + * asked, so `low` keeps the model's own thinking and the ceiling on the + * answer is what bounds it. + * - A cloud endpoint gets nothing: its bodies stay exactly as they were. + * + * The caller still sends `reasoning: { effort }` as before; this is added to it. + */ +export function selfHostedThinking(baseUrl: string, effort: string | undefined): { chat_template_kwargs?: { enable_thinking: false } } { + return effort === "none" && isSelfHosted(baseUrl) ? { chat_template_kwargs: { enable_thinking: false } } : {}; +} + +/** + * OpenRouter provider pinning, per model: MAAT_OPENROUTER_PROVIDER="minimax/minimax-m3=gmicloud/fp8" + * (several separated by ";"). The pinned model is sent only to that provider, with no fallback, + * so a cheaper or better endpoint is not silently swapped for another. Other models, such as a + * judge on the same key, route as usual. + */ +export function openRouterProvider( + baseUrl: string, + model: string, + spec: string | undefined = process.env.MAAT_OPENROUTER_PROVIDER, +): { provider?: { only: string[]; allow_fallbacks: false } } { + if (!spec || !/^https:\/\/openrouter\.ai\//.test(baseUrl)) return {}; + for (const pair of spec.split(";")) { + const at = pair.lastIndexOf("="); + if (at <= 0) continue; + const m = pair.slice(0, at).trim(); + const only = pair.slice(at + 1).split(",").map((s) => s.trim()).filter(Boolean); + if (m === model && only.length) return { provider: { only, allow_fallbacks: false } }; + } + return {}; +} diff --git a/src/receipts.ts b/src/receipts.ts index 5340fd1..045a67b 100644 --- a/src/receipts.ts +++ b/src/receipts.ts @@ -55,6 +55,8 @@ export type ReceiptRecord = { * questions before this field existed. */ ask?: boolean; + /** Set when the model was stopped and the tree judged as it stood. */ + endedBy?: "deadline" | "provider"; /** True when the cost rests on molt's own token estimate anywhere in the session. */ costEstimated?: boolean; /** @@ -70,6 +72,19 @@ export type ReceiptRecord = { /** The commit the judged tree sat on; `dirty` when it differed from it. */ head?: string; dirty?: boolean; + /** Hidden checks whose commands were shown to the model after a repeat failure. */ + revealed?: string[]; + /** + * What the passing checks earned (src/tiers.ts): "verified", or + * "passed-checks" with `tierReason` saying why not. Set on accepted claims; + * the independent review can lower it after the receipt is written + * (`amendTier`), so the index row is the last word and the receipt file, + * hash-bound when it was written, is never rewritten. + */ + tier?: "verified" | "passed-checks"; + tierReason?: string; + /** The strongest evidence class among the passing checks. */ + evidence?: string; }; /** What `repair()` changed, and what it left alone. */ @@ -293,8 +308,12 @@ export class Receipts { costUsd?: number; /** True when that figure rests on molt's own token estimate. */ costEstimated?: boolean; + /** The tier the passing checks earned, before any independent review (src/tiers.ts). */ + tier?: { tier: "verified" | "passed-checks"; reason?: string; evidence: string }; /** True for a question: the bar ran advisory and could not refuse. */ ask?: boolean; + /** The model was stopped (clock, provider) and the tree was judged as it stood. */ + endedBy?: "deadline" | "provider"; /** * Every file the turn changed, with the hashes that prove it — and which * lines it wrote, so the receipt can show the work rather than describe it. @@ -312,8 +331,16 @@ export class Receipts { * work out which is which. */ task?: { seal: string; checks: string[]; notes: string[] }; + /** + * Hidden checks whose commands were shown to the model after they failed + * the same way twice. A pass after that is a pass against a check the + * model had read, and the receipt says so. + */ + revealed?: string[]; /** What the model ran and read, in order, as one line each. */ did?: string[]; + /** The requirement sign-out put to the model before this claim (src/signout.ts). */ + signout?: { requirements: string[]; matched: { requirement: string; calls: string[] }[]; unexercised: string[] }; /** * The commit the tree sat on when the bar ran, and whether the tree * differed from it. @@ -335,7 +362,7 @@ export class Receipts { // believe it finished?" — so it answers in that order. It used to open // with a provider name and a token count, which answer neither question, // and put the work itself nowhere at all. - const verdictLine = + let verdictLine = args.verdict === "accepted" && args.ask ? "Maat recorded this answer. A question runs the bar advisory — a turn that wrote " + "nothing cannot have broken anything — so no check could refuse it, and nothing " + @@ -349,6 +376,15 @@ export class Receipts { "done.yml requires were not run. Nothing failed, and nothing established the rest." : "Maat reported failure: the attempt limit was reached with checks still failing."; + if (args.verdict === "accepted" && !args.ask && args.tier) { + verdictLine += args.tier.tier === "verified" + ? `\n\nEvidence: ${args.tier.evidence}. A passing check of this class earns the word "verified".` + : `\n\nPassed its checks, not verified: ${args.tier.reason}. The strongest passing check is ${args.tier.evidence}.`; + } + if (args.revealed?.length) { + verdictLine += `\n\nThe command of ${args.revealed.map((n) => `\`${n}\``).join(", ")} was shown to the model after it failed the same way twice; the work was judged against a check the model had read.`; + } + const changed = args.changed ?? []; // The task's own criteria go above what changed, because they are what the // change was supposed to achieve — a reader who sees the diff first has @@ -399,6 +435,21 @@ export class Receipts { work.push(...wroteSection(args.cwd ?? process.cwd(), changed)); } + const so = args.signout; + if (so && so.requirements.length) { + work.push("## Requirement sign-out", ""); + work.push( + "Before this claim was judged, each stated requirement was put to the model beside the", + "commands it ran that touch it. Matching is by keyword and path, not proof.", + "", + ); + for (const r of so.requirements) { + const calls = so.matched.find((m) => m.requirement === r)?.calls; + work.push(calls ? `- "${r}" — ran: ${calls.slice(-3).map((c) => `\`${c.slice(0, 110)}\``).join("; ")}` : `- "${r}" — not run before the sign-out`); + } + work.push(""); + } + const did = args.did ?? []; if (did.length > 0) { work.push("## What the model ran", ""); @@ -412,6 +463,14 @@ export class Receipts { "", verdictLine, "", + ...(args.endedBy + ? [ + args.endedBy === "deadline" + ? "The turn's time budget ran out before the model said it was done; the sealed checks ran on the tree as it stood." + : "The provider failed after work had been done; the sealed checks ran on the tree as it stood.", + "", + ] + : []), "## What the model claimed", "", "> " + (args.claim.trim() || "(no final message)").split("\n").join("\n> "), @@ -516,6 +575,8 @@ export class Receipts { `- attempt: ${args.attempt}`, `- provider: ${args.provider}`, `- model: ${args.model}`, + // Who wrote and reviewed the checks, when that is not the worker (judge.ts). + ...(process.env.MAAT_JUDGE_MODEL?.trim() ? [`- judge: ${process.env.MAAT_JUDGE_MODEL.trim()}${process.env.MAAT_JUDGE_URL?.trim() ? ` at ${process.env.MAAT_JUDGE_URL.trim()}` : ""}`] : []), `- session tokens: ${args.sessionTokens}`, ...(args.costUsd === undefined ? [] @@ -551,6 +612,11 @@ export class Receipts { ...(args.costUsd === undefined ? {} : { costUsd: args.costUsd }), ...(args.costEstimated ? { costEstimated: true } : {}), ...(args.ask ? { ask: true } : {}), + ...(args.revealed?.length ? { revealed: [...args.revealed] } : {}), + ...(args.endedBy ? { endedBy: args.endedBy } : {}), + ...(args.verdict === "accepted" && !args.ask && args.tier + ? { tier: args.tier.tier, evidence: args.tier.evidence, ...(args.tier.reason ? { tierReason: args.tier.reason } : {}) } + : {}), // Only when the caller said what changed. A row with no count is // unknown, and unknown is not zero — it is counted as a change, which is // what every row written before this field existed already was. @@ -567,6 +633,40 @@ export class Receipts { return { path: p, attempt: args.attempt, verdict: args.verdict }; } + /** + * Lower a receipt's recorded tier once the whole turn has been judged. + * + * The independent review reads the receipt, so it cannot be in it: a claim + * that passed on strong checks and was then contradicted would otherwise sit + * in the index as "verified". Only the index row changes; the receipt file + * is hash-bound and stays byte for byte as written. A row that is not + * there, or an index that cannot be read, is left alone. + */ + amendTier(file: string, tier: { tier: "verified" | "passed-checks"; reason?: string }): boolean { + try { + if (!existsSync(this.indexPath)) return false; + let hit = false; + const out = readFileSync(this.indexPath, "utf8") + .split("\n") + .map((l) => { + if (!l.trim()) return l; + try { + const r = JSON.parse(l) as ReceiptRecord; + if (r.file !== file) return l; + hit = true; + const { tierReason: _drop, ...rest } = r; + return JSON.stringify({ ...rest, tier: tier.tier, ...(tier.reason ? { tierReason: redact(tier.reason, this.secrets) } : {}) }); + } catch { + return l; + } + }); + if (hit) writeFileSync(this.indexPath, out.join("\n"), "utf8"); + return hit; + } catch { + return false; + } + } + /** * Reconcile the index against the files on disk. * diff --git a/src/reference.ts b/src/reference.ts new file mode 100644 index 0000000..8d55042 --- /dev/null +++ b/src/reference.ts @@ -0,0 +1,416 @@ +/** + * The reference check: an answer computed independently of the work. + * + * Drafted checks share the worker's misreadings. Asked to check cron-next, + * the drafter wrote `nextrun.py '0 12 1 * 1' ... | grep -q 2024-02-01` — a + * hand-worked answer, and a wrong one (the 8th is a Monday) — and on another + * draft no edge case at all, so `5-10/2` counted from zero and a leap-day + * search that gave up both sailed through as "verified". A check written by + * the party that misunderstood the task repeats the misunderstanding. + * + * So a second, fresh model reads only the task text and the project as it was + * before any work, and writes the most literal reference it can — brute force, + * no cleverness — with a harness that compares the deliverable to it on the + * task's examples, the edge cases its words imply, and seeded random inputs. + * Every expected value comes from running the reference, never from working + * an answer out by hand. On local tasks this caught every wrong cron-next + * solution with the input, the expected and the actual result, and a + * correct-by-the-grader one that broke a rule the grader never tested. + * + * One reference alone was not enough. On 20 local tasks it caught every + * wrong solution it judged, and refused 29 of 70 correct ones: its own bugs + * (Sunday off by one, a server's state) and behaviour the task never decided + * (symlinks, an error for ".5 KB"). A reference is a program written by a + * model, and it shares a model's failure rate. So there are two: a second + * reviewer who sees the task, the files and what an input is — never the + * first reviewer's code or answers — writes its own reference(), and Maat's + * own driver (never written by a model) judges the deliverable only on + * inputs where both agree. A pair that mostly disagrees is dropped whole. + * + * Its guarantees, and where each comes from: + * - It predates the work: it is built from the task text and a snapshot of + * the project taken before the first step, and only ever reads that + * snapshot. So it may join the sealed checks later than the drafted ones + * without the work having had any say in it. + * - Its own bugs are not the work's fault: the driver computes every + * expected value before running the deliverable, and exits 3 when a + * reference fails, the pair disagrees, or run_deliverable raises (it is + * told never to). Tried once on the untouched snapshot, a 3 there drops + * it; a 3 at a claim retires it (engine.ts). + * - It is withheld from the model like every drafted check, and lives + * outside the project, so a task that walks the tree never meets it. + */ +import { createHash } from "node:crypto"; +import { cpSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, relative } from "node:path"; +import { askModel, jsonIn, type AskOptions } from "./ask.js"; +import { runCommand } from "./run.js"; +import { STATE_DIRS } from "./statedir.js"; +import type { Check } from "./types.js"; + +export const REFERENCE_CHECK_NAME = "reference"; +/** The exit code a reference harness uses for its own failure. */ +export const REFERENCE_SELF_ERROR = 3; +/** A snapshot larger than this is not taken, and no reference is offered. */ +export const REFERENCE_MAX_SNAPSHOT_BYTES = 64 * 1024 * 1024; +export const REFERENCE_MAX_SNAPSHOT_FILES = 5_000; +/** Room for a whole program in the reply, plus a reasoning model's thinking. */ +export const REFERENCE_MAX_TOKENS = 16_000; +export const REFERENCE_TIMEOUT_MS = 120_000; + +const SKIP = new Set([...STATE_DIRS, ".git", "node_modules", "__pycache__", ".venv", "venv", ".tox", "target", "dist"]); + +export const REFERENCE_SYSTEM = [ + "You write an independent reference check for one coding task. Someone else does", + "the task; you never see their work. Your check runs after they finish and decides", + "whether their result is right. A second reviewer, who never sees your code,", + "independently writes a reference for the same inputs, and only inputs where both", + "references agree are used to judge the work.", + "", + "The danger you exist to remove: the person doing the task can misread it, and any", + "check they write repeats their misreading. So you work from the task's own words", + "only, and you write the most literal, obviously-correct version of the answer you", + "can — brute force, enumerate everything, no clever shortcuts, no optimisation,", + "every rule in the task applied exactly as written. Slow is fine (under 60 seconds", + "for all inputs together). Clever is how references go wrong.", + "", + "FIRST decide whether a reference applies. It applies when the right result can be", + "COMPUTED from the task text and inputs present in the project: a program or", + "function with a stated interface, a command whose output is specified, a file", + "transform, a report or query over files that are here. It does NOT apply when the", + "answer has to be discovered, searched for, trained, measured on hidden data,", + "depends on the network, or is a judgement call; or when the task is only about", + "repository or system state that a plain command checks directly; or when the task", + "does not name exactly how the result is used (a command line, an import path, an", + "output file). When it does not apply, say so — a wrong reference is worse than none.", + "", + "When it applies, write check.py, a Python 3 module (standard library only) that", + "Maat's driver imports. It defines:", + "", + ' INPUT_FORMAT = "one line: what one input is" # shown to the second reviewer', + ' OUTPUT_FORMAT = "one line: what reference() returns" # shown to the second reviewer', + " INPUTS = [...] # JSON-serialisable test inputs", + " def reference(inp): ... # the right result for inp, from the task text", + " def run_deliverable(inp): ... # the deliverable's result for inp, same shape", + " def same(expected, actual): ... # optional; default is equality", + "", + "INPUTS: every example the task gives; every edge case the task's words DECIDE —", + "boundaries of each stated range, each kind of item the task lists, with values", + "chosen to expose an off-by-one or a wrong starting point, calendar edges if dates", + "are involved, and empty input, duplicates, mixed case, whitespace or CRLF only if", + "the task speaks of such data; and a few seeded random ones (random.seed(1)). An", + "edge case is in scope only when the task's words decide its answer: if you would", + "have to choose how the deliverable should behave (a symlink, a malformed value,", + "an error the task never mentions), leave it out. Every input must be one the", + "task's own description allows — a line never contains a newline, a value stated", + "to be in a range stays in it.", + "", + "reference(inp): never hard-codes an answer you worked out by hand — compute it.", + "When the task says to use something the project already has (a script, a", + "library, a data file, a converter), call that very thing — its copy under the", + "folder named by the environment variable REFERENCE_BEFORE — never a rewrite.", + "Brute force may need to go far (an answer can be years of minutes away): never", + "cap a search so low that a legal input runs out.", + "", + "run_deliverable(inp): runs the deliverable exactly as the task says it is used", + "(the command line, import path or output file the task names), with exactly", + "what reference() takes — the same argument strings, stdin, files and working", + "directory — so anything the output echoes back (a file name as given, a path)", + "is the same string for both. It runs in the project folder (the current", + "directory). It must NEVER raise: if the deliverable is missing, crashes or times", + "out (give each run a timeout), return a short string saying what happened.", + "Return the result in the same shape reference() does, keeping only what the", + "task specifies: strip trailing whitespace at line ends and the final newline;", + "do not keep formatting, ordering or extra output the task does not state (if the", + "order does not matter, return a sorted list).", + "", + "Tasks that change files IN PLACE: the originals, as they were before any work,", + "are under REFERENCE_BEFORE (same relative paths); read inputs from there and the", + "deliverable's results from the project. Ignore Maat's own folders .maat/ and", + ".molt/ wherever you list or compare files. Write temporary files only under", + "tempfile.mkdtemp(), never into the project.", + "", + "Reply with one line of JSON, and when it applies the whole module after it in a", + "fenced python block — never inside the JSON:", + '{"applies": true, "reason": "one line"}', + "```python", + "...check.py...", + "```", + "or only", + '{"applies": false, "reason": "one line"}', +].join("\n"); + +/** The second, independent reviewer: one function, from the task and the input format only. */ +export const SECOND_REFERENCE_SYSTEM = [ + "You independently compute the right answers for one coding task, for a check that", + "compares two references written by reviewers who never see each other's code.", + "Inputs where the two disagree are not used, so a careful literal answer matters", + "more than a clever one.", + "", + "Write second.py, a Python 3 module (standard library only) defining", + " def reference(inp): ...", + "returning the right result for one input, computed from the task text: brute", + "force, every rule applied exactly as written, nothing worked out by hand. You are", + "told what an input is and what shape the result takes; match that shape exactly.", + "When the task says to use something the project already has (a script, a", + "library, a data file), call that very thing — its copy under the folder named by", + "the environment variable REFERENCE_BEFORE — never a rewrite. Files the task", + "works on are under REFERENCE_BEFORE as they were before any work. Ignore .maat/", + "and .molt/. Write temporary files only under tempfile.mkdtemp().", + "", + "Reply with the module only, in one fenced python block.", +].join("\n"); + +/** + * Maat's own driver: never written by a model. It computes both references on + * every input before the deliverable runs; judges the deliverable only where + * they agree; and exits 3 — the check's own failure, never the work's — when + * a reference fails, the two mostly disagree, or run_deliverable raises. + */ +export const DRIVER = [ + "import importlib.util, json, os, sys, traceback", + "HERE = os.path.dirname(os.path.abspath(__file__))", + "def load(name):", + " spec = importlib.util.spec_from_file_location(name, os.path.join(HERE, name + '.py'))", + " m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m); return m", + "def key(v):", + " try: return json.dumps(v, sort_keys=True, default=repr)", + " except Exception: return repr(v)", + "def own(msg):", + " sys.stdout.flush(); sys.stderr.write('REFERENCE ERROR: ' + msg + '\\n'); sys.exit(3)", + "try:", + " a = load('check'); b = load('second'); inputs = list(a.INPUTS)[:200]", + "except SystemExit: raise", + "except BaseException:", + " traceback.print_exc(); own('a reference could not be loaded')", + "if not inputs: own('the reference defines no inputs')", + "agreed, disagreed = [], []", + "for x in inputs:", + " try: ea = a.reference(x)", + " except BaseException: traceback.print_exc(); own('the first reference failed on input %r' % (x,))", + " try: eb = b.reference(x)", + " except BaseException as e: eb = ('', repr(e))", + " (agreed if key(ea) == key(eb) else disagreed).append((x, ea, eb))", + "need = len(inputs) if len(inputs) < 3 else max(3, (len(inputs) + 1) // 2)", + "if len(agreed) < need:", + " x, ea, eb = disagreed[0]", + " own('the two independent references agree on only %d of %d inputs; first disagreement, input %r: %r vs %r' % (len(agreed), len(inputs), x, ea, eb))", + "same = getattr(a, 'same', None)", + "for x, e, _ in agreed:", + " try: got = a.run_deliverable(x)", + " except BaseException: traceback.print_exc(); own('run_deliverable raised on input %r' % (x,))", + " try: ok = same(e, got) if same else key(e) == key(got)", + " except BaseException: traceback.print_exc(); own('the comparison failed on input %r' % (x,))", + " if not ok:", + " print('input: %r\\nexpected: %r\\nactual: %r\\n(two independent references agree on the expected result)' % (x, e, got)); sys.exit(1)", + "print('%d inputs match two independent references%s' % (len(agreed), '; %d where they disagreed were not judged' % len(disagreed) if disagreed else ''))", + "", +].join("\n"); + +/** What the second reviewer is told about the inputs: their format and a few of them, never an answer. */ +export const PROBE = [ + "import importlib.util, json, os, sys", + "spec = importlib.util.spec_from_file_location('check', sys.argv[1])", + "m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m)", + "print(json.dumps({'input': str(getattr(m, 'INPUT_FORMAT', '')), 'output': str(getattr(m, 'OUTPUT_FORMAT', '')), 'n': len(m.INPUTS), 'sample': [repr(x)[:300] for x in list(m.INPUTS)[:5]]}))", + "", +].join("\n"); + +/** + * The verdict line and the program from a reply. + * + * The program comes in its own fenced block, not as a JSON string: inside + * JSON a whole Python file has to be escaped twice over, and on local tasks + * one reply in four came back with a newline decoded into the middle of a + * string literal — a SyntaxError that, run as a check, exits 1 exactly like + * a real mismatch and refused six correct solutions in a row. A JSON + * `check_py` is still read when a model sends one anyway. + */ +export function parseReferenceReply(text: string): { applies: boolean; reason: string; source: string } | null { + const fence = text.search(/```(?:python|py)?[ \t]*\n/i); + const head = fence === -1 ? text : text.slice(0, fence); + const j = jsonIn(head) ?? (fence === -1 ? null : jsonIn(text)); + if (!j) return null; + let source = typeof j.check_py === "string" ? j.check_py : ""; + if (fence !== -1) { + const body = text.slice(fence).replace(/^```(?:python|py)?[ \t]*\n/i, ""); + const end = body.lastIndexOf("```"); + source = end === -1 ? body : body.slice(0, end); + } + return { applies: j.applies === true, reason: String(j.reason ?? "").slice(0, 200), source }; +} + +export type Snapshot = { dir: string; files: number; bytes: number; hash: string }; + +/** + * Copy the project as it is now into a fresh folder outside it. + * + * Taken before the first step, so everything the reference reads predates the + * work. Null when the project is too large to copy quickly — the reference is + * an extra, never a reason for a slow start. + */ +export function snapshotProject(cwd: string, root = mkdtempSync(join(tmpdir(), "ref-"))): Snapshot | null { + const files: string[] = []; + let bytes = 0; + const walk = (dir: string): boolean => { + let entries: import("node:fs").Dirent[]; + try { + entries = readdirSync(dir, { withFileTypes: true }); + } catch { + return true; + } + for (const e of entries) { + if (SKIP.has(e.name)) continue; + const p = join(dir, e.name); + if (e.isDirectory()) { + if (!walk(p)) return false; + } else if (e.isFile()) { + bytes += statSync(p).size; + files.push(relative(cwd, p)); + if (bytes > REFERENCE_MAX_SNAPSHOT_BYTES || files.length > REFERENCE_MAX_SNAPSHOT_FILES) return false; + } + } + return true; + }; + if (!walk(cwd)) return null; + const before = join(root, "before"); + mkdirSync(before, { recursive: true }); + const h = createHash("sha256"); + for (const f of files.sort()) { + const to = join(before, f); + mkdirSync(join(to, ".."), { recursive: true }); + cpSync(join(cwd, f), to); + h.update(f).update("\0").update(readFileSync(to)).update("\0"); + } + return { dir: root, files: files.length, bytes, hash: h.digest("hex") }; +} + +/** What the reference writer sees of the project: names, sizes, first lines. */ +export function describeSnapshot(before: string, limit = 40): string { + const out: string[] = []; + const walk = (dir: string) => { + for (const e of readdirSync(dir, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) { + if (out.length >= limit) return; + const p = join(dir, e.name); + if (e.isDirectory()) walk(p); + else if (e.isFile()) { + const buf = readFileSync(p); + const binary = buf.subarray(0, 4096).includes(0); + const head = binary ? "(binary)" : buf.toString("utf8").split("\n").slice(0, 8).join("\n"); + out.push(`--- ${relative(before, p)} (${buf.length} bytes)\n${head}`); + } + } + }; + walk(before); + return out.join("\n") || "(empty)"; +} + +export type Reference = + | { ok: true; check: Check; snapshot: Snapshot; reason: string; source: string } + | { ok: false; why: string }; + +type Ask = Omit & { task: string; snapshot: Snapshot; python?: string }; + +/** + * Two independent references and Maat's driver, tried once on the untouched + * project. The first writer is asked once more when its module reports its + * own bug there; a pair that mostly disagrees is dropped. + */ +export async function draftReference(opts: Ask): Promise { + const before = join(opts.snapshot.dir, "before"); + const prompt = `TASK TEXT:\n${opts.task}\n\nFILES IN THE PROJECT BEFORE ANY WORK:\n${describeSnapshot(before)}`; + let r = await writeAndTry(opts, prompt); + if (!r.ok && "selfError" in r && r.selfError && !/independent references agree on only/.test(r.selfError)) { + const again = + `${prompt}\n\nYour previous check.py failed in its own code on the untouched project:\n${r.selfError}\n\n` + + "Fix it. Every expected value must come from running your reference; run_deliverable must never raise. " + + "Reply with the JSON line and the full corrected module."; + r = await writeAndTry(opts, again); + } + return r.ok ? r : { ok: false, why: r.why }; +} + +async function writeAndTry(opts: Ask, prompt: string): Promise { + const dir = opts.snapshot.dir; + const before = join(dir, "before"); + const python = opts.python ?? "python3"; + const env = `REFERENCE_BEFORE='${before}'`; + const asked = await askModel({ ...opts, system: REFERENCE_SYSTEM, prompt, maxTokens: REFERENCE_MAX_TOKENS, what: "writing a reference check" }); + if (!asked.ok) return { ok: false, why: asked.error }; + const parsed = parseReferenceReply(asked.text); + if (!parsed) return { ok: false, why: "the reply was not JSON" }; + const { reason, source } = parsed; + if (!parsed.applies) return { ok: false, why: `does not apply: ${reason || "no reason given"}` }; + if (!source.trim()) return { ok: false, why: "no program in the reply" }; + const file = join(dir, "check.py"); + writeFileSync(file, source); + writeFileSync(join(dir, "probe.py"), PROBE); + writeFileSync(join(dir, "driver.py"), DRIVER); + // Every try runs on a scratch copy of the untouched project, never on the + // project: by now the work may have begun. + const scratch = mkdtempSync(join(tmpdir(), "ref-try-")); + cpSync(before, scratch, { recursive: true }); + const compiled = await runCommand(`${python} -m py_compile '${file}'`, { cwd: scratch, timeoutMs: 30_000, maxBuffer: 256 * 1024 }); + if (compiled.code !== 0) { + const said = lastLines(`${compiled.stdout}\n${compiled.stderr}`); + return { ok: false, why: `the reference does not compile: ${said}`, selfError: said }; + } + const probed = await runCommand(`${env} ${python} '${join(dir, "probe.py")}' '${file}'`, { cwd: scratch, timeoutMs: 30_000, maxBuffer: 256 * 1024 }); + let shape: { input: string; output: string; n: number; sample: string[] }; + try { + shape = JSON.parse(probed.stdout.trim().split("\n").at(-1) ?? ""); + } catch { + const said = lastLines(`${probed.stdout}\n${probed.stderr}`); + return { ok: false, why: `the reference module does not load or defines no INPUTS: ${said}`, selfError: said }; + } + if (!shape.n) return { ok: false, why: "the reference defines no inputs", selfError: "INPUTS is empty" }; + // The second reviewer: the task, the files, what an input is and what shape + // the answer takes — never the first reviewer's code or answers. + const second = await askModel({ + ...opts, + system: SECOND_REFERENCE_SYSTEM, + prompt: + `TASK TEXT:\n${opts.task}\n\nFILES IN THE PROJECT BEFORE ANY WORK:\n${describeSnapshot(before)}\n\n` + + `ONE INPUT IS: ${shape.input || "(not described)"}\nreference(inp) RETURNS: ${shape.output || "(not described)"}\n` + + `SOME OF THE INPUTS (Python repr):\n${shape.sample.map((x) => `- ${x}`).join("\n")}`, + maxTokens: REFERENCE_MAX_TOKENS, + what: "writing the second reference", + }); + if (!second.ok) return { ok: false, why: `no second reference: ${second.error}` }; + const secondSource = fencedPython(second.text); + if (!secondSource) return { ok: false, why: "the second reference reply held no program" }; + writeFileSync(join(dir, "second.py"), secondSource); + const compiled2 = await runCommand(`${python} -m py_compile '${join(dir, "second.py")}'`, { cwd: scratch, timeoutMs: 30_000, maxBuffer: 256 * 1024 }); + if (compiled2.code !== 0) return { ok: false, why: `the second reference does not compile: ${lastLines(compiled2.stderr)}` }; + const check: Check = { + name: REFERENCE_CHECK_NAME, + kind: "command", + run: `${env} ${python} '${join(dir, "driver.py")}'`, + timeoutMs: REFERENCE_TIMEOUT_MS, + expectExit: 0, + // It runs the deliverable on inputs and compares what it returns to two + // independently written references: a run that asserts values. + tags: ["task", "reference", "value"], + hidden: true, + }; + const tried = await runCommand(check.run, { cwd: scratch, timeoutMs: REFERENCE_TIMEOUT_MS, maxBuffer: 1024 * 1024 }); + if (tried.timedOut) return { ok: false, why: "the references did not finish on the untouched project" }; + if (tried.code === REFERENCE_SELF_ERROR) { + const said = lastLines(`${tried.stdout}\n${tried.stderr}`); + return { ok: false, why: `the reference reported its own bug: ${said}`, selfError: said }; + } + if (tried.code === 0) return { ok: false, why: "the reference already passes on the untouched project, so it cannot tell work from none" }; + return { ok: true, check, snapshot: opts.snapshot, reason, source: `${source}\n\n# ---- second.py ----\n${secondSource}` }; +} + +/** The first fenced python block in a reply, or the whole reply when it has none and looks like code. */ +export function fencedPython(text: string): string { + const m = /```(?:python|py)?[ \t]*\n([\s\S]*?)```/i.exec(text); + if (m) return m[1]!; + return /^\s*(?:import|from|def)\s/m.test(text) ? text : ""; +} + +function lastLines(s: string, n = 3): string { + return s.trim().split("\n").slice(-n).join(" / ").slice(0, 300); +} diff --git a/src/run.ts b/src/run.ts index 468d466..cba5922 100644 --- a/src/run.ts +++ b/src/run.ts @@ -17,6 +17,7 @@ * process gets to keep breathing while it waits. */ import { spawn, type ChildProcess } from "node:child_process"; +import { existsSync } from "node:fs"; export type RunOptions = { cwd: string; @@ -25,6 +26,8 @@ export type RunOptions = { /** Cap on captured output, per stream. */ maxBuffer?: number; env?: NodeJS.ProcessEnv; + /** The shell to run it with: `true` is the platform's `sh` (the default); a path runs ` -c command`. */ + shell?: string | true; /** Kills the command when it aborts, so a turn can be cancelled mid-command. */ signal?: AbortSignal; }; @@ -55,6 +58,28 @@ const KILL_GRACE_MS = 2_000; */ export const DRAIN_GRACE_MS = 1_000; +let bashFound: string | null | undefined; + +/** + * Where bash is, or null. Drafted checks run under it rather than `sh`: on a + * Debian image `/bin/sh` is dash, which has no job control (`kill %1` after a + * backgrounded server), no `[[`, no `<<<`. Replayed under bash, every + * `kill %1` check that dash refused passed (reports/checkquality-2026-10-06). + * `MAAT_CHECK_SHELL=sh` turns it off. + */ +export function bashPath(): string | null { + if (process.platform === "win32") return null; + if ((process.env.MAAT_CHECK_SHELL ?? process.env.MOLT_CHECK_SHELL) === "sh") return null; + if (bashFound !== undefined) return bashFound; + bashFound = ["/bin/bash", "/usr/bin/bash", "/usr/local/bin/bash", "/opt/homebrew/bin/bash"].find((p) => existsSync(p)) ?? null; + return bashFound; +} + +/** The shell a drafted (hidden, task-tagged) check runs under; anything else keeps `sh`. */ +export function draftedShell(check: { hidden?: boolean; tags?: readonly string[] }): string | true { + return check.hidden === true && check.tags?.includes("task") ? (bashPath() ?? true) : true; +} + /** * Run a command through the shell and resolve with what it did. * @@ -68,7 +93,7 @@ export function runCommand(command: string, opts: RunOptions): Promise