From 51b51ec5da5eeea4865f635fe2cf38704b59d024 Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Sun, 25 Oct 2026 16:29:41 -0500
Subject: [PATCH 01/49] fix(run): run and compare exit 2 with a message on a
 bad spec or snapshot

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/commands/run.py           | 40 ++++++++++----
 dokimi/dokimi/runner/compare.py         |  4 +-
 dokimi/dokimi/runner/spec.py            |  2 +-
 dokimi/tests/test_run_compare_errors.py | 71 +++++++++++++++++++++++++
 4 files changed, 103 insertions(+), 14 deletions(-)
 create mode 100644 dokimi/tests/test_run_compare_errors.py

diff --git a/dokimi/dokimi/commands/run.py b/dokimi/dokimi/commands/run.py
index 5ddd54783..b9aa2fb4b 100644
--- a/dokimi/dokimi/commands/run.py
+++ b/dokimi/dokimi/commands/run.py
@@ -13,7 +13,21 @@
 from dokimi.report.json_report import write_json
 from dokimi.report.markdown_report import write_compare_report, write_run_report
 from dokimi.runner.compare import compare_snapshots
-from dokimi.runner.run import DocLoadError, run_benchmark, save_snapshot
+from dokimi.runner.run import DocLoadError, _load_spec, run_benchmark, save_snapshot
+
+
+def _run_or_exit(root: Path, spec_path: Path) -> dict:
+    """Run the benchmark; a bad spec exits 2 (1 is the verdict code of --strict), a bad doc exits 1."""
+    try:
+        _load_spec(spec_path)  # fail on a spec that cannot be read, parsed or validated, before the run
+    except (RuntimeError, ValueError) as exc:
+        print(f"Error: {spec_path}: {exc}", file=sys.stderr)
+        sys.exit(2)
+    try:
+        return run_benchmark(root, spec_path)
+    except DocLoadError as exc:
+        print(f"❌ {exc}", file=sys.stderr)
+        sys.exit(1)
 
 
 def cmd_run(args: argparse.Namespace) -> None:
@@ -23,11 +37,7 @@ def cmd_run(args: argparse.Namespace) -> None:
     out_json = Path(args.out_json).resolve()
     out_md = Path(args.out_md).resolve()
 
-    try:
-        result = run_benchmark(root, spec_path)
-    except DocLoadError as exc:
-        print(f"❌ {exc}", file=sys.stderr)
-        sys.exit(1)
+    result = _run_or_exit(root, spec_path)
     save_snapshot(result, out_json)
     write_run_report(result, out_md)
     print(json.dumps(result["summary"], indent=2))
@@ -76,11 +86,19 @@ def cmd_compare(args: argparse.Namespace) -> None:
     out_md = Path(args.out_md).resolve()
 
     spec_data = None
-    if spec_path:
-        import yaml
-        spec_data = yaml.safe_load(spec_path.read_text(encoding="utf-8"))
-
-    result = compare_snapshots(base, cand, spec=spec_data)
+    try:
+        if spec_path:
+            import yaml
+            try:
+                spec_data = yaml.safe_load(spec_path.read_text(encoding="utf-8"))
+            except (OSError, ValueError, yaml.YAMLError) as exc:
+                raise ValueError(f"Invalid spec {spec_path}: {exc}") from exc
+            if spec_data is not None and not isinstance(spec_data, dict):
+                raise ValueError(f"Invalid spec {spec_path}: root must be a mapping")
+        result = compare_snapshots(base, cand, spec=spec_data)
+    except ValueError as exc:
+        print(f"Error: {exc}", file=sys.stderr)
+        sys.exit(2)
     write_json(result, out_json)
     write_compare_report(result, out_md)
     print(json.dumps(result["diff"], indent=2))
diff --git a/dokimi/dokimi/runner/compare.py b/dokimi/dokimi/runner/compare.py
index a9028f2dc..7416a8fa7 100644
--- a/dokimi/dokimi/runner/compare.py
+++ b/dokimi/dokimi/runner/compare.py
@@ -15,12 +15,12 @@ def compare_snapshots(base_path: Path, cand_path: Path, spec: dict | None = None
     """Compare two run snapshots and return summary and metric deltas."""
     try:
         base = json.loads(base_path.read_text(encoding="utf-8"))
-    except (OSError, json.JSONDecodeError) as exc:
+    except (OSError, ValueError) as exc:  # ValueError covers JSONDecodeError and bad UTF-8
         raise ValueError(f"Invalid base snapshot {base_path}: {exc}") from exc
 
     try:
         cand = json.loads(cand_path.read_text(encoding="utf-8"))
-    except (OSError, json.JSONDecodeError) as exc:
+    except (OSError, ValueError) as exc:
         raise ValueError(f"Invalid candidate snapshot {cand_path}: {exc}") from exc
 
     required = {"docs", "score", "coverage", "freshness_lite", "readability"}
diff --git a/dokimi/dokimi/runner/spec.py b/dokimi/dokimi/runner/spec.py
index 9b39c247d..1d34b0999 100644
--- a/dokimi/dokimi/runner/spec.py
+++ b/dokimi/dokimi/runner/spec.py
@@ -62,7 +62,7 @@ def load_spec(spec_path: Path, *, validate: bool = True) -> dict:
     """Read a spec YAML and, for full v1 specs, validate it against the schema."""
     try:
         content = Path(spec_path).read_text(encoding="utf-8")
-    except OSError as exc:
+    except (OSError, ValueError) as exc:  # ValueError covers bad UTF-8
         raise RuntimeError(f"Failed to read spec file: {spec_path}: {exc}") from exc
 
     try:
diff --git a/dokimi/tests/test_run_compare_errors.py b/dokimi/tests/test_run_compare_errors.py
new file mode 100644
index 000000000..812379d81
--- /dev/null
+++ b/dokimi/tests/test_run_compare_errors.py
@@ -0,0 +1,71 @@
+# Copyright (C) 2026 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+"""`run` and `compare` name a bad input file and exit 2 instead of ending in a traceback."""
+import json
+
+import pytest
+
+from dokimi import cli
+
+SNAPSHOT = {"summary": {"docs": 1, "score": 0.5, "coverage": 0.5, "freshness_lite": 0.5, "readability": 0.5}}
+
+
+def _main(monkeypatch, *argv):
+    monkeypatch.setattr("sys.argv", ["dokimi", *argv])
+    with pytest.raises(SystemExit) as exc:
+        cli.main()
+    return exc.value.code
+
+
+def _snapshot(path):
+    path.write_text(json.dumps(SNAPSHOT), encoding="utf-8")
+    return str(path)
+
+
+def _write(path, content):
+    if isinstance(content, bytes):
+        path.write_bytes(content)
+    elif content is not None:
+        path.write_text(content, encoding="utf-8")
+
+
+@pytest.mark.parametrize("content", [None, "weights: [", "- a\n- b\n", "weights: {}\n", b"\xff\xfe", "version: 1\n"],
+                         ids=["missing", "malformed", "not-a-mapping", "missing-fields", "not-utf8", "schema-invalid"])
+def test_run_exits_2_on_a_bad_spec(tmp_path, monkeypatch, capsys, content):
+    spec = tmp_path / "spec.yaml"
+    _write(spec, content)
+    code = _main(monkeypatch, "run", "--root", str(tmp_path), "--spec", str(spec),
+                 "--out-json", str(tmp_path / "o.json"), "--out-md", str(tmp_path / "o.md"))
+    assert code == 2
+    err = capsys.readouterr().err
+    assert err.startswith("Error: ") and str(spec) in err
+    assert not (tmp_path / "o.json").exists()
+
+
+@pytest.mark.parametrize("which", ["--base", "--candidate"])
+@pytest.mark.parametrize("content", [None, "{not json", b"\xff\xfe", "[1]", "{}"],
+                         ids=["missing", "malformed", "not-utf8", "not-an-object", "no-summary"])
+def test_compare_exits_2_on_a_bad_snapshot(tmp_path, monkeypatch, capsys, which, content):
+    bad = tmp_path / "bad.json"
+    _write(bad, content)
+    good = _snapshot(tmp_path / "good.json")
+    pair = {"--base": good, "--candidate": good, which: str(bad)}
+    code = _main(monkeypatch, "compare", "--base", pair["--base"], "--candidate", pair["--candidate"],
+                 "--out-json", str(tmp_path / "c.json"), "--out-md", str(tmp_path / "c.md"))
+    assert code == 2
+    err = capsys.readouterr().err
+    assert err.startswith("Error: ") and str(bad) in err
+    assert not (tmp_path / "c.json").exists()
+
+
+@pytest.mark.parametrize("content", [None, "a: [", "- a\n"], ids=["missing", "malformed", "not-a-mapping"])
+def test_compare_exits_2_on_a_bad_spec(tmp_path, monkeypatch, capsys, content):
+    spec = tmp_path / "spec.yaml"
+    _write(spec, content)
+    good = _snapshot(tmp_path / "good.json")
+    code = _main(monkeypatch, "compare", "--base", good, "--candidate", good, "--spec", str(spec),
+                 "--out-json", str(tmp_path / "c.json"), "--out-md", str(tmp_path / "c.md"))
+    assert code == 2
+    err = capsys.readouterr().err
+    assert err.startswith("Error: ") and str(spec) in err

From 850f58f3ab9ed2e39e77b3d3c3ac682f7aaa774c Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Sun, 25 Oct 2026 16:29:41 -0500
Subject: [PATCH 02/49] fix(repo): repo run exits 1 with a message on an
 unreadable batch pin

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/commands/repo_run.py |  7 ++++++-
 dokimi/tests/test_repo_run.py      | 12 ++++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/dokimi/dokimi/commands/repo_run.py b/dokimi/dokimi/commands/repo_run.py
index 0b965b1df..21d45e0dd 100644
--- a/dokimi/dokimi/commands/repo_run.py
+++ b/dokimi/dokimi/commands/repo_run.py
@@ -124,7 +124,12 @@ def _uses_builtin_graders(content: Path, manifest: RepoManifest) -> bool:
 def _pin_batch(batch_dir: Path, pin: dict[str, str]) -> None:
     path = batch_dir / "_eval" / "batch.json"
     if path.is_file():
-        old = json.loads(path.read_text(encoding="utf-8"))
+        try:
+            old = json.loads(path.read_text(encoding="utf-8"))
+        except ValueError as exc:  # JSONDecodeError and bad UTF-8
+            old = exc
+        if not isinstance(old, dict):
+            raise ValueError(f"batch {batch_dir.name} has an unreadable pin ({path}); use another --batch")
         changed = [k for k in pin if old.get(k) != pin[k]]
         if changed:
             was = ", ".join(f"{k} {old.get(k)}" for k in changed)
diff --git a/dokimi/tests/test_repo_run.py b/dokimi/tests/test_repo_run.py
index 48c92e89c..fda202cd5 100644
--- a/dokimi/tests/test_repo_run.py
+++ b/dokimi/tests/test_repo_run.py
@@ -663,6 +663,18 @@ def test_a_task_json_the_pin_cannot_read_pins_the_graders_and_a_non_object_one_d
     assert ("graders" in json.loads((setup["out"] / "b1/_eval/batch.json").read_text())) is pinned
 
 
+@pytest.mark.parametrize("text", ["{", "[]", '"pin"'])
+def test_a_batch_whose_pin_is_unreadable_exits_1_naming_the_file(setup, capsys, text):
+    _run(setup, "--arm", "guidance:raw", "--task", "fix_add")
+    (setup["out"] / "b1/_eval/batch.json").write_text(text)
+    capsys.readouterr()
+    with pytest.raises(SystemExit) as exc:
+        _run(setup, "--arm", "guidance:none", "--task", "fix_add")
+    assert exc.value.code == 1
+    err = capsys.readouterr().err
+    assert err.startswith("✗ ") and "_eval/batch.json" in err and "use another --batch" in err
+
+
 def test_graders_digest_follows_the_grader_sources(tmp_path, monkeypatch):
     from dokimi.repos import graders
 

From d9fba9fe87b287e70976af30fe98b14578a90c46 Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Sun, 25 Oct 2026 16:29:41 -0500
Subject: [PATCH 03/49] fix(repo): scan exits 1 with a message on a --static or
 --manifest-file that is not UTF-8

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/commands/repo.py     |  4 ++--
 dokimi/tests/test_repo_l0_cases.py | 19 +++++++++++++++++++
 2 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/dokimi/dokimi/commands/repo.py b/dokimi/dokimi/commands/repo.py
index 159a606c9..b27f19088 100644
--- a/dokimi/dokimi/commands/repo.py
+++ b/dokimi/dokimi/commands/repo.py
@@ -83,7 +83,7 @@ def cmd_repo_scan(args: argparse.Namespace) -> None:
         tree = open_source(RepoSource.parse(args.source), _cache_dir(args))
         manifest = _load_manifest(tree, args, guidance_default=True)
         static = json.loads(Path(args.static).read_text(encoding="utf-8")) if args.static else None
-    except (SourceError, ManifestError, OSError, json.JSONDecodeError) as exc:
+    except (SourceError, ManifestError, OSError, ValueError) as exc:  # ValueError: bad JSON or UTF-8
         print(f"✗ {exc}", file=sys.stderr)
         sys.exit(1)
     payload = scan_repo(tree, manifest, static)
@@ -314,7 +314,7 @@ def _load_manifest(tree, args: argparse.Namespace, *, guidance_default: bool = F
         return load_manifest(tree, path)
     try:
         raw = yaml.safe_load(Path(args.manifest_file).read_text(encoding="utf-8"))
-    except (OSError, yaml.YAMLError) as exc:
+    except (OSError, ValueError, yaml.YAMLError) as exc:  # ValueError: not UTF-8
         raise ManifestError(args.manifest_file, [str(exc)]) from exc
     return parse_manifest(raw, path)
 
diff --git a/dokimi/tests/test_repo_l0_cases.py b/dokimi/tests/test_repo_l0_cases.py
index d92f38785..e3e07060c 100644
--- a/dokimi/tests/test_repo_l0_cases.py
+++ b/dokimi/tests/test_repo_l0_cases.py
@@ -358,6 +358,25 @@ def test_cli_exit_codes(tmp_path, capsys):
     assert _cli(tmp_path, empty, "--fail-on-fail") == 1
 
 
+def test_scan_exits_1_with_a_message_on_a_file_that_is_not_utf8(tmp_path, capsys):
+    root = tmp_path / "r"
+    _git_repo(root, {"AGENTS.md": "`src/a.h`\n", "src/a.h": "", "ev/tasks/t/task.json": "{}"})
+    binary = tmp_path / "static.json"
+    binary.write_bytes(b'{"schema": "\xff\xfe"}')
+    assert _cli(tmp_path, root, "--static", str(binary)) == 1
+    err = capsys.readouterr().err
+    assert err.startswith("✗ ") and "utf-8" in err
+
+    manifest = tmp_path / "m.yaml"
+    manifest.write_bytes(b"id: \xff\xfe\n")
+    args = build_parser().parse_args(["repo", "scan", str(root), "--manifest-file", str(manifest)])
+    with pytest.raises(SystemExit) as exc:
+        args.func(args)
+    assert exc.value.code == 1
+    err = capsys.readouterr().err
+    assert err.startswith("✗ ") and str(manifest) in err
+
+
 def _linked_repo(root: Path, files: dict[str, str], links: dict[str, str]) -> GitTree:
     """A repository with ``links`` (path -> link text) committed as symlinks."""
     for rel, text in links.items():

From f18c2694a8c8bb76c7e2d41ba7a2eca1d2928f60 Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Sun, 25 Oct 2026 16:29:41 -0500
Subject: [PATCH 04/49] fix(arms): plugin-delta exits 1 with a message on an
 unreadable arms file

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/commands/arms.py      |  2 +-
 dokimi/tests/test_record_readers.py | 15 +++++++++++++++
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/dokimi/dokimi/commands/arms.py b/dokimi/dokimi/commands/arms.py
index 5696708f8..c4b4cbac8 100644
--- a/dokimi/dokimi/commands/arms.py
+++ b/dokimi/dokimi/commands/arms.py
@@ -189,7 +189,7 @@ def cmd_arms_plugin_delta(args: argparse.Namespace) -> None:
             for section in ("answers", "evaluations"):
                 records(artifact.get(section) or [], source=f"{path} {section}", kind=section)
         output = compare_plugin_runs(baseline, plugin)
-    except (PluginDeltaError, ValueError, FileNotFoundError) as exc:
+    except (PluginDeltaError, ValueError, OSError) as exc:
         print(f"Error computing plugin delta: {exc}", file=sys.stderr)
         sys.exit(1)
 
diff --git a/dokimi/tests/test_record_readers.py b/dokimi/tests/test_record_readers.py
index b67d45160..5536c6ee8 100644
--- a/dokimi/tests/test_record_readers.py
+++ b/dokimi/tests/test_record_readers.py
@@ -326,6 +326,21 @@ def test_plugin_delta_names_the_file_with_the_malformed_record(tmp_path, monkeyp
     assert not (tmp_path / "d.json").exists()
 
 
+def test_plugin_delta_exits_1_with_a_message_on_an_unreadable_arms_file(tmp_path, capsys):
+    from dokimi.cli import build_parser
+
+    unreadable = tmp_path / "b.json"
+    unreadable.mkdir()  # reading a directory raises IsADirectoryError, as a permission error would
+    args = build_parser().parse_args(["arms", "plugin-delta", "--baseline-json", str(unreadable),
+                                      "--plugin-json", str(tmp_path / "p.json"),
+                                      "--out-json", str(tmp_path / "d.json")])
+    with pytest.raises(SystemExit) as exc:
+        args.func(args)
+    assert exc.value.code == 1
+    assert "b.json" in capsys.readouterr().err
+    assert not (tmp_path / "d.json").exists()
+
+
 def _ev(qid, w, wo, delta):
     return {"question_id": qid, "question_text": f"q{qid}",
             "with_docs": {"aggregate": w}, "without_docs": {"aggregate": wo}, "delta": delta}

From 3b5478b10936de4e110a98ede1a0f25f35f2ac75 Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Sun, 25 Oct 2026 16:29:41 -0500
Subject: [PATCH 05/49] fix(explore): list and run exit with a message on a
 malformed task file

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/commands/explore.py       |  4 +-
 dokimi/dokimi/explore/tasks.py          | 30 ++++++++---
 dokimi/tests/test_explore_task_files.py | 67 +++++++++++++++++++++++++
 3 files changed, 93 insertions(+), 8 deletions(-)
 create mode 100644 dokimi/tests/test_explore_task_files.py

diff --git a/dokimi/dokimi/commands/explore.py b/dokimi/dokimi/commands/explore.py
index 64900fdc9..a34bd9074 100644
--- a/dokimi/dokimi/commands/explore.py
+++ b/dokimi/dokimi/commands/explore.py
@@ -26,7 +26,7 @@ def cmd_explore_list(args: argparse.Namespace) -> None:
     default_repo_root = Path(args.repo_root) if getattr(args, "repo_root", None) else None
     try:
         tasks = discover_explore_tasks(root, default_repo_root=default_repo_root)
-    except FileNotFoundError as exc:
+    except (FileNotFoundError, ValueError) as exc:  # a task file that cannot be read or parsed
         raise SystemExit(str(exc)) from exc
     for task in tasks:
         n_refs = len(task.references)
@@ -90,7 +90,7 @@ def cmd_explore_run(args: argparse.Namespace) -> None:
             tasks = [
                 load_explore_task(t, root=root, default_repo_root=default_repo_root) for t in ids
             ]
-    except FileNotFoundError as exc:
+    except (FileNotFoundError, ValueError) as exc:  # a task file that cannot be read or parsed
         raise SystemExit(str(exc)) from exc
     if not tasks:
         raise SystemExit("No tasks selected. Pass --tasks <id[,id]> or --all.")
diff --git a/dokimi/dokimi/explore/tasks.py b/dokimi/dokimi/explore/tasks.py
index 2d24b8ba4..cc9cdaa90 100644
--- a/dokimi/dokimi/explore/tasks.py
+++ b/dokimi/dokimi/explore/tasks.py
@@ -107,19 +107,39 @@ def _parse_lines(value) -> tuple[int, int] | None:
     return (min(start, end), max(start, end))
 
 
+def _read_task_yaml(path: Path) -> dict:
+    """The mapping in the task file at ``path``; any unreadable or malformed file is a ``ValueError`` naming it."""
+    try:
+        data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+    except OSError as exc:
+        raise ValueError(f"Explore task {path} cannot be read: {exc}") from exc
+    except (UnicodeDecodeError, yaml.YAMLError) as exc:
+        raise ValueError(f"Explore task {path} is not valid UTF-8 YAML: {exc}") from exc
+    if not isinstance(data, dict):
+        raise ValueError(f"Explore task {path} must hold a YAML mapping")
+    return data
+
+
 def _task_from_data(
     data: dict, path: Path, default_repo_root: Path | None = None
 ) -> ExploreTask:
     missing = [k for k in ("id", "query", "references") if k not in data]
     if missing:
         raise ValueError(f"Explore task {path} missing required field(s): {missing}")
+    raw_refs = data["references"]
+    if not isinstance(raw_refs, list) or not all(
+        isinstance(ref, dict) and "path" in ref for ref in raw_refs
+    ):
+        raise ValueError(f"Explore task {path} references must be a list of entries with a 'path'")
     references = tuple(
         ExploreReference(path=str(ref["path"]), lines=_parse_lines(ref.get("lines")))
-        for ref in data["references"]
+        for ref in raw_refs
     )
     if not references:
         raise ValueError(f"Explore task {path} has no references")
     repo_root = data.get("repo_root")
+    if repo_root and not isinstance(repo_root, str):
+        raise ValueError(f"Explore task {path} repo_root must be a path string")
     if repo_root:
         resolved_root = (path.parent / repo_root).resolve()
     elif default_repo_root is not None:
@@ -157,12 +177,11 @@ def load_explore_task(
     """
     candidate = Path(ref)
     if candidate.is_file():
-        data = yaml.safe_load(candidate.read_text(encoding="utf-8")) or {}
-        return _task_from_data(data, candidate, default_repo_root)
+        return _task_from_data(_read_task_yaml(candidate), candidate, default_repo_root)
 
     root = _tasks_root(root)
     files = _task_files(root)
-    loaded = [(path, yaml.safe_load(path.read_text(encoding="utf-8")) or {}) for path in files]
+    loaded = [(path, _read_task_yaml(path)) for path in files]
     id_matches = [(path, data) for path, data in loaded if str(data.get("id")) == ref]
     if len(id_matches) > 1:
         paths = ", ".join(str(p) for p, _ in id_matches)
@@ -188,8 +207,7 @@ def discover_explore_tasks(
     root = _tasks_root(root)
     tasks = []
     for path in _task_files(root):
-        data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
-        tasks.append(_task_from_data(data, path, default_repo_root))
+        tasks.append(_task_from_data(_read_task_yaml(path), path, default_repo_root))
     by_id: dict[str, list[Path]] = {}
     for t in tasks:
         by_id.setdefault(t.id, []).append(t.path)
diff --git a/dokimi/tests/test_explore_task_files.py b/dokimi/tests/test_explore_task_files.py
new file mode 100644
index 000000000..e8cc4b3f0
--- /dev/null
+++ b/dokimi/tests/test_explore_task_files.py
@@ -0,0 +1,67 @@
+# Copyright (C) 2026 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+"""`explore list` / `explore run`: an unreadable or malformed task file exits with a message."""
+
+from __future__ import annotations
+
+import pytest
+
+from dokimi.cli import build_parser
+
+GOOD = "id: ok\nproduct: p\nquery: q\nreferences:\n  - path: a.py\n"
+
+
+def _exit_message(argv) -> str:
+    args = build_parser().parse_args(argv)
+    with pytest.raises(SystemExit) as exc:
+        args.func(args)
+    return str(exc.value)
+
+
+@pytest.mark.parametrize("argv_tail", [["list"], ["run", "--all"]])
+@pytest.mark.parametrize(
+    ("content", "message"),
+    [
+        (b"id: [unclosed\n", "is not valid UTF-8 YAML"),
+        (bytes([0xFF, 0xFE, 0x00]), "is not valid UTF-8 YAML"),
+        (b"- just\n- a list\n", "must hold a YAML mapping"),
+        (b"id: only-an-id\n", "missing required field(s)"),
+        (b"id: x\nquery: q\nreferences: 5\n", "references must be a list"),
+        (b"id: x\nquery: q\nreferences: [a.py]\n", "references must be a list"),
+        (b"id: x\nquery: q\nreferences:\n  - lines: [1]\n", "references must be a list"),
+        (b"id: x\nquery: q\nreferences:\n  - path: a.py\nrepo_root: 5\n", "repo_root must be"),
+    ],
+)
+def test_a_malformed_task_file_names_the_file(tmp_path, argv_tail, content, message):
+    (tmp_path / "ok.yaml").write_text(GOOD, encoding="utf-8")
+    bad = tmp_path / "bad.yaml"
+    bad.write_bytes(content)
+    out = _exit_message(["explore", *argv_tail, "--tasks-root", str(tmp_path)])
+    assert str(bad) in out
+    assert message in out
+
+
+def test_a_task_file_passed_by_path_names_the_file(tmp_path):
+    bad = tmp_path / "bad.yaml"
+    bad.write_text("references: {", encoding="utf-8")
+    out = _exit_message(["explore", "run", "--tasks", str(bad), "--tasks-root", str(tmp_path)])
+    assert str(bad) in out
+    assert "is not valid UTF-8 YAML" in out
+
+
+def test_a_task_file_that_cannot_be_read_names_the_file(tmp_path, monkeypatch):
+    from pathlib import Path
+
+    unreadable = tmp_path / "locked.yaml"
+    unreadable.write_text(GOOD, encoding="utf-8")
+    real = Path.read_text
+
+    def deny(self, *args, **kwargs):
+        if self == unreadable:
+            raise PermissionError(13, "Permission denied")
+        return real(self, *args, **kwargs)
+
+    monkeypatch.setattr(Path, "read_text", deny)
+    out = _exit_message(["explore", "list", "--tasks-root", str(tmp_path)])
+    assert f"Explore task {unreadable} cannot be read" in out

From 382b2d98093fbaa1d22f5226c5a86aed1e3dfeb2 Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Sun, 25 Oct 2026 16:29:41 -0500
Subject: [PATCH 06/49] fix(arms): run and matrix-run exit 1 with a message on
 an unreadable --matrix-cells file

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/eval/cells.py                  |  7 +++-
 dokimi/tests/test_arms_matrix_cells_input.py | 43 ++++++++++++++++++++
 2 files changed, 49 insertions(+), 1 deletion(-)
 create mode 100644 dokimi/tests/test_arms_matrix_cells_input.py

diff --git a/dokimi/dokimi/eval/cells.py b/dokimi/dokimi/eval/cells.py
index 24d7f3470..d953f6e1c 100644
--- a/dokimi/dokimi/eval/cells.py
+++ b/dokimi/dokimi/eval/cells.py
@@ -150,7 +150,12 @@ def load_matrix_cells(path: str | Path) -> list[MatrixCellDescriptor]:
     Accepted shapes are ``{"matrix": {"cells": [...]}}``, ``{"cells": [...]}``,
     or a top-level list of cells.
     """
-    raw = json.loads(Path(path).read_text(encoding="utf-8"))
+    try:
+        raw = json.loads(Path(path).read_text(encoding="utf-8"))
+    except OSError as exc:
+        raise ValueError(f"cannot read {path}: {exc}") from None
+    except ValueError as exc:  # JSONDecodeError, or bytes that are not UTF-8
+        raise ValueError(f"{path} is not valid JSON: {exc}") from None
     cells = _extract_cells(raw)
     if not cells:
         raise ValueError("matrix config must contain at least one cell")
diff --git a/dokimi/tests/test_arms_matrix_cells_input.py b/dokimi/tests/test_arms_matrix_cells_input.py
new file mode 100644
index 000000000..c621d29a9
--- /dev/null
+++ b/dokimi/tests/test_arms_matrix_cells_input.py
@@ -0,0 +1,43 @@
+# Copyright (C) 2026 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+"""`arms run` / `arms matrix-run`: an unreadable --matrix-cells file exits 1 with a message."""
+
+from __future__ import annotations
+
+import json
+
+import pytest
+
+from dokimi.cli import build_parser
+
+
+def _questions(tmp_path):
+    path = tmp_path / "questions.json"
+    path.write_text(json.dumps({"questions": [{"id": "q1", "question": "What is x?"}]}), encoding="utf-8")
+    return path
+
+
+def _run(tmp_path, cells, capsys, *argv):
+    args = build_parser().parse_args([
+        "arms", *argv, "--product", "p", "--questions", str(_questions(tmp_path)),
+        "--arms", "baseline", "--matrix-cells", str(cells),
+        *(["--matrix-cell", "c1"] if argv[0] == "run" else ["--out-dir", str(tmp_path / "out")]),
+    ])
+    with pytest.raises(SystemExit) as exc:
+        args.func(args)
+    assert exc.value.code == 1
+    return capsys.readouterr().err
+
+
+@pytest.mark.parametrize("command", ["run", "matrix-run"])
+def test_a_directory_as_matrix_cells_is_reported_as_unreadable(tmp_path, capsys, command):
+    err = _run(tmp_path, tmp_path, capsys, command)
+    assert f"cannot read {tmp_path}" in err
+
+
+@pytest.mark.parametrize("command", ["run", "matrix-run"])
+def test_matrix_cells_that_are_not_json_name_the_file(tmp_path, capsys, command):
+    bad = tmp_path / "cells.json"
+    bad.write_text("{not json", encoding="utf-8")
+    assert f"{bad} is not valid JSON" in _run(tmp_path, bad, capsys, command)

From e2e9b832b3af95db8a7ca96792e5cd456111fc0b Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Sun, 25 Oct 2026 16:29:41 -0500
Subject: [PATCH 07/49] fix(tasks): run exits 1 with a message when --out-json
 or --out-md cannot be written

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/commands/tasks.py    | 12 +++++++++---
 dokimi/tests/test_tasks_command.py | 31 ++++++++++++++++++++++++++++++
 2 files changed, 40 insertions(+), 3 deletions(-)

diff --git a/dokimi/dokimi/commands/tasks.py b/dokimi/dokimi/commands/tasks.py
index bb082c52e..e5dc5a951 100644
--- a/dokimi/dokimi/commands/tasks.py
+++ b/dokimi/dokimi/commands/tasks.py
@@ -204,14 +204,20 @@ def cmd_tasks_run(args: argparse.Namespace) -> None:
     out_json = Path(args.out_json) if args.out_json else Path(args.output_dir) / "task_runs.json"
     from dokimi.artifacts import save_artifact
 
-    save_artifact("task_runs", output, out_json)
+    try:
+        save_artifact("task_runs", output, out_json)
+    except OSError as exc:
+        raise SystemExit(f"error: cannot write {out_json}: {exc}") from None
     print(f"Saved task run artifact: {out_json}")
 
     out_md = Path(args.out_md) if args.out_md else out_json.with_suffix(".md")
     from dokimi.report.task_runs_report import render_task_runs_report
 
-    out_md.parent.mkdir(parents=True, exist_ok=True)
-    out_md.write_text(render_task_runs_report(output), encoding="utf-8")
+    try:
+        out_md.parent.mkdir(parents=True, exist_ok=True)
+        out_md.write_text(render_task_runs_report(output), encoding="utf-8")
+    except OSError as exc:
+        raise SystemExit(f"error: cannot write {out_md}: {exc} (the artifact is saved at {out_json})") from None
     print(f"Saved task run report:   {out_md}")
 
     _print_summary(output)
diff --git a/dokimi/tests/test_tasks_command.py b/dokimi/tests/test_tasks_command.py
index de1204ea9..d413d47d4 100644
--- a/dokimi/tests/test_tasks_command.py
+++ b/dokimi/tests/test_tasks_command.py
@@ -122,6 +122,37 @@ def test_tasks_run_with_a_missing_skill_path_exits_before_writing_anything(tmp_p
     assert not out.exists()
 
 
+def test_tasks_run_with_an_unwritable_out_json_exits_with_a_message(tmp_path):
+    _fixture_task(tmp_path)
+    blocker = tmp_path / "blocker"
+    blocker.write_text("a file, not a directory", encoding="utf-8")
+    out = blocker / "runs.json"
+    args = build_parser().parse_args([
+        "tasks", "run", "--tasks", "toy-task", "--tasks-root", str(tmp_path),
+        "--harnesses", "codex", "--model", "m", "--out-json", str(out), "--dry-run",
+    ])
+    with pytest.raises(SystemExit) as excinfo:
+        args.func(args)
+    assert f"cannot write {out}" in str(excinfo.value)
+
+
+def test_tasks_run_with_an_unwritable_out_md_keeps_the_artifact_and_exits_with_a_message(tmp_path):
+    _fixture_task(tmp_path)
+    blocker = tmp_path / "blocker"
+    blocker.write_text("a file, not a directory", encoding="utf-8")
+    out = tmp_path / "runs.json"
+    md = blocker / "report.md"
+    args = build_parser().parse_args([
+        "tasks", "run", "--tasks", "toy-task", "--tasks-root", str(tmp_path),
+        "--harnesses", "codex", "--model", "m", "--out-json", str(out), "--out-md", str(md),
+        "--dry-run",
+    ])
+    with pytest.raises(SystemExit) as excinfo:
+        args.func(args)
+    assert f"cannot write {md}" in str(excinfo.value)
+    assert out.is_file()
+
+
 def test_tasks_list_rejects_a_root_that_is_not_a_directory(tmp_path):
     """A mistyped --tasks-root must not read as a suite with zero tasks."""
     parser = build_parser()

From 57f1f91f8fdbc660d970c411297e7dd78c8e11a3 Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Sun, 25 Oct 2026 16:29:41 -0500
Subject: [PATCH 08/49] fix(repo): report and compare exit 1 with a message on
 a corrupt batch file

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/commands/repo_run.py            | 11 ++-
 dokimi/dokimi/repos/report.py                 | 31 +++++++-
 .../tests/test_repo_report_unreadable_cell.py | 76 +++++++++++++++++++
 3 files changed, 112 insertions(+), 6 deletions(-)
 create mode 100644 dokimi/tests/test_repo_report_unreadable_cell.py

diff --git a/dokimi/dokimi/commands/repo_run.py b/dokimi/dokimi/commands/repo_run.py
index 21d45e0dd..ff964b231 100644
--- a/dokimi/dokimi/commands/repo_run.py
+++ b/dokimi/dokimi/commands/repo_run.py
@@ -289,19 +289,24 @@ def _harness_bin(name: str) -> Path:
 
 def cmd_repo_report(args: argparse.Namespace) -> None:
     """Per (task, arm, model): gated success, grader failures, missing cells, cost, turns."""
-    from dokimi.repos.report import format_report, summarize
+    from dokimi.repos.report import CellFileError, format_report, summarize
 
     batch_dir = Path(args.batch_dir)
     if not batch_dir.is_dir():
         print(f"✗ not a directory: {batch_dir}", file=sys.stderr)
         sys.exit(1)
-    summary = summarize(batch_dir)
+    try:
+        summary = summarize(batch_dir)
+    except CellFileError as exc:
+        print(f"✗ {exc}", file=sys.stderr)
+        sys.exit(1)
     print(json.dumps(summary, indent=2) if args.json else format_report(summary))
 
 
 def cmd_repo_compare(args: argparse.Namespace) -> None:
     """Per (treatment, model, measure): per-task deltas against the baseline, 95% CI, sign-flip p."""
     from dokimi.repos.compare import CompareError, compare, format_compare
+    from dokimi.repos.report import CellFileError
 
     dirs = [Path(d) for d in args.batch_dirs]
     for d in dirs:
@@ -311,7 +316,7 @@ def cmd_repo_compare(args: argparse.Namespace) -> None:
     arms = [a for spec in args.arm for a in spec.split(",") if a] if args.arm else None
     try:
         summary = compare(dirs, args.baseline, arms, args.model, tuple(args.metric or ()))
-    except CompareError as exc:
+    except (CompareError, CellFileError) as exc:
         print(f"✗ {exc}", file=sys.stderr)
         sys.exit(1)
     print(json.dumps(summary, indent=2) if args.json else format_compare(summary))
diff --git a/dokimi/dokimi/repos/report.py b/dokimi/dokimi/repos/report.py
index 576bb28d1..7de095ca2 100644
--- a/dokimi/dokimi/repos/report.py
+++ b/dokimi/dokimi/repos/report.py
@@ -35,14 +35,39 @@
 from .runner import UNSUPPORTED_FILE, slug
 
 
+class CellFileError(ValueError):
+    """A batch file that cannot be read as the JSON object ``repo run`` wrote."""
+
+
+def _read_object(path: Path) -> dict[str, Any]:
+    """The JSON object in ``path``, or a :class:`CellFileError` naming the file."""
+    try:
+        rec = json.loads(path.read_text(encoding="utf-8"))
+    except OSError as exc:
+        raise CellFileError(f"cannot read {path}: {exc}") from exc
+    except (ValueError, RecursionError) as exc:  # JSONDecodeError, bad UTF-8, deep nesting
+        raise CellFileError(f"{path} is not valid JSON: {exc}") from exc
+    if not isinstance(rec, dict):
+        raise CellFileError(f"{path} must hold a JSON object")
+    return rec
+
+
+def _planned_cells(plan_path: Path) -> set[str]:
+    """The cell names in ``plan.json``, or a :class:`CellFileError` when it holds no ``cells`` list of names."""
+    cells = _read_object(plan_path).get("cells")
+    if not isinstance(cells, list) or not all(isinstance(c, str) for c in cells):
+        raise CellFileError(f"{plan_path} must hold a `cells` list of cell names")
+    return set(cells)
+
+
 def load_cells(batch_dir: Path) -> tuple[list[dict[str, Any]], list[str]]:
     """Finished cells (their ``result.json``) and the names of missing ones."""
     plan_path = batch_dir / "_eval" / "plan.json"
-    planned = set(json.loads(plan_path.read_text(encoding="utf-8"))["cells"]) if plan_path.is_file() else set()
+    planned = _planned_cells(plan_path) if plan_path.is_file() else set()
     started = {p.parent.name for p in batch_dir.glob("*/meta.json")}
     finished = []
     for path in sorted(batch_dir.glob("*/result.json")):
-        rec = json.loads(path.read_text(encoding="utf-8"))
+        rec = _read_object(path)
         rec["cell"] = path.parent.name
         # The row's kind is the task's, not what its cells' grades say. The runner records it from the
         # eval dir's task.json; the cell's copy is in the run dir, which the grader could rewrite.
@@ -59,7 +84,7 @@ def load_unsupported(batch_dir: Path) -> list[dict[str, Any]]:
     for path in sorted(batch_dir.glob(f"*/{UNSUPPORTED_FILE}")):
         if (path.parent / "result.json").is_file():
             continue
-        rec = json.loads(path.read_text(encoding="utf-8"))
+        rec = _read_object(path)
         out.append({"cell": path.parent.name, "task": rec.get("task"), "arm": rec.get("arm"),
                     "model": rec.get("model"), "reason": rec.get("reason")})
     return out
diff --git a/dokimi/tests/test_repo_report_unreadable_cell.py b/dokimi/tests/test_repo_report_unreadable_cell.py
new file mode 100644
index 000000000..cb271541f
--- /dev/null
+++ b/dokimi/tests/test_repo_report_unreadable_cell.py
@@ -0,0 +1,76 @@
+# Copyright (C) 2026 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+"""`repo report` and `repo compare` exit 1 with a message, not a traceback, on a corrupt cell file."""
+
+from __future__ import annotations
+
+import json
+
+import pytest
+
+from dokimi.cli import build_parser
+
+
+def _batch(tmp_path, name: str, files: dict[str, bytes]):
+    batch = tmp_path / name
+    for rel, content in files.items():
+        path = batch / rel
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_bytes(content)
+    return batch
+
+
+def _run(*argv: str) -> None:
+    args = build_parser().parse_args(["repo", *argv])
+    args.func(args)
+
+
+_CORRUPT = [
+    pytest.param(b"{not json", id="not-json"),
+    pytest.param(b"\xff\xfe", id="not-utf8"),
+    pytest.param(b"[1, 2]", id="not-an-object"),
+]
+
+
+@pytest.mark.parametrize("content", _CORRUPT)
+def test_report_exits_1_naming_a_corrupt_result_file(tmp_path, capsys, content):
+    batch = _batch(tmp_path, "b1", {"t__a__m__r1/result.json": content})
+    with pytest.raises(SystemExit) as exc:
+        _run("report", str(batch))
+    assert exc.value.code == 1
+    err = capsys.readouterr().err
+    assert str(batch / "t__a__m__r1" / "result.json") in err
+    assert "Traceback" not in err
+
+
+@pytest.mark.parametrize("content", _CORRUPT)
+def test_compare_exits_1_naming_a_corrupt_result_file(tmp_path, capsys, content):
+    batch = _batch(tmp_path, "b1", {"t__a__m__r1/result.json": content})
+    with pytest.raises(SystemExit) as exc:
+        _run("compare", str(batch), "--baseline", "a")
+    assert exc.value.code == 1
+    assert str(batch / "t__a__m__r1" / "result.json") in capsys.readouterr().err
+
+
+def test_report_exits_1_naming_a_corrupt_unsupported_file(tmp_path, capsys):
+    batch = _batch(tmp_path, "b1", {"t__a__m__r1/unsupported.json": b"{not json"})
+    with pytest.raises(SystemExit) as exc:
+        _run("report", str(batch))
+    assert exc.value.code == 1
+    assert str(batch / "t__a__m__r1" / "unsupported.json") in capsys.readouterr().err
+
+
+@pytest.mark.parametrize("content", [b"{not json", b"{}", b'{"cells": 3}', b'{"cells": [1]}'])
+def test_report_exits_1_naming_a_corrupt_plan_file(tmp_path, capsys, content):
+    batch = _batch(tmp_path, "b1", {"_eval/plan.json": content})
+    with pytest.raises(SystemExit) as exc:
+        _run("report", str(batch))
+    assert exc.value.code == 1
+    assert str(batch / "_eval" / "plan.json") in capsys.readouterr().err
+
+
+def test_report_still_reads_an_empty_batch(tmp_path, capsys):
+    batch = _batch(tmp_path, "b1", {"_eval/plan.json": json.dumps({"cells": []}).encode()})
+    _run("report", str(batch), "--json")
+    assert json.loads(capsys.readouterr().out)["rows"] == []

From e7bf343a81b740989bde87d9db9b26b763b294c1 Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Sun, 25 Oct 2026 16:29:41 -0500
Subject: [PATCH 09/49] fix(explore): run exits 1 with a message when
 --out-json or --out-md cannot be written

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/commands/explore.py        | 15 ++++++--
 dokimi/tests/test_explore_run_outputs.py | 45 ++++++++++++++++++++++++
 2 files changed, 57 insertions(+), 3 deletions(-)
 create mode 100644 dokimi/tests/test_explore_run_outputs.py

diff --git a/dokimi/dokimi/commands/explore.py b/dokimi/dokimi/commands/explore.py
index a34bd9074..8fa7c000c 100644
--- a/dokimi/dokimi/commands/explore.py
+++ b/dokimi/dokimi/commands/explore.py
@@ -6,6 +6,7 @@
 from __future__ import annotations
 
 import argparse
+import sys
 from pathlib import Path
 
 _TASKS_ROOT_HELP = (
@@ -111,14 +112,22 @@ def cmd_explore_run(args: argparse.Namespace) -> None:
     output = runner.run()
 
     out_json = Path(args.out_json) if args.out_json else output_root / "explore_runs.json"
-    save_artifact("explore_runs", output, out_json)
+    try:
+        save_artifact("explore_runs", output, out_json)
+    except OSError as exc:
+        print(f"Error: cannot write {out_json}: {exc}", file=sys.stderr)
+        raise SystemExit(1) from exc
     print(f"Saved explore run artifact: {out_json}")
 
     if args.out_md:
         from dokimi.report.exploration_report import render_explore_report
 
-        Path(args.out_md).parent.mkdir(parents=True, exist_ok=True)
-        Path(args.out_md).write_text(render_explore_report(output), encoding="utf-8")
+        try:
+            Path(args.out_md).parent.mkdir(parents=True, exist_ok=True)
+            Path(args.out_md).write_text(render_explore_report(output), encoding="utf-8")
+        except OSError as exc:
+            print(f"Error: cannot write {args.out_md}: {exc}", file=sys.stderr)
+            raise SystemExit(1) from exc
         print(f"Saved exploration report: {args.out_md}")
 
     for arm, stats in output["summary"]["per_arm"].items():
diff --git a/dokimi/tests/test_explore_run_outputs.py b/dokimi/tests/test_explore_run_outputs.py
new file mode 100644
index 000000000..72d6af8fe
--- /dev/null
+++ b/dokimi/tests/test_explore_run_outputs.py
@@ -0,0 +1,45 @@
+# Copyright (C) 2026 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+"""`explore run`: an output path that cannot be written exits 1 with a message."""
+
+from __future__ import annotations
+
+import pytest
+
+from dokimi.cli import build_parser
+
+TASK = "id: t\nproduct: p\nquery: q\nreferences:\n  - path: a.py\n"
+
+
+def _run(tmp_path, *extra):
+    tasks = tmp_path / "tasks"
+    tasks.mkdir(exist_ok=True)
+    (tasks / "t.yaml").write_text(TASK, encoding="utf-8")
+    args = build_parser().parse_args([
+        "explore", "run", "--all", "--arms", "oracle,empty", "--tasks-root", str(tasks),
+        "--repo-root", str(tmp_path), "--output-dir", str(tmp_path / "out"), *extra,
+    ])
+    args.func(args)
+
+
+def _blocker(tmp_path):
+    blocker = tmp_path / "blocker"
+    blocker.write_text("a file, not a directory", encoding="utf-8")
+    return blocker
+
+
+@pytest.mark.parametrize("flag", ["--out-json", "--out-md"])
+def test_an_unwritable_output_is_reported_not_a_traceback(tmp_path, capsys, flag):
+    target = _blocker(tmp_path) / "report"
+    with pytest.raises(SystemExit) as exc:
+        _run(tmp_path, flag, str(target))
+    assert exc.value.code == 1
+    assert f"Error: cannot write {target}" in capsys.readouterr().err
+
+
+def test_a_writable_output_still_saves_both_files(tmp_path, capsys):
+    out_json, out_md = tmp_path / "r.json", tmp_path / "r.md"
+    _run(tmp_path, "--out-json", str(out_json), "--out-md", str(out_md))
+    assert out_json.is_file() and out_md.is_file()
+    assert "oracle: n=1" in capsys.readouterr().out

From fc00a6dec43197576a2881d6d7b907a5e3f200d2 Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Sun, 25 Oct 2026 16:29:41 -0500
Subject: [PATCH 10/49] fix(repo): budget and reach exit 1 with a message on an
 unreadable or non-UTF-8 --harness file

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/repos/harness_profile.py        |  2 +
 .../test_repo_harness_profile_unreadable.py   | 54 +++++++++++++++++++
 2 files changed, 56 insertions(+)
 create mode 100644 dokimi/tests/test_repo_harness_profile_unreadable.py

diff --git a/dokimi/dokimi/repos/harness_profile.py b/dokimi/dokimi/repos/harness_profile.py
index c4ac84fe2..fafc4250d 100644
--- a/dokimi/dokimi/repos/harness_profile.py
+++ b/dokimi/dokimi/repos/harness_profile.py
@@ -150,6 +150,8 @@ def load_profile(name_or_path: str | Path) -> HarnessProfile:
         raw = yaml.safe_load(path.read_text(encoding="utf-8"))
     except yaml.YAMLError as exc:
         raise ProfileError(f"{path}: not valid YAML: {exc}") from exc
+    except (OSError, UnicodeDecodeError) as exc:
+        raise ProfileError(f"{path}: cannot read: {exc}") from exc
     return parse_profile(raw, str(path))
 
 
diff --git a/dokimi/tests/test_repo_harness_profile_unreadable.py b/dokimi/tests/test_repo_harness_profile_unreadable.py
new file mode 100644
index 000000000..c82592fa8
--- /dev/null
+++ b/dokimi/tests/test_repo_harness_profile_unreadable.py
@@ -0,0 +1,54 @@
+# Copyright (C) 2026 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+"""`repo budget` and `repo reach` exit 1 with a message on a --harness file that cannot be read."""
+
+from __future__ import annotations
+
+import os
+
+import pytest
+
+from dokimi.cli import build_parser
+from dokimi.repos.harness_profile import ProfileError, load_profile
+
+
+def _run(*argv: str) -> None:
+    args = build_parser().parse_args(["repo", *argv])
+    args.func(args)
+
+
+def _bad_profile(tmp_path, kind: str):
+    path = tmp_path / "profile.yaml"
+    path.write_bytes(b"name: \xff\xfe\n")
+    if kind == "unreadable":
+        if os.name == "nt" or os.geteuid() == 0:
+            pytest.skip("file permissions do not stop this user from reading")
+        path.write_text("name: x\n", encoding="utf-8")
+        path.chmod(0)
+    return path
+
+
+@pytest.mark.parametrize("kind", ["non-utf8", "unreadable"])
+def test_load_profile_names_the_file_it_cannot_read(tmp_path, kind):
+    path = _bad_profile(tmp_path, kind)
+    with pytest.raises(ProfileError, match="profile.yaml"):
+        load_profile(path)
+
+
+@pytest.mark.parametrize("kind", ["non-utf8", "unreadable"])
+def test_repo_budget_exits_1_on_a_harness_file_it_cannot_read(tmp_path, capsys, kind):
+    path = _bad_profile(tmp_path, kind)
+    with pytest.raises(SystemExit) as exc:
+        _run("budget", str(tmp_path), "--harness", str(path))
+    assert exc.value.code == 1
+    assert str(path) in capsys.readouterr().err
+
+
+@pytest.mark.parametrize("kind", ["non-utf8", "unreadable"])
+def test_repo_reach_exits_1_on_a_harness_file_it_cannot_read(tmp_path, capsys, kind):
+    path = _bad_profile(tmp_path, kind)
+    with pytest.raises(SystemExit) as exc:
+        _run("reach", str(tmp_path), "--model", "m", "--harness", str(path), "--dry-run")
+    assert exc.value.code == 1
+    assert str(path) in capsys.readouterr().err

From eea0ff2f16daedf5b95b2de2f3ae78e460327cff Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Sun, 25 Oct 2026 16:29:41 -0500
Subject: [PATCH 11/49] fix(arms): plugin-delta exits 1 with a message when an
 output cannot be written

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/commands/arms.py          | 28 ++++++++++++---
 dokimi/tests/test_arms_output_errors.py | 47 +++++++++++++++++++++++++
 2 files changed, 71 insertions(+), 4 deletions(-)
 create mode 100644 dokimi/tests/test_arms_output_errors.py

diff --git a/dokimi/dokimi/commands/arms.py b/dokimi/dokimi/commands/arms.py
index c4b4cbac8..9b5b1549b 100644
--- a/dokimi/dokimi/commands/arms.py
+++ b/dokimi/dokimi/commands/arms.py
@@ -177,7 +177,7 @@ def cmd_arms_matrix_run(args: argparse.Namespace) -> None:
 
 def cmd_arms_plugin_delta(args: argparse.Namespace) -> None:
     """Compare paired no-plugin/plugin arms artifacts."""
-    from dokimi.artifacts import load_artifact, save_artifact
+    from dokimi.artifacts import load_artifact
     from dokimi.eval.plugin_delta import PluginDeltaError, compare_plugin_runs
     from dokimi.records import records
     from dokimi.report.plugin_delta_report import render_plugin_delta_report
@@ -194,12 +194,11 @@ def cmd_arms_plugin_delta(args: argparse.Namespace) -> None:
         sys.exit(1)
 
     out_json = Path(args.out_json) if args.out_json else Path("results/arms/plugin_delta.json")
-    save_artifact("plugin_delta", output, out_json)
+    _save_artifact_or_exit("plugin_delta", output, out_json)
     print(f"OK Saved plugin delta: {out_json}")
 
     out_md = Path(args.out_md) if args.out_md else out_json.with_suffix(".md")
-    out_md.parent.mkdir(parents=True, exist_ok=True)
-    out_md.write_text(render_plugin_delta_report(output), encoding="utf-8")
+    _write_report(out_md, render_plugin_delta_report(output))
     print(f"OK Saved plugin delta report: {out_md}")
 
     print(
@@ -726,6 +725,27 @@ def _build_arms_output(args: argparse.Namespace, specs: list[str], questions: li
     )
 
 
+def _write_report(path: Path, text: str) -> None:
+    """Write a Markdown report, or exit 1 naming ``path`` when it cannot be written."""
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(text, encoding="utf-8")
+    except OSError as exc:
+        print(f"Error: cannot write {path}: {exc}", file=sys.stderr)
+        sys.exit(1)
+
+
+def _save_artifact_or_exit(kind: str, payload: dict, path: Path) -> None:
+    """``save_artifact``, or exit 1 naming ``path`` when it cannot be written."""
+    from dokimi.artifacts import save_artifact
+
+    try:
+        save_artifact(kind, payload, path)
+    except OSError as exc:
+        print(f"Error: cannot write {path}: {exc}", file=sys.stderr)
+        sys.exit(1)
+
+
 def _save_arms_output(output: dict, out_json: Path, out_md: Path) -> None:
     from dokimi.eval.arm_runner import ArmRunner
     from dokimi.report.arms_report import render_arms_report
diff --git a/dokimi/tests/test_arms_output_errors.py b/dokimi/tests/test_arms_output_errors.py
new file mode 100644
index 000000000..bd3e514f3
--- /dev/null
+++ b/dokimi/tests/test_arms_output_errors.py
@@ -0,0 +1,47 @@
+# Copyright (C) 2026 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+"""`arms plugin-delta`: an unwritable output exits 1 with a message."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from dokimi.cli import build_parser
+
+from test_plugin_delta import _arms_artifact
+
+
+def _plugin_delta(tmp_path: Path, *out: str) -> None:
+    baseline = tmp_path / "baseline.json"
+    plugin = tmp_path / "plugin.json"
+    baseline.write_text(json.dumps(_arms_artifact()), encoding="utf-8")
+    plugin.write_text(json.dumps(_arms_artifact("caveman:full")), encoding="utf-8")
+    args = build_parser().parse_args([
+        "arms", "plugin-delta", "--baseline-json", str(baseline), "--plugin-json", str(plugin), *out,
+    ])
+    args.func(args)
+
+
+def test_unwritable_out_json_exits_1_naming_the_path(tmp_path, capsys):
+    blocker = tmp_path / "blocker"
+    blocker.write_text("x", encoding="utf-8")  # a regular file: nothing can be created below it
+    target = blocker / "delta.json"
+    with pytest.raises(SystemExit) as exc:
+        _plugin_delta(tmp_path, "--out-json", str(target))
+    assert exc.value.code == 1
+    err = capsys.readouterr().err
+    assert f"Error: cannot write {target}" in err
+    assert "Traceback" not in err
+
+
+def test_unwritable_out_md_exits_1_naming_the_path(tmp_path, capsys):
+    out_md = tmp_path / "is-a-dir"
+    out_md.mkdir()
+    with pytest.raises(SystemExit) as exc:
+        _plugin_delta(tmp_path, "--out-json", str(tmp_path / "delta.json"), "--out-md", str(out_md))
+    assert exc.value.code == 1
+    assert f"Error: cannot write {out_md}" in capsys.readouterr().err

From 4eaef5bc04ccfb775a142056b83d7c769678e71c Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Sun, 25 Oct 2026 22:37:28 -0500
Subject: [PATCH 12/49] chore: changelog fragment and growth baseline

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 .../20261025_120000_vchamarthi_cli-input-run-arms-repo.md      | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-run-arms-repo.md

diff --git a/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-run-arms-repo.md b/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-run-arms-repo.md
new file mode 100644
index 000000000..7255afe9e
--- /dev/null
+++ b/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-run-arms-repo.md
@@ -0,0 +1,3 @@
+### Fixed
+
+- **`run`, `compare`, `arms`, `tasks`, `explore` and `repo`** exit with a message on an unreadable, non-UTF-8 or corrupt input or an unwritable output, instead of a traceback. `run` and `compare` exit 2 on a bad spec or snapshot.

From 51a65ee93b59184a11b577b4778c992bd6c97e20 Mon Sep 17 00:00:00 2001
From: Anton Volkov <antonwolfy@gmail.com>
Date: Mon, 26 Oct 2026 13:22:56 +0000
Subject: [PATCH 13/49] Import a public load_benchmark_spec in the run command

Signed-off-by: Anton Volkov <antonwolfy@gmail.com>
---
 dokimi/dokimi/commands/run.py | 4 ++--
 dokimi/dokimi/runner/run.py   | 3 +++
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/dokimi/dokimi/commands/run.py b/dokimi/dokimi/commands/run.py
index b9aa2fb4b..31ba8cbbc 100644
--- a/dokimi/dokimi/commands/run.py
+++ b/dokimi/dokimi/commands/run.py
@@ -13,13 +13,13 @@
 from dokimi.report.json_report import write_json
 from dokimi.report.markdown_report import write_compare_report, write_run_report
 from dokimi.runner.compare import compare_snapshots
-from dokimi.runner.run import DocLoadError, _load_spec, run_benchmark, save_snapshot
+from dokimi.runner.run import DocLoadError, load_benchmark_spec, run_benchmark, save_snapshot
 
 
 def _run_or_exit(root: Path, spec_path: Path) -> dict:
     """Run the benchmark; a bad spec exits 2 (1 is the verdict code of --strict), a bad doc exits 1."""
     try:
-        _load_spec(spec_path)  # fail on a spec that cannot be read, parsed or validated, before the run
+        load_benchmark_spec(spec_path)  # fail on a spec that cannot be read, parsed or validated, before the run
     except (RuntimeError, ValueError) as exc:
         print(f"Error: {spec_path}: {exc}", file=sys.stderr)
         sys.exit(2)
diff --git a/dokimi/dokimi/runner/run.py b/dokimi/dokimi/runner/run.py
index a4f99e85f..e2b9c27d8 100644
--- a/dokimi/dokimi/runner/run.py
+++ b/dokimi/dokimi/runner/run.py
@@ -69,6 +69,9 @@ def _load_spec(spec_path: Path) -> dict:
     return data
 
 
+load_benchmark_spec = _load_spec
+
+
 class DocLoadError(RuntimeError):
     """A document the run selected could not be read: the run refuses rather than score around it."""
 

From 9ed23f4b90a4f7a2c0a78d304a90a22b075a34c6 Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Mon, 26 Oct 2026 11:04:09 -0500
Subject: [PATCH 14/49] Exit with a message on unwritable run, compare and arms
 outputs and on bad snapshot values

run and compare exit 1 naming the path when an output cannot be written, and
run exits 2 when the manifest's min_docs is not met. compare rejects
non-numeric summary values. The arms output, matrix rollup and plugin delta
writes go through the shared write helpers. repo report and compare reject a
result file without a usable grade. tasks error messages use the Error: prefix.

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 ...0000_vchamarthi_cli-input-run-arms-repo.md |  2 +-
 dokimi/dokimi/commands/arms.py                | 27 ++++----
 dokimi/dokimi/commands/run.py                 | 19 ++++--
 dokimi/dokimi/commands/tasks.py               | 10 +--
 dokimi/dokimi/repos/report.py                 |  3 +
 dokimi/dokimi/runner/compare.py               |  5 ++
 dokimi/tests/test_arms_output_errors.py       | 47 ++++++++++++++
 .../tests/test_repo_report_unreadable_cell.py | 20 ++++++
 dokimi/tests/test_run_compare_errors.py       | 65 +++++++++++++++++++
 9 files changed, 176 insertions(+), 22 deletions(-)

diff --git a/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-run-arms-repo.md b/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-run-arms-repo.md
index 7255afe9e..2be49e3d3 100644
--- a/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-run-arms-repo.md
+++ b/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-run-arms-repo.md
@@ -1,3 +1,3 @@
 ### Fixed
 
-- **`run`, `compare`, `arms`, `tasks`, `explore` and `repo`** exit with a message on an unreadable, non-UTF-8 or corrupt input or an unwritable output, instead of a traceback. `run` and `compare` exit 2 on a bad spec or snapshot.
+- **`run`, `compare`, `arms`, `tasks`, `explore` and `repo`** exit with a message on an unreadable, non-UTF-8 or corrupt input or an unwritable output, instead of a traceback. `run` and `compare` exit 2 on a bad spec or snapshot (a non-numeric summary value, or fewer documents than the manifest's `min_docs`) and exit 1 when an output cannot be written, as `arms` does. `repo report` and `repo compare` exit 1 on a result file without a usable `grade`. `tasks` error messages use the `Error:` prefix.
diff --git a/dokimi/dokimi/commands/arms.py b/dokimi/dokimi/commands/arms.py
index 9b5b1549b..749ce2113 100644
--- a/dokimi/dokimi/commands/arms.py
+++ b/dokimi/dokimi/commands/arms.py
@@ -108,7 +108,6 @@ def cmd_arms_run(args: argparse.Namespace) -> None:
 
 def cmd_arms_matrix_run(args: argparse.Namespace) -> None:
     """Run every cell in a matrix.cells descriptor and write a rollup."""
-    from dokimi.artifacts import save_artifact
     from dokimi.eval.cells import load_matrix_cells
     from dokimi.eval.matrix_rollup import build_matrix_rollup, strip_internal_payloads
     from dokimi.report.matrix_rollup_report import render_matrix_rollup_report
@@ -129,9 +128,12 @@ def cmd_arms_matrix_run(args: argparse.Namespace) -> None:
         sys.exit(1)
 
     out_dir = Path(args.out_dir) if args.out_dir else Path(f"results/arms/{args.product}-matrix")
-    out_dir.mkdir(parents=True, exist_ok=True)
     cells_dir = out_dir / "cells"
-    cells_dir.mkdir(parents=True, exist_ok=True)
+    try:
+        cells_dir.mkdir(parents=True, exist_ok=True)
+    except OSError as exc:
+        print(f"Error: cannot write {cells_dir}: {exc}", file=sys.stderr)
+        sys.exit(1)
     cell_artifacts = []
     for cell in cells:
         cell_args = argparse.Namespace(**vars(args))
@@ -160,17 +162,15 @@ def cmd_arms_matrix_run(args: argparse.Namespace) -> None:
         if not payload:
             continue
         delta_json = Path(row["artifact"])
-        save_artifact("plugin_delta", payload, delta_json)
-        delta_md = delta_json.with_suffix(".md")
-        delta_md.write_text(render_plugin_delta_report(payload), encoding="utf-8")
+        _save_artifact_or_exit("plugin_delta", payload, delta_json)
+        _write_report(delta_json.with_suffix(".md"), render_plugin_delta_report(payload))
         print(f"OK Saved plugin delta:    {delta_json}")
 
     persisted = strip_internal_payloads(rollup)
     out_json = Path(args.out_json) if args.out_json else out_dir / "matrix-rollup.json"
     out_md = Path(args.out_md) if args.out_md else out_json.with_suffix(".md")
-    save_artifact("matrix_rollup", persisted, out_json)
-    out_md.parent.mkdir(parents=True, exist_ok=True)
-    out_md.write_text(render_matrix_rollup_report(persisted), encoding="utf-8")
+    _save_artifact_or_exit("matrix_rollup", persisted, out_json)
+    _write_report(out_md, render_matrix_rollup_report(persisted))
     print(f"OK Saved matrix rollup:   {out_json}")
     print(f"OK Saved matrix report:   {out_md}")
 
@@ -750,10 +750,13 @@ def _save_arms_output(output: dict, out_json: Path, out_md: Path) -> None:
     from dokimi.eval.arm_runner import ArmRunner
     from dokimi.report.arms_report import render_arms_report
 
-    ArmRunner.save(output, out_json)
+    try:
+        ArmRunner.save(output, out_json)
+    except OSError as exc:
+        print(f"Error: cannot write {out_json}: {exc}", file=sys.stderr)
+        sys.exit(1)
     print(f"OK Saved arms comparison: {out_json}")
-    out_md.parent.mkdir(parents=True, exist_ok=True)
-    out_md.write_text(render_arms_report(output), encoding="utf-8")
+    _write_report(out_md, render_arms_report(output))
     print(f"OK Saved arms report:     {out_md}")
 
 
diff --git a/dokimi/dokimi/commands/run.py b/dokimi/dokimi/commands/run.py
index 31ba8cbbc..68d33d299 100644
--- a/dokimi/dokimi/commands/run.py
+++ b/dokimi/dokimi/commands/run.py
@@ -28,6 +28,19 @@ def _run_or_exit(root: Path, spec_path: Path) -> dict:
     except DocLoadError as exc:
         print(f"❌ {exc}", file=sys.stderr)
         sys.exit(1)
+    except ValueError as exc:  # e.g. fewer documents than the manifest's min_docs
+        print(f"Error: {exc}", file=sys.stderr)
+        sys.exit(2)
+
+
+def _write_outputs(writes) -> None:
+    """Run each (writer, data, path); an unwritable path exits 1 naming it."""
+    for writer, data, path in writes:
+        try:
+            writer(data, path)
+        except OSError as exc:
+            print(f"Error: cannot write {path}: {exc}", file=sys.stderr)
+            sys.exit(1)
 
 
 def cmd_run(args: argparse.Namespace) -> None:
@@ -38,8 +51,7 @@ def cmd_run(args: argparse.Namespace) -> None:
     out_md = Path(args.out_md).resolve()
 
     result = _run_or_exit(root, spec_path)
-    save_snapshot(result, out_json)
-    write_run_report(result, out_md)
+    _write_outputs(((save_snapshot, result, out_json), (write_run_report, result, out_md)))
     print(json.dumps(result["summary"], indent=2))
 
     # Strict mode: check hard gate and critical bands
@@ -99,8 +111,7 @@ def cmd_compare(args: argparse.Namespace) -> None:
     except ValueError as exc:
         print(f"Error: {exc}", file=sys.stderr)
         sys.exit(2)
-    write_json(result, out_json)
-    write_compare_report(result, out_md)
+    _write_outputs(((write_json, result, out_json), (write_compare_report, result, out_md)))
     print(json.dumps(result["diff"], indent=2))
 
 
diff --git a/dokimi/dokimi/commands/tasks.py b/dokimi/dokimi/commands/tasks.py
index e5dc5a951..49f3da24a 100644
--- a/dokimi/dokimi/commands/tasks.py
+++ b/dokimi/dokimi/commands/tasks.py
@@ -78,13 +78,13 @@ def cmd_tasks_inspect(args: argparse.Namespace) -> None:
     try:
         reports = [r.as_dict() for r in inspect_path(Path(args.path))]
     except FileNotFoundError as exc:
-        raise SystemExit(f"error: {exc}") from None
+        raise SystemExit(f"Error: {exc}") from None
     payload = {"root": str(args.path), "items": reports}
     if args.out:
         try:
             save_artifact("task_inspect", payload, Path(args.out))
         except OSError as exc:
-            raise SystemExit(f"error: cannot write {args.out}: {exc}") from None
+            raise SystemExit(f"Error: cannot write {args.out}: {exc}") from None
     if args.json:
         print(json.dumps(stamp("task_inspect", dict(payload)), indent=2, default=str))
     else:
@@ -199,7 +199,7 @@ def cmd_tasks_run(args: argparse.Namespace) -> None:
         output = runner.run()
     except PreflightError as exc:
         # A missing tool (bench, Compose v2): one line, not a traceback.
-        raise SystemExit(f"error: {exc}") from None
+        raise SystemExit(f"Error: {exc}") from None
 
     out_json = Path(args.out_json) if args.out_json else Path(args.output_dir) / "task_runs.json"
     from dokimi.artifacts import save_artifact
@@ -207,7 +207,7 @@ def cmd_tasks_run(args: argparse.Namespace) -> None:
     try:
         save_artifact("task_runs", output, out_json)
     except OSError as exc:
-        raise SystemExit(f"error: cannot write {out_json}: {exc}") from None
+        raise SystemExit(f"Error: cannot write {out_json}: {exc}") from None
     print(f"Saved task run artifact: {out_json}")
 
     out_md = Path(args.out_md) if args.out_md else out_json.with_suffix(".md")
@@ -217,7 +217,7 @@ def cmd_tasks_run(args: argparse.Namespace) -> None:
         out_md.parent.mkdir(parents=True, exist_ok=True)
         out_md.write_text(render_task_runs_report(output), encoding="utf-8")
     except OSError as exc:
-        raise SystemExit(f"error: cannot write {out_md}: {exc} (the artifact is saved at {out_json})") from None
+        raise SystemExit(f"Error: cannot write {out_md}: {exc} (the artifact is saved at {out_json})") from None
     print(f"Saved task run report:   {out_md}")
 
     _print_summary(output)
diff --git a/dokimi/dokimi/repos/report.py b/dokimi/dokimi/repos/report.py
index 7de095ca2..346cdab08 100644
--- a/dokimi/dokimi/repos/report.py
+++ b/dokimi/dokimi/repos/report.py
@@ -68,6 +68,9 @@ def load_cells(batch_dir: Path) -> tuple[list[dict[str, Any]], list[str]]:
     finished = []
     for path in sorted(batch_dir.glob("*/result.json")):
         rec = _read_object(path)
+        grade = rec.get("grade")
+        if not isinstance(grade, dict) or "status" not in grade:
+            raise CellFileError(f"{path} has no usable `grade`")
         rec["cell"] = path.parent.name
         # The row's kind is the task's, not what its cells' grades say. The runner records it from the
         # eval dir's task.json; the cell's copy is in the run dir, which the grader could rewrite.
diff --git a/dokimi/dokimi/runner/compare.py b/dokimi/dokimi/runner/compare.py
index 7416a8fa7..3fe2cddbc 100644
--- a/dokimi/dokimi/runner/compare.py
+++ b/dokimi/dokimi/runner/compare.py
@@ -32,6 +32,11 @@ def compare_snapshots(base_path: Path, cand_path: Path, spec: dict | None = None
         # example_pass_rate is optional (may not be present in old snapshots)
         if missing:
             raise ValueError(f"Snapshot {path} missing summary keys: {sorted(missing)}")
+        role = "base" if path == base_path else "candidate"
+        for key in sorted(required | {"example_pass_rate"}):
+            value = summary.get(key)
+            if key in summary and (isinstance(value, bool) or not isinstance(value, (int, float))):
+                raise ValueError(f"Invalid {role} snapshot {path}: summary `{key}` must be a number, got {value!r}")
 
     diff = {
         "docs": cand["summary"]["docs"] - base["summary"]["docs"],
diff --git a/dokimi/tests/test_arms_output_errors.py b/dokimi/tests/test_arms_output_errors.py
index bd3e514f3..0c05038ae 100644
--- a/dokimi/tests/test_arms_output_errors.py
+++ b/dokimi/tests/test_arms_output_errors.py
@@ -45,3 +45,50 @@ def test_unwritable_out_md_exits_1_naming_the_path(tmp_path, capsys):
         _plugin_delta(tmp_path, "--out-json", str(tmp_path / "delta.json"), "--out-md", str(out_md))
     assert exc.value.code == 1
     assert f"Error: cannot write {out_md}" in capsys.readouterr().err
+
+
+def _args(*argv: str):
+    return build_parser().parse_args(["arms", *argv])
+
+
+def _blocker(tmp_path: Path) -> Path:
+    blocker = tmp_path / "blocker"
+    blocker.write_text("x", encoding="utf-8")
+    return blocker
+
+
+def test_save_arms_output_exits_1_naming_an_unwritable_json(tmp_path, capsys):
+    from dokimi.commands.arms import _save_arms_output
+
+    target = _blocker(tmp_path) / "arms.json"
+    with pytest.raises(SystemExit) as exc:
+        _save_arms_output({}, target, tmp_path / "arms.md")
+    assert exc.value.code == 1
+    assert f"Error: cannot write {target}" in capsys.readouterr().err
+
+
+def test_save_arms_output_exits_1_naming_an_unwritable_report(tmp_path, capsys):
+    from dokimi.commands.arms import _save_arms_output
+
+    out_md = tmp_path / "is-a-dir"
+    out_md.mkdir()
+    output = _arms_artifact()
+    with pytest.raises(SystemExit) as exc:
+        _save_arms_output(output, tmp_path / "arms.json", out_md)
+    assert exc.value.code == 1
+    assert f"Error: cannot write {out_md}" in capsys.readouterr().err
+
+
+def test_matrix_run_exits_1_naming_an_uncreatable_out_dir(tmp_path, capsys):
+    cells = tmp_path / "cells.json"
+    cells.write_text(json.dumps({"cells": [{"name": "c1", "model": "m", "provider": "anthropic", "harness": "arms-runner"}]}), encoding="utf-8")
+    questions = tmp_path / "questions.json"
+    questions.write_text(json.dumps({"questions": [{"id": "q1", "question": "What is x?"}]}), encoding="utf-8")
+    out_dir = _blocker(tmp_path) / "out"
+    args = _args("matrix-run", "--product", "p", "--questions", str(questions), "--arms", "baseline",
+                 "--matrix-cells", str(cells), "--out-dir", str(out_dir))
+    with pytest.raises(SystemExit) as exc:
+        args.func(args)
+    assert exc.value.code == 1
+    err = capsys.readouterr().err
+    assert f"Error: cannot write {out_dir / 'cells'}" in err and "Traceback" not in err
diff --git a/dokimi/tests/test_repo_report_unreadable_cell.py b/dokimi/tests/test_repo_report_unreadable_cell.py
index cb271541f..c0093e3a0 100644
--- a/dokimi/tests/test_repo_report_unreadable_cell.py
+++ b/dokimi/tests/test_repo_report_unreadable_cell.py
@@ -74,3 +74,23 @@ def test_report_still_reads_an_empty_batch(tmp_path, capsys):
     batch = _batch(tmp_path, "b1", {"_eval/plan.json": json.dumps({"cells": []}).encode()})
     _run("report", str(batch), "--json")
     assert json.loads(capsys.readouterr().out)["rows"] == []
+
+
+_NO_GRADE = [
+    pytest.param(b"{}", id="empty"),
+    pytest.param(b'{"grade": null}', id="null"),
+    pytest.param(b'{"grade": []}', id="list"),
+    pytest.param(b'{"grade": {"score_only": false}, "task": "t", "arm": "a", "model": "m"}', id="no-status"),
+]
+
+
+@pytest.mark.parametrize("content", _NO_GRADE)
+@pytest.mark.parametrize("argv", [("report",), ("compare", "--baseline", "a")], ids=["report", "compare"])
+def test_a_result_file_without_a_usable_grade_exits_1(tmp_path, capsys, argv, content):
+    batch = _batch(tmp_path, "b1", {"t__a__m__r1/result.json": content})
+    with pytest.raises(SystemExit) as exc:
+        _run(argv[0], str(batch), *argv[1:])
+    assert exc.value.code == 1
+    err = capsys.readouterr().err
+    assert f"{batch / 't__a__m__r1' / 'result.json'} has no usable `grade`" in err
+    assert "Traceback" not in err
diff --git a/dokimi/tests/test_run_compare_errors.py b/dokimi/tests/test_run_compare_errors.py
index 812379d81..198142692 100644
--- a/dokimi/tests/test_run_compare_errors.py
+++ b/dokimi/tests/test_run_compare_errors.py
@@ -69,3 +69,68 @@ def test_compare_exits_2_on_a_bad_spec(tmp_path, monkeypatch, capsys, content):
     assert code == 2
     err = capsys.readouterr().err
     assert err.startswith("Error: ") and str(spec) in err
+
+
+_SPEC_MIN_DOCS = (
+    "weights:\n  coverage: 0.4\n  freshness_lite: 0.3\n  readability: 0.3\n"
+    "metrics:\n  freshness_lite:\n    max_age_days: 365\n  readability:\n    grade_max: 12\n"
+    "golden_manifest:\n  include: ['docs/**/*.md']\n  exclude: []\n  min_docs: 3\n  max_docs: 10\n"
+)
+
+
+def test_run_exits_2_when_the_root_has_fewer_docs_than_min_docs(tmp_path, monkeypatch, capsys):
+    (tmp_path / "docs").mkdir()
+    (tmp_path / "docs" / "a.md").write_text("# A\n\ntext\n", encoding="utf-8")
+    spec = tmp_path / "spec.yaml"
+    spec.write_text(_SPEC_MIN_DOCS, encoding="utf-8")
+    code = _main(monkeypatch, "run", "--root", str(tmp_path), "--spec", str(spec),
+                 "--out-json", str(tmp_path / "o.json"), "--out-md", str(tmp_path / "o.md"))
+    assert code == 2
+    err = capsys.readouterr().err
+    assert err.startswith("Error: ") and "min_docs" in err and "Traceback" not in err
+
+
+@pytest.mark.parametrize("which", ["--base", "--candidate"])
+@pytest.mark.parametrize("field,value", [("docs", "1"), ("score", None), ("coverage", True)])
+def test_compare_exits_2_on_a_non_numeric_summary_value(tmp_path, monkeypatch, capsys, which, field, value):
+    bad = tmp_path / "bad.json"
+    bad.write_text(json.dumps({"summary": {**SNAPSHOT["summary"], field: value}}), encoding="utf-8")
+    good = _snapshot(tmp_path / "good.json")
+    pair = {"--base": good, "--candidate": good, which: str(bad)}
+    code = _main(monkeypatch, "compare", "--base", pair["--base"], "--candidate", pair["--candidate"],
+                 "--out-json", str(tmp_path / "c.json"), "--out-md", str(tmp_path / "c.md"))
+    assert code == 2
+    err = capsys.readouterr().err
+    assert err.startswith("Error: ") and str(bad) in err and f"`{field}`" in err
+
+
+def _blocked(tmp_path):
+    blocker = tmp_path / "blocker"
+    blocker.write_text("x", encoding="utf-8")  # a regular file: nothing can be created below it
+    return blocker
+
+
+def test_compare_exits_1_when_an_output_cannot_be_written(tmp_path, monkeypatch, capsys):
+    good = _snapshot(tmp_path / "good.json")
+    target = _blocked(tmp_path) / "c.json"
+    code = _main(monkeypatch, "compare", "--base", good, "--candidate", good,
+                 "--out-json", str(target), "--out-md", str(tmp_path / "c.md"))
+    assert code == 1
+    err = capsys.readouterr().err
+    assert f"Error: cannot write {target}" in err and "Traceback" not in err
+
+
+@pytest.mark.parametrize("which", ["json", "md"])
+def test_run_exits_1_when_an_output_cannot_be_written(tmp_path, monkeypatch, capsys, which):
+    (tmp_path / "docs").mkdir()
+    (tmp_path / "docs" / "a.md").write_text("# A\n\ntext\n", encoding="utf-8")
+    spec = tmp_path / "spec.yaml"
+    spec.write_text(_SPEC_MIN_DOCS.replace("min_docs: 3", "min_docs: 1"), encoding="utf-8")
+    blocked = _blocked(tmp_path) / "out"
+    outs = {"json": str(tmp_path / "o.json"), "md": str(tmp_path / "o.md")}
+    outs[which] = str(blocked)
+    code = _main(monkeypatch, "run", "--root", str(tmp_path), "--spec", str(spec),
+                 "--out-json", outs["json"], "--out-md", outs["md"])
+    assert code == 1
+    err = capsys.readouterr().err
+    assert f"Error: cannot write {blocked}" in err and "Traceback" not in err

From 345757bee1b73693146650b449fb71187321eb9b Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Mon, 26 Oct 2026 11:12:09 -0500
Subject: [PATCH 15/49] Report deeply nested JSON in matrix cells and snapshots
 as an input error

load_matrix_cells and the compare snapshot readers now catch RecursionError next to ValueError, so a pathologically nested file exits with a message naming it instead of a traceback.

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/eval/cells.py                  | 2 +-
 dokimi/dokimi/runner/compare.py              | 4 ++--
 dokimi/tests/test_arms_matrix_cells_input.py | 5 +++--
 dokimi/tests/test_run_compare_errors.py      | 4 ++--
 4 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/dokimi/dokimi/eval/cells.py b/dokimi/dokimi/eval/cells.py
index d953f6e1c..01070ccbd 100644
--- a/dokimi/dokimi/eval/cells.py
+++ b/dokimi/dokimi/eval/cells.py
@@ -154,7 +154,7 @@ def load_matrix_cells(path: str | Path) -> list[MatrixCellDescriptor]:
         raw = json.loads(Path(path).read_text(encoding="utf-8"))
     except OSError as exc:
         raise ValueError(f"cannot read {path}: {exc}") from None
-    except ValueError as exc:  # JSONDecodeError, or bytes that are not UTF-8
+    except (ValueError, RecursionError) as exc:  # JSONDecodeError, bad UTF-8, deep nesting
         raise ValueError(f"{path} is not valid JSON: {exc}") from None
     cells = _extract_cells(raw)
     if not cells:
diff --git a/dokimi/dokimi/runner/compare.py b/dokimi/dokimi/runner/compare.py
index 3fe2cddbc..efb9241c2 100644
--- a/dokimi/dokimi/runner/compare.py
+++ b/dokimi/dokimi/runner/compare.py
@@ -15,12 +15,12 @@ def compare_snapshots(base_path: Path, cand_path: Path, spec: dict | None = None
     """Compare two run snapshots and return summary and metric deltas."""
     try:
         base = json.loads(base_path.read_text(encoding="utf-8"))
-    except (OSError, ValueError) as exc:  # ValueError covers JSONDecodeError and bad UTF-8
+    except (OSError, ValueError, RecursionError) as exc:  # ValueError covers JSONDecodeError and bad UTF-8
         raise ValueError(f"Invalid base snapshot {base_path}: {exc}") from exc
 
     try:
         cand = json.loads(cand_path.read_text(encoding="utf-8"))
-    except (OSError, ValueError) as exc:
+    except (OSError, ValueError, RecursionError) as exc:
         raise ValueError(f"Invalid candidate snapshot {cand_path}: {exc}") from exc
 
     required = {"docs", "score", "coverage", "freshness_lite", "readability"}
diff --git a/dokimi/tests/test_arms_matrix_cells_input.py b/dokimi/tests/test_arms_matrix_cells_input.py
index c621d29a9..6ef1cff25 100644
--- a/dokimi/tests/test_arms_matrix_cells_input.py
+++ b/dokimi/tests/test_arms_matrix_cells_input.py
@@ -37,7 +37,8 @@ def test_a_directory_as_matrix_cells_is_reported_as_unreadable(tmp_path, capsys,
 
 
 @pytest.mark.parametrize("command", ["run", "matrix-run"])
-def test_matrix_cells_that_are_not_json_name_the_file(tmp_path, capsys, command):
+@pytest.mark.parametrize("content", ["{not json", "[" * 100000], ids=["malformed", "too-deep"])
+def test_matrix_cells_that_are_not_json_name_the_file(tmp_path, capsys, command, content):
     bad = tmp_path / "cells.json"
-    bad.write_text("{not json", encoding="utf-8")
+    bad.write_text(content, encoding="utf-8")
     assert f"{bad} is not valid JSON" in _run(tmp_path, bad, capsys, command)
diff --git a/dokimi/tests/test_run_compare_errors.py b/dokimi/tests/test_run_compare_errors.py
index 198142692..cb4a66749 100644
--- a/dokimi/tests/test_run_compare_errors.py
+++ b/dokimi/tests/test_run_compare_errors.py
@@ -44,8 +44,8 @@ def test_run_exits_2_on_a_bad_spec(tmp_path, monkeypatch, capsys, content):
 
 
 @pytest.mark.parametrize("which", ["--base", "--candidate"])
-@pytest.mark.parametrize("content", [None, "{not json", b"\xff\xfe", "[1]", "{}"],
-                         ids=["missing", "malformed", "not-utf8", "not-an-object", "no-summary"])
+@pytest.mark.parametrize("content", [None, "{not json", b"\xff\xfe", "[1]", "{}", "[" * 100000],
+                         ids=["missing", "malformed", "not-utf8", "not-an-object", "no-summary", "too-deep"])
 def test_compare_exits_2_on_a_bad_snapshot(tmp_path, monkeypatch, capsys, which, content):
     bad = tmp_path / "bad.json"
     _write(bad, content)

From 14e1738343dedf18855ff1802ec8bba49f736769 Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Mon, 26 Oct 2026 11:48:27 -0500
Subject: [PATCH 16/49] Split compare_snapshots and match the Error: prefix in
 benchflow tests

compare_snapshots went over the complexity cap, so snapshot loading and
summary validation move into helpers. The tasks CLI now prints Error: for
a failed preflight, so the two benchflow tests expect that prefix.

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/runner/compare.py        | 47 ++++++++++++++------------
 dokimi/tests/test_benchflow_harness.py |  4 +--
 2 files changed, 28 insertions(+), 23 deletions(-)

diff --git a/dokimi/dokimi/runner/compare.py b/dokimi/dokimi/runner/compare.py
index efb9241c2..fd4c8e4e1 100644
--- a/dokimi/dokimi/runner/compare.py
+++ b/dokimi/dokimi/runner/compare.py
@@ -11,32 +11,37 @@
 from dokimi.gate.regression import detect_regressions
 
 
-def compare_snapshots(base_path: Path, cand_path: Path, spec: dict | None = None) -> dict:
-    """Compare two run snapshots and return summary and metric deltas."""
+_REQUIRED_SUMMARY_KEYS = {"docs", "score", "coverage", "freshness_lite", "readability"}
+
+
+def _load_snapshot(path: Path, role: str) -> object:
     try:
-        base = json.loads(base_path.read_text(encoding="utf-8"))
+        return json.loads(path.read_text(encoding="utf-8"))
     except (OSError, ValueError, RecursionError) as exc:  # ValueError covers JSONDecodeError and bad UTF-8
-        raise ValueError(f"Invalid base snapshot {base_path}: {exc}") from exc
+        raise ValueError(f"Invalid {role} snapshot {path}: {exc}") from exc
 
-    try:
-        cand = json.loads(cand_path.read_text(encoding="utf-8"))
-    except (OSError, ValueError, RecursionError) as exc:
-        raise ValueError(f"Invalid candidate snapshot {cand_path}: {exc}") from exc
 
-    required = {"docs", "score", "coverage", "freshness_lite", "readability"}
+def _check_summary(path: Path, payload: object, role: str) -> None:
+    summary = payload.get("summary") if isinstance(payload, dict) else None
+    if not isinstance(summary, dict):
+        raise ValueError(f"Snapshot missing summary object: {path}")
+    missing = _REQUIRED_SUMMARY_KEYS - set(summary.keys())
+    # example_pass_rate is optional (may not be present in old snapshots)
+    if missing:
+        raise ValueError(f"Snapshot {path} missing summary keys: {sorted(missing)}")
+    for key in sorted(_REQUIRED_SUMMARY_KEYS | {"example_pass_rate"}):
+        value = summary.get(key)
+        if key in summary and (isinstance(value, bool) or not isinstance(value, (int, float))):
+            raise ValueError(f"Invalid {role} snapshot {path}: summary `{key}` must be a number, got {value!r}")
+
+
+def compare_snapshots(base_path: Path, cand_path: Path, spec: dict | None = None) -> dict:
+    """Compare two run snapshots and return summary and metric deltas."""
+    base = _load_snapshot(base_path, "base")
+    cand = _load_snapshot(cand_path, "candidate")
+
     for path, payload in ((base_path, base), (cand_path, cand)):
-        summary = payload.get("summary") if isinstance(payload, dict) else None
-        if not isinstance(summary, dict):
-            raise ValueError(f"Snapshot missing summary object: {path}")
-        missing = required - set(summary.keys())
-        # example_pass_rate is optional (may not be present in old snapshots)
-        if missing:
-            raise ValueError(f"Snapshot {path} missing summary keys: {sorted(missing)}")
-        role = "base" if path == base_path else "candidate"
-        for key in sorted(required | {"example_pass_rate"}):
-            value = summary.get(key)
-            if key in summary and (isinstance(value, bool) or not isinstance(value, (int, float))):
-                raise ValueError(f"Invalid {role} snapshot {path}: summary `{key}` must be a number, got {value!r}")
+        _check_summary(path, payload, "base" if path == base_path else "candidate")
 
     diff = {
         "docs": cand["summary"]["docs"] - base["summary"]["docs"],
diff --git a/dokimi/tests/test_benchflow_harness.py b/dokimi/tests/test_benchflow_harness.py
index 9fa778ade..79f93f738 100644
--- a/dokimi/tests/test_benchflow_harness.py
+++ b/dokimi/tests/test_benchflow_harness.py
@@ -1149,7 +1149,7 @@ def failing():
     with pytest.raises(SystemExit) as exc:
         args.func(args)
     assert str(exc.value) == (
-        "error: benchflow preflight: `docker compose version` exited 1; install it"
+        "Error: benchflow preflight: `docker compose version` exited 1; install it"
     )
 
 
@@ -1595,7 +1595,7 @@ def failing():
                                       "benchflow:oracle", "--tasks-root", str(tmp_path / "tasks")])
     with pytest.raises(SystemExit) as exc:
         _run_subject_work(load_subject(subject), args, out_dir=tmp_path / "out")
-    assert str(exc.value).startswith("error: benchflow preflight")
+    assert str(exc.value).startswith("Error: benchflow preflight")
 
 
 def test_a_pack_loaded_by_its_frontmatter_name_is_treatment(tmp_path):

From 8d62e915177751b37445bf7c37671bb59e26595f Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Mon, 26 Oct 2026 10:04:34 -0700
Subject: [PATCH 17/49] fix(repo): repo run exits 1 on a deeply nested batch
 pin

A batch.json nested past the recursion limit raised RecursionError,
which neither _pin_batch nor _errors() caught.

Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/dokimi/commands/repo_run.py | 2 +-
 dokimi/tests/test_repo_run.py      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dokimi/dokimi/commands/repo_run.py b/dokimi/dokimi/commands/repo_run.py
index ff964b231..dd4ef7c0c 100644
--- a/dokimi/dokimi/commands/repo_run.py
+++ b/dokimi/dokimi/commands/repo_run.py
@@ -126,7 +126,7 @@ def _pin_batch(batch_dir: Path, pin: dict[str, str]) -> None:
     if path.is_file():
         try:
             old = json.loads(path.read_text(encoding="utf-8"))
-        except ValueError as exc:  # JSONDecodeError and bad UTF-8
+        except (ValueError, RecursionError) as exc:  # JSONDecodeError, bad UTF-8, deep nesting
             old = exc
         if not isinstance(old, dict):
             raise ValueError(f"batch {batch_dir.name} has an unreadable pin ({path}); use another --batch")
diff --git a/dokimi/tests/test_repo_run.py b/dokimi/tests/test_repo_run.py
index fda202cd5..88f4d0359 100644
--- a/dokimi/tests/test_repo_run.py
+++ b/dokimi/tests/test_repo_run.py
@@ -663,7 +663,7 @@ def test_a_task_json_the_pin_cannot_read_pins_the_graders_and_a_non_object_one_d
     assert ("graders" in json.loads((setup["out"] / "b1/_eval/batch.json").read_text())) is pinned
 
 
-@pytest.mark.parametrize("text", ["{", "[]", '"pin"'])
+@pytest.mark.parametrize("text", ["{", "[]", '"pin"', pytest.param("[" * 100000, id="deep")])
 def test_a_batch_whose_pin_is_unreadable_exits_1_naming_the_file(setup, capsys, text):
     _run(setup, "--arm", "guidance:raw", "--task", "fix_add")
     (setup["out"] / "b1/_eval/batch.json").write_text(text)

From 71587b3aea6c74f400161276ddcae3852a642f4c Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Mon, 26 Oct 2026 12:11:12 -0500
Subject: [PATCH 18/49] fix(repo): repo report and compare exit 1 on a result
 file without a usable agent, task, arm or model

load_cells now checks the fields summarize and compare read directly and
raises CellFileError naming the file, as it does for a missing grade.

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 ...0000_vchamarthi_cli-input-run-arms-repo.md |  2 +-
 dokimi/dokimi/repos/report.py                 | 20 +++++++++++--
 .../tests/test_repo_report_unreadable_cell.py | 29 +++++++++++++++++++
 3 files changed, 47 insertions(+), 4 deletions(-)

diff --git a/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-run-arms-repo.md b/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-run-arms-repo.md
index 2be49e3d3..111323026 100644
--- a/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-run-arms-repo.md
+++ b/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-run-arms-repo.md
@@ -1,3 +1,3 @@
 ### Fixed
 
-- **`run`, `compare`, `arms`, `tasks`, `explore` and `repo`** exit with a message on an unreadable, non-UTF-8 or corrupt input or an unwritable output, instead of a traceback. `run` and `compare` exit 2 on a bad spec or snapshot (a non-numeric summary value, or fewer documents than the manifest's `min_docs`) and exit 1 when an output cannot be written, as `arms` does. `repo report` and `repo compare` exit 1 on a result file without a usable `grade`. `tasks` error messages use the `Error:` prefix.
+- **`run`, `compare`, `arms`, `tasks`, `explore` and `repo`** exit with a message on an unreadable, non-UTF-8 or corrupt input or an unwritable output, instead of a traceback. `run` and `compare` exit 2 on a bad spec or snapshot (a non-numeric summary value, or fewer documents than the manifest's `min_docs`) and exit 1 when an output cannot be written, as `arms` does. `repo report` and `repo compare` exit 1 on a result file without a usable `grade`, `agent`, `task`, `arm` or `model`. `tasks` error messages use the `Error:` prefix.
diff --git a/dokimi/dokimi/repos/report.py b/dokimi/dokimi/repos/report.py
index 346cdab08..79148c6fd 100644
--- a/dokimi/dokimi/repos/report.py
+++ b/dokimi/dokimi/repos/report.py
@@ -60,6 +60,22 @@ def _planned_cells(plan_path: Path) -> set[str]:
     return set(cells)
 
 
+_AGENT_KEYS = ("timed_out", "is_error", "over_budget", "cost_usd", "num_turns")
+
+
+def _check_cell(path: Path, rec: dict[str, Any]) -> None:
+    """Raise a :class:`CellFileError` unless ``rec`` has the fields the report and compare read."""
+    grade = rec.get("grade")
+    if not isinstance(grade, dict) or "status" not in grade:
+        raise CellFileError(f"{path} has no usable `grade`")
+    agent = rec.get("agent")
+    if not isinstance(agent, dict) or any(k not in agent for k in _AGENT_KEYS):
+        raise CellFileError(f"{path} has no usable `agent`")
+    for key in ("task", "arm", "model"):
+        if not isinstance(rec.get(key), str):
+            raise CellFileError(f"{path} has no usable `{key}`")
+
+
 def load_cells(batch_dir: Path) -> tuple[list[dict[str, Any]], list[str]]:
     """Finished cells (their ``result.json``) and the names of missing ones."""
     plan_path = batch_dir / "_eval" / "plan.json"
@@ -68,9 +84,7 @@ def load_cells(batch_dir: Path) -> tuple[list[dict[str, Any]], list[str]]:
     finished = []
     for path in sorted(batch_dir.glob("*/result.json")):
         rec = _read_object(path)
-        grade = rec.get("grade")
-        if not isinstance(grade, dict) or "status" not in grade:
-            raise CellFileError(f"{path} has no usable `grade`")
+        _check_cell(path, rec)
         rec["cell"] = path.parent.name
         # The row's kind is the task's, not what its cells' grades say. The runner records it from the
         # eval dir's task.json; the cell's copy is in the run dir, which the grader could rewrite.
diff --git a/dokimi/tests/test_repo_report_unreadable_cell.py b/dokimi/tests/test_repo_report_unreadable_cell.py
index c0093e3a0..7bec1122b 100644
--- a/dokimi/tests/test_repo_report_unreadable_cell.py
+++ b/dokimi/tests/test_repo_report_unreadable_cell.py
@@ -94,3 +94,32 @@ def test_a_result_file_without_a_usable_grade_exits_1(tmp_path, capsys, argv, co
     err = capsys.readouterr().err
     assert f"{batch / 't__a__m__r1' / 'result.json'} has no usable `grade`" in err
     assert "Traceback" not in err
+
+
+_GOOD = {"grade": {"status": "ok", "score_only": False, "success": True}, "task": "t", "arm": "a", "model": "m",
+         "agent": {"timed_out": False, "is_error": False, "over_budget": False, "cost_usd": 0.1, "num_turns": 2}}
+
+_BAD_FIELDS = [
+    pytest.param("agent", None, id="missing-agent"),
+    pytest.param("agent", [], id="agent-not-a-dict"),
+    pytest.param("agent", {"timed_out": False}, id="agent-missing-keys"),
+    pytest.param("task", None, id="missing-task"),
+    pytest.param("arm", None, id="missing-arm"),
+    pytest.param("model", None, id="missing-model"),
+    pytest.param("task", 3, id="task-not-a-string"),
+]
+
+
+@pytest.mark.parametrize("key, value", _BAD_FIELDS)
+@pytest.mark.parametrize("argv", [("report",), ("compare", "--baseline", "a")], ids=["report", "compare"])
+def test_a_result_file_without_usable_agent_task_arm_or_model_exits_1(tmp_path, capsys, argv, key, value):
+    rec = {k: v for k, v in _GOOD.items() if k != key}
+    if value is not None:
+        rec[key] = value
+    batch = _batch(tmp_path, "b1", {"t__a__m__r1/result.json": json.dumps(rec).encode()})
+    with pytest.raises(SystemExit) as exc:
+        _run(argv[0], str(batch), *argv[1:])
+    assert exc.value.code == 1
+    err = capsys.readouterr().err
+    assert f"{batch / 't__a__m__r1' / 'result.json'} has no usable `{key}`" in err
+    assert "Traceback" not in err

From 6db68b0b547cb86c17e39a5d35cfbe68104e3d72 Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Mon, 26 Oct 2026 12:34:12 -0500
Subject: [PATCH 19/49] Type _load_snapshot as Any and match the lowercase
 error: prefix in the subject preflight test

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/runner/compare.py        | 3 ++-
 dokimi/tests/test_benchflow_harness.py | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/dokimi/dokimi/runner/compare.py b/dokimi/dokimi/runner/compare.py
index fd4c8e4e1..abfa7109c 100644
--- a/dokimi/dokimi/runner/compare.py
+++ b/dokimi/dokimi/runner/compare.py
@@ -7,6 +7,7 @@
 
 import json
 from pathlib import Path
+from typing import Any
 
 from dokimi.gate.regression import detect_regressions
 
@@ -14,7 +15,7 @@
 _REQUIRED_SUMMARY_KEYS = {"docs", "score", "coverage", "freshness_lite", "readability"}
 
 
-def _load_snapshot(path: Path, role: str) -> object:
+def _load_snapshot(path: Path, role: str) -> Any:
     try:
         return json.loads(path.read_text(encoding="utf-8"))
     except (OSError, ValueError, RecursionError) as exc:  # ValueError covers JSONDecodeError and bad UTF-8
diff --git a/dokimi/tests/test_benchflow_harness.py b/dokimi/tests/test_benchflow_harness.py
index 79f93f738..b5ee52f38 100644
--- a/dokimi/tests/test_benchflow_harness.py
+++ b/dokimi/tests/test_benchflow_harness.py
@@ -1595,7 +1595,7 @@ def failing():
                                       "benchflow:oracle", "--tasks-root", str(tmp_path / "tasks")])
     with pytest.raises(SystemExit) as exc:
         _run_subject_work(load_subject(subject), args, out_dir=tmp_path / "out")
-    assert str(exc.value).startswith("Error: benchflow preflight")
+    assert str(exc.value).startswith("error: benchflow preflight")
 
 
 def test_a_pack_loaded_by_its_frontmatter_name_is_treatment(tmp_path):

From 89b9aa73fc008d8142617d62e89efa5fbc8060bf Mon Sep 17 00:00:00 2001
From: Anton Volkov <antonwolfy@gmail.com>
Date: Mon, 26 Oct 2026 17:50:03 +0000
Subject: [PATCH 20/49] Print the benchflow preflight failure with an Error:
 prefix in subjects

Signed-off-by: Anton Volkov <antonwolfy@gmail.com>
---
 dokimi/dokimi/commands/subjects.py     | 2 +-
 dokimi/tests/test_benchflow_harness.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dokimi/dokimi/commands/subjects.py b/dokimi/dokimi/commands/subjects.py
index 3e9dc566f..dfc0e6bb7 100644
--- a/dokimi/dokimi/commands/subjects.py
+++ b/dokimi/dokimi/commands/subjects.py
@@ -400,7 +400,7 @@ def _run_subject_work(descriptor, args: argparse.Namespace, *, out_dir: Path, ta
     try:
         artifact = runner.run()
     except PreflightError as exc:
-        raise SystemExit(f"error: {exc}") from None
+        raise SystemExit(f"Error: {exc}") from None
     save_artifact("task_runs", artifact, task_runs_path)
     task_runs_md = task_runs_path.with_suffix(".md")
     task_runs_md.parent.mkdir(parents=True, exist_ok=True)
diff --git a/dokimi/tests/test_benchflow_harness.py b/dokimi/tests/test_benchflow_harness.py
index b5ee52f38..79f93f738 100644
--- a/dokimi/tests/test_benchflow_harness.py
+++ b/dokimi/tests/test_benchflow_harness.py
@@ -1595,7 +1595,7 @@ def failing():
                                       "benchflow:oracle", "--tasks-root", str(tmp_path / "tasks")])
     with pytest.raises(SystemExit) as exc:
         _run_subject_work(load_subject(subject), args, out_dir=tmp_path / "out")
-    assert str(exc.value).startswith("error: benchflow preflight")
+    assert str(exc.value).startswith("Error: benchflow preflight")
 
 
 def test_a_pack_loaded_by_its_frontmatter_name_is_treatment(tmp_path):

From 48d0d7fffd62163289ca8d9824ea5b24e33444a5 Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Mon, 26 Oct 2026 13:03:43 -0500
Subject: [PATCH 21/49] fix(cli): validate compare spec shape, finite snapshot
 numbers and cell grade/cost fields

compare --spec rejects a non-mapping thresholds or thresholds.regressions and a
deeply nested spec with exit 2. Snapshot summary values must be finite numbers
and docs an integer. repo report requires grade.score_only (unless score_task
is set), grade.success and a numeric or null agent.cost_usd. A deeply nested
explore task file or arms plugin-delta artifact is reported instead of
raising RecursionError. Add tests for the matrix-run delta and rollup writes
and the report loader error branches.

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/commands/arms.py                |  2 +-
 dokimi/dokimi/commands/run.py                 | 28 +++++---
 dokimi/dokimi/explore/tasks.py                |  2 +-
 dokimi/dokimi/repos/report.py                 |  8 ++-
 dokimi/dokimi/runner/compare.py               | 14 +++-
 dokimi/tests/test_arms_output_errors.py       | 64 +++++++++++++++++++
 dokimi/tests/test_explore_task_files.py       |  1 +
 .../tests/test_repo_report_unreadable_cell.py | 49 ++++++++++++++
 dokimi/tests/test_run_compare_errors.py       | 17 +++--
 9 files changed, 166 insertions(+), 19 deletions(-)

diff --git a/dokimi/dokimi/commands/arms.py b/dokimi/dokimi/commands/arms.py
index 749ce2113..ccf910004 100644
--- a/dokimi/dokimi/commands/arms.py
+++ b/dokimi/dokimi/commands/arms.py
@@ -189,7 +189,7 @@ def cmd_arms_plugin_delta(args: argparse.Namespace) -> None:
             for section in ("answers", "evaluations"):
                 records(artifact.get(section) or [], source=f"{path} {section}", kind=section)
         output = compare_plugin_runs(baseline, plugin)
-    except (PluginDeltaError, ValueError, OSError) as exc:
+    except (PluginDeltaError, ValueError, OSError, RecursionError) as exc:
         print(f"Error computing plugin delta: {exc}", file=sys.stderr)
         sys.exit(1)
 
diff --git a/dokimi/dokimi/commands/run.py b/dokimi/dokimi/commands/run.py
index 68d33d299..e3603f6e6 100644
--- a/dokimi/dokimi/commands/run.py
+++ b/dokimi/dokimi/commands/run.py
@@ -89,6 +89,24 @@ def cmd_run(args: argparse.Namespace) -> None:
             sys.exit(1)
 
 
+def _load_compare_spec(spec_path: Path) -> dict | None:
+    """The `compare --spec` mapping; any unreadable or wrong-shape spec is a ``ValueError`` naming the file."""
+    import yaml
+
+    try:
+        spec_data = yaml.safe_load(spec_path.read_text(encoding="utf-8"))
+    except (OSError, ValueError, RecursionError, yaml.YAMLError) as exc:
+        raise ValueError(f"Invalid spec {spec_path}: {exc}") from exc
+    if spec_data is not None and not isinstance(spec_data, dict):
+        raise ValueError(f"Invalid spec {spec_path}: root must be a mapping")
+    thresholds = (spec_data or {}).get("thresholds", {})
+    if not isinstance(thresholds, dict):
+        raise ValueError(f"Invalid spec {spec_path}: `thresholds` must be a mapping")
+    if not isinstance(thresholds.get("regressions", {}), dict):
+        raise ValueError(f"Invalid spec {spec_path}: `thresholds.regressions` must be a mapping")
+    return spec_data
+
+
 def cmd_compare(args: argparse.Namespace) -> None:
     """Compare two snapshots and show regression analysis."""
     base = Path(args.base).resolve()
@@ -97,16 +115,8 @@ def cmd_compare(args: argparse.Namespace) -> None:
     out_json = Path(args.out_json).resolve()
     out_md = Path(args.out_md).resolve()
 
-    spec_data = None
     try:
-        if spec_path:
-            import yaml
-            try:
-                spec_data = yaml.safe_load(spec_path.read_text(encoding="utf-8"))
-            except (OSError, ValueError, yaml.YAMLError) as exc:
-                raise ValueError(f"Invalid spec {spec_path}: {exc}") from exc
-            if spec_data is not None and not isinstance(spec_data, dict):
-                raise ValueError(f"Invalid spec {spec_path}: root must be a mapping")
+        spec_data = _load_compare_spec(spec_path) if spec_path else None
         result = compare_snapshots(base, cand, spec=spec_data)
     except ValueError as exc:
         print(f"Error: {exc}", file=sys.stderr)
diff --git a/dokimi/dokimi/explore/tasks.py b/dokimi/dokimi/explore/tasks.py
index cc9cdaa90..09b98bb6d 100644
--- a/dokimi/dokimi/explore/tasks.py
+++ b/dokimi/dokimi/explore/tasks.py
@@ -113,7 +113,7 @@ def _read_task_yaml(path: Path) -> dict:
         data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
     except OSError as exc:
         raise ValueError(f"Explore task {path} cannot be read: {exc}") from exc
-    except (UnicodeDecodeError, yaml.YAMLError) as exc:
+    except (UnicodeDecodeError, yaml.YAMLError, RecursionError) as exc:
         raise ValueError(f"Explore task {path} is not valid UTF-8 YAML: {exc}") from exc
     if not isinstance(data, dict):
         raise ValueError(f"Explore task {path} must hold a YAML mapping")
diff --git a/dokimi/dokimi/repos/report.py b/dokimi/dokimi/repos/report.py
index 79148c6fd..8de2db4e9 100644
--- a/dokimi/dokimi/repos/report.py
+++ b/dokimi/dokimi/repos/report.py
@@ -66,11 +66,15 @@ def _planned_cells(plan_path: Path) -> set[str]:
 def _check_cell(path: Path, rec: dict[str, Any]) -> None:
     """Raise a :class:`CellFileError` unless ``rec`` has the fields the report and compare read."""
     grade = rec.get("grade")
-    if not isinstance(grade, dict) or "status" not in grade:
+    needed = ("status", "success") if "score_task" in rec else ("status", "score_only", "success")
+    if not isinstance(grade, dict) or any(k not in grade for k in needed):
         raise CellFileError(f"{path} has no usable `grade`")
     agent = rec.get("agent")
     if not isinstance(agent, dict) or any(k not in agent for k in _AGENT_KEYS):
         raise CellFileError(f"{path} has no usable `agent`")
+    cost = agent["cost_usd"]
+    if isinstance(cost, bool) or not (cost is None or isinstance(cost, (int, float))):
+        raise CellFileError(f"{path} has no usable `agent`")
     for key in ("task", "arm", "model"):
         if not isinstance(rec.get(key), str):
             raise CellFileError(f"{path} has no usable `{key}`")
@@ -88,7 +92,7 @@ def load_cells(batch_dir: Path) -> tuple[list[dict[str, Any]], list[str]]:
         rec["cell"] = path.parent.name
         # The row's kind is the task's, not what its cells' grades say. The runner records it from the
         # eval dir's task.json; the cell's copy is in the run dir, which the grader could rewrite.
-        rec["score_task"] = bool(rec.get("score_task", rec["grade"]["score_only"]))
+        rec["score_task"] = bool(rec["score_task"] if "score_task" in rec else rec["grade"]["score_only"])
         finished.append(rec)
     done = {r["cell"] for r in finished}
     unsupported = {p.parent.name for p in batch_dir.glob(f"*/{UNSUPPORTED_FILE}")}
diff --git a/dokimi/dokimi/runner/compare.py b/dokimi/dokimi/runner/compare.py
index abfa7109c..a04cfc181 100644
--- a/dokimi/dokimi/runner/compare.py
+++ b/dokimi/dokimi/runner/compare.py
@@ -6,6 +6,7 @@
 from __future__ import annotations
 
 import json
+import math
 from pathlib import Path
 from typing import Any
 
@@ -32,8 +33,17 @@ def _check_summary(path: Path, payload: object, role: str) -> None:
         raise ValueError(f"Snapshot {path} missing summary keys: {sorted(missing)}")
     for key in sorted(_REQUIRED_SUMMARY_KEYS | {"example_pass_rate"}):
         value = summary.get(key)
-        if key in summary and (isinstance(value, bool) or not isinstance(value, (int, float))):
-            raise ValueError(f"Invalid {role} snapshot {path}: summary `{key}` must be a number, got {value!r}")
+        if key in summary and not _valid_number(key, value):
+            kind = "an integer" if key == "docs" else "a finite number"
+            raise ValueError(f"Invalid {role} snapshot {path}: summary `{key}` must be {kind}, got {value!r}")
+
+
+def _valid_number(key: str, value: object) -> bool:
+    if isinstance(value, bool):
+        return False
+    if key == "docs":
+        return isinstance(value, int)
+    return isinstance(value, (int, float)) and math.isfinite(value)
 
 
 def compare_snapshots(base_path: Path, cand_path: Path, spec: dict | None = None) -> dict:
diff --git a/dokimi/tests/test_arms_output_errors.py b/dokimi/tests/test_arms_output_errors.py
index 0c05038ae..8bce80ebb 100644
--- a/dokimi/tests/test_arms_output_errors.py
+++ b/dokimi/tests/test_arms_output_errors.py
@@ -92,3 +92,67 @@ def test_matrix_run_exits_1_naming_an_uncreatable_out_dir(tmp_path, capsys):
     assert exc.value.code == 1
     err = capsys.readouterr().err
     assert f"Error: cannot write {out_dir / 'cells'}" in err and "Traceback" not in err
+
+
+def test_plugin_delta_exits_1_on_a_deeply_nested_artifact(tmp_path, capsys):
+    deep = tmp_path / "deep.json"
+    deep.write_text("[" * 100000, encoding="utf-8")
+    plugin = tmp_path / "plugin.json"
+    plugin.write_text(json.dumps(_arms_artifact("caveman:full")), encoding="utf-8")
+    args = _args("plugin-delta", "--baseline-json", str(deep), "--plugin-json", str(plugin))
+    with pytest.raises(SystemExit) as exc:
+        args.func(args)
+    assert exc.value.code == 1
+    err = capsys.readouterr().err
+    assert "Error computing plugin delta" in err and "Traceback" not in err
+
+
+def _matrix_run(tmp_path: Path, out_dir: Path, *extra: str):
+    cells = tmp_path / "cells.json"
+    cells.write_text(json.dumps({"cells": [{"name": "c1", "model": "m", "provider": "anthropic", "harness": "arms-runner"}]}), encoding="utf-8")
+    questions = tmp_path / "questions.json"
+    questions.write_text(json.dumps({"questions": [{"id": "q1", "question": "What is x?"}]}), encoding="utf-8")
+    return _args("matrix-run", "--product", "p", "--questions", str(questions), "--arms", "baseline",
+                 "--matrix-cells", str(cells), "--out-dir", str(out_dir), *extra)
+
+
+def _stub_matrix(monkeypatch, tmp_path):
+    from dokimi.commands import arms
+
+    monkeypatch.setattr(arms, "_build_arms_output", lambda *a, **k: {})
+    monkeypatch.setattr(arms, "_save_arms_output", lambda *a, **k: None)
+    delta = tmp_path / "out" / "delta.json"
+    rollup = {"plugin_deltas": [{"artifact": str(delta), "_artifact_payload": {"d": 1}}]}
+    monkeypatch.setattr("dokimi.eval.matrix_rollup.build_matrix_rollup", lambda **k: rollup)
+    monkeypatch.setattr("dokimi.eval.matrix_rollup.strip_internal_payloads", lambda r: {"r": 1})
+    monkeypatch.setattr("dokimi.report.plugin_delta_report.render_plugin_delta_report", lambda p: "delta")
+    monkeypatch.setattr("dokimi.report.matrix_rollup_report.render_matrix_rollup_report", lambda p: "rollup")
+    return delta
+
+
+def _fake_save_artifact(monkeypatch, bad: Path):
+    def save(kind, payload, path, *a, **k):
+        if Path(path) == bad:
+            raise OSError("denied")
+        Path(path).parent.mkdir(parents=True, exist_ok=True)
+        Path(path).write_text("{}", encoding="utf-8")
+
+    monkeypatch.setattr("dokimi.artifacts.save_artifact", save)
+
+
+@pytest.mark.parametrize("which", ["delta_json", "delta_md", "rollup_json", "rollup_md"])
+def test_matrix_run_exits_1_naming_an_unwritable_delta_or_rollup(tmp_path, monkeypatch, capsys, which):
+    delta = _stub_matrix(monkeypatch, tmp_path)
+    out_dir = tmp_path / "out"
+    rollup_json = out_dir / "matrix-rollup.json"
+    bad = {"delta_json": delta, "delta_md": delta.with_suffix(".md"),
+           "rollup_json": rollup_json, "rollup_md": rollup_json.with_suffix(".md")}[which]
+    _fake_save_artifact(monkeypatch, bad)
+    if which.endswith("_md"):
+        bad.mkdir(parents=True)  # a directory where the report file belongs
+    args = _matrix_run(tmp_path, out_dir)
+    with pytest.raises(SystemExit) as exc:
+        args.func(args)
+    assert exc.value.code == 1
+    err = capsys.readouterr().err
+    assert f"Error: cannot write {bad}" in err and "Traceback" not in err
diff --git a/dokimi/tests/test_explore_task_files.py b/dokimi/tests/test_explore_task_files.py
index e8cc4b3f0..c298420c2 100644
--- a/dokimi/tests/test_explore_task_files.py
+++ b/dokimi/tests/test_explore_task_files.py
@@ -25,6 +25,7 @@ def _exit_message(argv) -> str:
     [
         (b"id: [unclosed\n", "is not valid UTF-8 YAML"),
         (bytes([0xFF, 0xFE, 0x00]), "is not valid UTF-8 YAML"),
+        pytest.param(b"[" * 100000, "is not valid UTF-8 YAML", id="too-deep"),
         (b"- just\n- a list\n", "must hold a YAML mapping"),
         (b"id: only-an-id\n", "missing required field(s)"),
         (b"id: x\nquery: q\nreferences: 5\n", "references must be a list"),
diff --git a/dokimi/tests/test_repo_report_unreadable_cell.py b/dokimi/tests/test_repo_report_unreadable_cell.py
index 7bec1122b..d3ce67a39 100644
--- a/dokimi/tests/test_repo_report_unreadable_cell.py
+++ b/dokimi/tests/test_repo_report_unreadable_cell.py
@@ -107,6 +107,10 @@ def test_a_result_file_without_a_usable_grade_exits_1(tmp_path, capsys, argv, co
     pytest.param("arm", None, id="missing-arm"),
     pytest.param("model", None, id="missing-model"),
     pytest.param("task", 3, id="task-not-a-string"),
+    pytest.param("grade", {"status": "ok", "success": True}, id="grade-without-score-only"),
+    pytest.param("grade", {"status": "ok", "score_only": False}, id="grade-without-success"),
+    pytest.param("agent", {**_GOOD["agent"], "cost_usd": "0.1"}, id="cost-a-string"),
+    pytest.param("agent", {**_GOOD["agent"], "cost_usd": True}, id="cost-a-bool"),
 ]
 
 
@@ -123,3 +127,48 @@ def test_a_result_file_without_usable_agent_task_arm_or_model_exits_1(tmp_path,
     err = capsys.readouterr().err
     assert f"{batch / 't__a__m__r1' / 'result.json'} has no usable `{key}`" in err
     assert "Traceback" not in err
+
+
+def test_a_result_file_with_a_score_task_and_no_score_only_is_accepted(tmp_path, capsys):
+    rec = {**_GOOD, "score_task": False, "grade": {"status": "ok", "success": True},
+           "agent": {**_GOOD["agent"], "cost_usd": None}}
+    batch = _batch(tmp_path, "b1", {"t__a__m__r1/result.json": json.dumps(rec).encode()})
+    _run("report", str(batch), "--json")
+    assert json.loads(capsys.readouterr().out)["rows"][0]["cost_unknown"] == 1
+
+
+def test_a_deeply_nested_result_file_exits_1_naming_the_file(tmp_path, capsys):
+    batch = _batch(tmp_path, "b1", {"t__a__m__r1/result.json": b"[" * 100000})
+    with pytest.raises(SystemExit) as exc:
+        _run("report", str(batch))
+    assert exc.value.code == 1
+    err = capsys.readouterr().err
+    assert str(batch / "t__a__m__r1" / "result.json") in err and "Traceback" not in err
+
+
+def test_an_unreadable_result_file_exits_1_naming_the_file(tmp_path, capsys):
+    batch = _batch(tmp_path, "b1", {})
+    (batch / "t__a__m__r1" / "result.json").mkdir(parents=True)  # a directory where the file belongs
+    with pytest.raises(SystemExit) as exc:
+        _run("report", str(batch))
+    assert exc.value.code == 1
+    err = capsys.readouterr().err
+    assert f"cannot read {batch / 't__a__m__r1' / 'result.json'}" in err and "Traceback" not in err
+
+
+def test_a_result_file_without_num_turns_exits_1(tmp_path, capsys):
+    agent = {k: v for k, v in _GOOD["agent"].items() if k != "num_turns"}
+    batch = _batch(tmp_path, "b1", {"t__a__m__r1/result.json": json.dumps({**_GOOD, "agent": agent}).encode()})
+    with pytest.raises(SystemExit) as exc:
+        _run("report", str(batch))
+    assert exc.value.code == 1
+    assert "has no usable `agent`" in capsys.readouterr().err
+
+
+def test_a_score_task_result_file_without_grade_success_exits_1(tmp_path, capsys):
+    rec = {**_GOOD, "score_task": True, "grade": {"status": "ok"}}
+    batch = _batch(tmp_path, "b1", {"t__a__m__r1/result.json": json.dumps(rec).encode()})
+    with pytest.raises(SystemExit) as exc:
+        _run("report", str(batch))
+    assert exc.value.code == 1
+    assert "has no usable `grade`" in capsys.readouterr().err
diff --git a/dokimi/tests/test_run_compare_errors.py b/dokimi/tests/test_run_compare_errors.py
index cb4a66749..1bbb8b2b3 100644
--- a/dokimi/tests/test_run_compare_errors.py
+++ b/dokimi/tests/test_run_compare_errors.py
@@ -59,7 +59,12 @@ def test_compare_exits_2_on_a_bad_snapshot(tmp_path, monkeypatch, capsys, which,
     assert not (tmp_path / "c.json").exists()
 
 
-@pytest.mark.parametrize("content", [None, "a: [", "- a\n"], ids=["missing", "malformed", "not-a-mapping"])
+@pytest.mark.parametrize(
+    "content",
+    [None, "a: [", "- a\n", "[" * 100000, "thresholds:\n", "thresholds: []\n", "thresholds: {regressions: 3}\n",
+     "thresholds: {regressions: }\n"],
+    ids=["missing", "malformed", "not-a-mapping", "too-deep", "thresholds-empty", "thresholds-list",
+         "regressions-scalar", "regressions-empty"])
 def test_compare_exits_2_on_a_bad_spec(tmp_path, monkeypatch, capsys, content):
     spec = tmp_path / "spec.yaml"
     _write(spec, content)
@@ -68,7 +73,7 @@ def test_compare_exits_2_on_a_bad_spec(tmp_path, monkeypatch, capsys, content):
                  "--out-json", str(tmp_path / "c.json"), "--out-md", str(tmp_path / "c.md"))
     assert code == 2
     err = capsys.readouterr().err
-    assert err.startswith("Error: ") and str(spec) in err
+    assert err.startswith("Error: ") and str(spec) in err and "Traceback" not in err
 
 
 _SPEC_MIN_DOCS = (
@@ -91,7 +96,10 @@ def test_run_exits_2_when_the_root_has_fewer_docs_than_min_docs(tmp_path, monkey
 
 
 @pytest.mark.parametrize("which", ["--base", "--candidate"])
-@pytest.mark.parametrize("field,value", [("docs", "1"), ("score", None), ("coverage", True)])
+@pytest.mark.parametrize("field,value", [
+    ("docs", "1"), ("score", None), ("coverage", True), ("score", float("nan")), ("score", float("inf")),
+    ("readability", 1e999), ("docs", 1.5), ("example_pass_rate", float("nan")), ("example_pass_rate", "x"),
+])
 def test_compare_exits_2_on_a_non_numeric_summary_value(tmp_path, monkeypatch, capsys, which, field, value):
     bad = tmp_path / "bad.json"
     bad.write_text(json.dumps({"summary": {**SNAPSHOT["summary"], field: value}}), encoding="utf-8")
@@ -101,7 +109,8 @@ def test_compare_exits_2_on_a_non_numeric_summary_value(tmp_path, monkeypatch, c
                  "--out-json", str(tmp_path / "c.json"), "--out-md", str(tmp_path / "c.md"))
     assert code == 2
     err = capsys.readouterr().err
-    assert err.startswith("Error: ") and str(bad) in err and f"`{field}`" in err
+    role = which[2:]
+    assert err.startswith("Error: ") and f"Invalid {role} snapshot {bad}" in err and f"`{field}`" in err
 
 
 def _blocked(tmp_path):

From 2aff0e3aad9f321c9fa697ca6421ddf99a808830 Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Mon, 26 Oct 2026 14:18:22 -0500
Subject: [PATCH 22/49] fix(cli): reject non-finite cell costs, missing score
 and bad score_task, huge ints and bad regression thresholds

repo report and compare require grade.score for a score task, a boolean
score_task and a finite cost_usd. compare and the snapshot check treat an
int too large for a float as not finite. compare --spec requires every
thresholds.regressions value to be a finite non-negative number. The
changelog names the readers that still raise.

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 ...0000_vchamarthi_cli-input-run-arms-repo.md |  2 +-
 dokimi/dokimi/commands/run.py                 | 18 +++++-
 dokimi/dokimi/repos/report.py                 | 19 ++++++-
 dokimi/dokimi/runner/compare.py               | 10 +++-
 .../tests/test_repo_report_unreadable_cell.py | 57 +++++++++++++++++--
 dokimi/tests/test_run_compare_errors.py       | 10 +++-
 6 files changed, 103 insertions(+), 13 deletions(-)

diff --git a/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-run-arms-repo.md b/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-run-arms-repo.md
index 111323026..c371cbfb1 100644
--- a/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-run-arms-repo.md
+++ b/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-run-arms-repo.md
@@ -1,3 +1,3 @@
 ### Fixed
 
-- **`run`, `compare`, `arms`, `tasks`, `explore` and `repo`** exit with a message on an unreadable, non-UTF-8 or corrupt input or an unwritable output, instead of a traceback. `run` and `compare` exit 2 on a bad spec or snapshot (a non-numeric summary value, or fewer documents than the manifest's `min_docs`) and exit 1 when an output cannot be written, as `arms` does. `repo report` and `repo compare` exit 1 on a result file without a usable `grade`, `agent`, `task`, `arm` or `model`. `tasks` error messages use the `Error:` prefix.
+- **`run`, `compare`, `arms`, `tasks`, `explore` and `repo`** exit with a message on the unreadable, non-UTF-8 or corrupt spec, snapshot, result, plan, task and question files their tests cover, or an unwritable output, instead of a traceback. Some other readers still raise, for example `repo scan --manifest-file`, `repo scan --static`, `repo reach` and `repo budget --harness`, `arms run --questions` and `tasks run --tasks`. `run` and `compare` exit 2 on a bad spec or snapshot (a non-numeric summary value, or fewer documents than the manifest's `min_docs`) and exit 1 when an output cannot be written, as `arms` does. `repo report` and `repo compare` exit 1 on a result file without a usable `grade`, `agent`, `task`, `arm` or `model`. `tasks` error messages use the `Error:` prefix.
diff --git a/dokimi/dokimi/commands/run.py b/dokimi/dokimi/commands/run.py
index e3603f6e6..097bb52f8 100644
--- a/dokimi/dokimi/commands/run.py
+++ b/dokimi/dokimi/commands/run.py
@@ -7,6 +7,7 @@
 
 import argparse
 import json
+import math
 import sys
 from pathlib import Path
 
@@ -102,11 +103,26 @@ def _load_compare_spec(spec_path: Path) -> dict | None:
     thresholds = (spec_data or {}).get("thresholds", {})
     if not isinstance(thresholds, dict):
         raise ValueError(f"Invalid spec {spec_path}: `thresholds` must be a mapping")
-    if not isinstance(thresholds.get("regressions", {}), dict):
+    regressions = thresholds.get("regressions", {})
+    if not isinstance(regressions, dict):
         raise ValueError(f"Invalid spec {spec_path}: `thresholds.regressions` must be a mapping")
+    for name, value in regressions.items():
+        if not _non_negative_number(value):
+            raise ValueError(
+                f"Invalid spec {spec_path}: `thresholds.regressions.{name}` must be a finite non-negative number, got {value!r}"
+            )
     return spec_data
 
 
+def _non_negative_number(value: object) -> bool:
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        return False
+    try:
+        return math.isfinite(value) and value >= 0
+    except OverflowError:
+        return False
+
+
 def cmd_compare(args: argparse.Namespace) -> None:
     """Compare two snapshots and show regression analysis."""
     base = Path(args.base).resolve()
diff --git a/dokimi/dokimi/repos/report.py b/dokimi/dokimi/repos/report.py
index 8de2db4e9..3e8c36db3 100644
--- a/dokimi/dokimi/repos/report.py
+++ b/dokimi/dokimi/repos/report.py
@@ -63,18 +63,31 @@ def _planned_cells(plan_path: Path) -> set[str]:
 _AGENT_KEYS = ("timed_out", "is_error", "over_budget", "cost_usd", "num_turns")
 
 
+def _valid_cost(cost: object) -> bool:
+    """True for a missing cost or a finite, non-bool number."""
+    if cost is None:
+        return True
+    if isinstance(cost, bool) or not isinstance(cost, (int, float)):
+        return False
+    try:
+        return math.isfinite(float(cost))
+    except OverflowError:
+        return False
+
+
 def _check_cell(path: Path, rec: dict[str, Any]) -> None:
     """Raise a :class:`CellFileError` unless ``rec`` has the fields the report and compare read."""
     grade = rec.get("grade")
-    needed = ("status", "success") if "score_task" in rec else ("status", "score_only", "success")
+    needed = ("status", "score", "success") if "score_task" in rec else ("status", "score_only", "success")
     if not isinstance(grade, dict) or any(k not in grade for k in needed):
         raise CellFileError(f"{path} has no usable `grade`")
     agent = rec.get("agent")
     if not isinstance(agent, dict) or any(k not in agent for k in _AGENT_KEYS):
         raise CellFileError(f"{path} has no usable `agent`")
-    cost = agent["cost_usd"]
-    if isinstance(cost, bool) or not (cost is None or isinstance(cost, (int, float))):
+    if not _valid_cost(agent["cost_usd"]):
         raise CellFileError(f"{path} has no usable `agent`")
+    if "score_task" in rec and not isinstance(rec["score_task"], bool):
+        raise CellFileError(f"{path} has no usable `score_task`")
     for key in ("task", "arm", "model"):
         if not isinstance(rec.get(key), str):
             raise CellFileError(f"{path} has no usable `{key}`")
diff --git a/dokimi/dokimi/runner/compare.py b/dokimi/dokimi/runner/compare.py
index a04cfc181..fba63257e 100644
--- a/dokimi/dokimi/runner/compare.py
+++ b/dokimi/dokimi/runner/compare.py
@@ -43,7 +43,15 @@ def _valid_number(key: str, value: object) -> bool:
         return False
     if key == "docs":
         return isinstance(value, int)
-    return isinstance(value, (int, float)) and math.isfinite(value)
+    return isinstance(value, (int, float)) and _is_finite(value)
+
+
+def _is_finite(value: int | float) -> bool:
+    """True for a finite number; an int too large for a float is not finite."""
+    try:
+        return math.isfinite(value)
+    except OverflowError:
+        return False
 
 
 def compare_snapshots(base_path: Path, cand_path: Path, spec: dict | None = None) -> dict:
diff --git a/dokimi/tests/test_repo_report_unreadable_cell.py b/dokimi/tests/test_repo_report_unreadable_cell.py
index d3ce67a39..b138e2a63 100644
--- a/dokimi/tests/test_repo_report_unreadable_cell.py
+++ b/dokimi/tests/test_repo_report_unreadable_cell.py
@@ -10,6 +10,7 @@
 import pytest
 
 from dokimi.cli import build_parser
+from dokimi.repos.report import _AGENT_KEYS
 
 
 def _batch(tmp_path, name: str, files: dict[str, bytes]):
@@ -130,7 +131,7 @@ def test_a_result_file_without_usable_agent_task_arm_or_model_exits_1(tmp_path,
 
 
 def test_a_result_file_with_a_score_task_and_no_score_only_is_accepted(tmp_path, capsys):
-    rec = {**_GOOD, "score_task": False, "grade": {"status": "ok", "success": True},
+    rec = {**_GOOD, "score_task": False, "grade": {"status": "ok", "score": 1.0, "success": True},
            "agent": {**_GOOD["agent"], "cost_usd": None}}
     batch = _batch(tmp_path, "b1", {"t__a__m__r1/result.json": json.dumps(rec).encode()})
     _run("report", str(batch), "--json")
@@ -156,8 +157,9 @@ def test_an_unreadable_result_file_exits_1_naming_the_file(tmp_path, capsys):
     assert f"cannot read {batch / 't__a__m__r1' / 'result.json'}" in err and "Traceback" not in err
 
 
-def test_a_result_file_without_num_turns_exits_1(tmp_path, capsys):
-    agent = {k: v for k, v in _GOOD["agent"].items() if k != "num_turns"}
+@pytest.mark.parametrize("key", _AGENT_KEYS)
+def test_a_result_file_missing_an_agent_key_exits_1(tmp_path, capsys, key):
+    agent = {k: v for k, v in _GOOD["agent"].items() if k != key}
     batch = _batch(tmp_path, "b1", {"t__a__m__r1/result.json": json.dumps({**_GOOD, "agent": agent}).encode()})
     with pytest.raises(SystemExit) as exc:
         _run("report", str(batch))
@@ -165,8 +167,55 @@ def test_a_result_file_without_num_turns_exits_1(tmp_path, capsys):
     assert "has no usable `agent`" in capsys.readouterr().err
 
 
+def test_a_result_file_without_grade_status_exits_1(tmp_path, capsys):
+    grade = {k: v for k, v in _GOOD["grade"].items() if k != "status"}
+    batch = _batch(tmp_path, "b1", {"t__a__m__r1/result.json": json.dumps({**_GOOD, "grade": grade}).encode()})
+    with pytest.raises(SystemExit) as exc:
+        _run("report", str(batch))
+    assert exc.value.code == 1
+    assert "has no usable `grade`" in capsys.readouterr().err
+
+
+@pytest.mark.parametrize("cost", ["NaN", "Infinity", "-Infinity", "1e999", "9" * 400])
+@pytest.mark.parametrize("cmd", [("report",), ("compare", "--baseline", "a")], ids=["report", "compare"])
+def test_a_result_file_with_a_non_finite_cost_exits_1(tmp_path, capsys, cmd, cost):
+    text = json.dumps(_GOOD).replace("0.1", cost)
+    batch = _batch(tmp_path, "b1", {"t__a__m__r1/result.json": text.encode()})
+    with pytest.raises(SystemExit) as exc:
+        _run(cmd[0], str(batch), *cmd[1:])
+    assert exc.value.code == 1
+    err = capsys.readouterr().err
+    assert "has no usable `agent`" in err and "Traceback" not in err
+
+
+def test_a_score_task_result_file_without_grade_score_exits_1(tmp_path, capsys):
+    rec = {**_GOOD, "score_task": True, "grade": {"status": "ok", "success": True}}
+    batch = _batch(tmp_path, "b1", {"t__a__m__r1/result.json": json.dumps(rec).encode()})
+    with pytest.raises(SystemExit) as exc:
+        _run("report", str(batch))
+    assert exc.value.code == 1
+    assert "has no usable `grade`" in capsys.readouterr().err
+
+
+def test_a_score_task_result_file_with_a_null_grade_score_is_accepted(tmp_path, capsys):
+    rec = {**_GOOD, "score_task": True, "grade": {"status": "grader_failure", "score": None, "success": False}}
+    batch = _batch(tmp_path, "b1", {"t__a__m__r1/result.json": json.dumps(rec).encode()})
+    _run("report", str(batch), "--json")
+    assert json.loads(capsys.readouterr().out)["rows"]
+
+
+@pytest.mark.parametrize("value", ["false", 0, None, []])
+def test_a_result_file_with_a_non_bool_score_task_exits_1(tmp_path, capsys, value):
+    rec = {**_GOOD, "score_task": value, "grade": {"status": "ok", "score": 1.0, "success": True}}
+    batch = _batch(tmp_path, "b1", {"t__a__m__r1/result.json": json.dumps(rec).encode()})
+    with pytest.raises(SystemExit) as exc:
+        _run("report", str(batch))
+    assert exc.value.code == 1
+    assert "has no usable `score_task`" in capsys.readouterr().err
+
+
 def test_a_score_task_result_file_without_grade_success_exits_1(tmp_path, capsys):
-    rec = {**_GOOD, "score_task": True, "grade": {"status": "ok"}}
+    rec = {**_GOOD, "score_task": True, "grade": {"status": "ok", "score": 1.0}}
     batch = _batch(tmp_path, "b1", {"t__a__m__r1/result.json": json.dumps(rec).encode()})
     with pytest.raises(SystemExit) as exc:
         _run("report", str(batch))
diff --git a/dokimi/tests/test_run_compare_errors.py b/dokimi/tests/test_run_compare_errors.py
index 1bbb8b2b3..3d610f9b6 100644
--- a/dokimi/tests/test_run_compare_errors.py
+++ b/dokimi/tests/test_run_compare_errors.py
@@ -62,9 +62,13 @@ def test_compare_exits_2_on_a_bad_snapshot(tmp_path, monkeypatch, capsys, which,
 @pytest.mark.parametrize(
     "content",
     [None, "a: [", "- a\n", "[" * 100000, "thresholds:\n", "thresholds: []\n", "thresholds: {regressions: 3}\n",
-     "thresholds: {regressions: }\n"],
+     "thresholds: {regressions: }\n", "thresholds: {regressions: {score: null}}\n",
+     "thresholds: {regressions: {score: [1]}}\n", "thresholds: {regressions: {score: .nan}}\n",
+     "thresholds: {regressions: {score: .inf}}\n", "thresholds: {regressions: {score: -1}}\n",
+     "thresholds: {regressions: {score: true}}\n", "thresholds: {regressions: {score: " + "9" * 400 + "}}\n"],
     ids=["missing", "malformed", "not-a-mapping", "too-deep", "thresholds-empty", "thresholds-list",
-         "regressions-scalar", "regressions-empty"])
+         "regressions-scalar", "regressions-empty", "value-null", "value-list", "value-nan",
+         "value-inf", "value-negative", "value-bool", "value-huge-int"])
 def test_compare_exits_2_on_a_bad_spec(tmp_path, monkeypatch, capsys, content):
     spec = tmp_path / "spec.yaml"
     _write(spec, content)
@@ -98,7 +102,7 @@ def test_run_exits_2_when_the_root_has_fewer_docs_than_min_docs(tmp_path, monkey
 @pytest.mark.parametrize("which", ["--base", "--candidate"])
 @pytest.mark.parametrize("field,value", [
     ("docs", "1"), ("score", None), ("coverage", True), ("score", float("nan")), ("score", float("inf")),
-    ("readability", 1e999), ("docs", 1.5), ("example_pass_rate", float("nan")), ("example_pass_rate", "x"),
+    ("readability", 1e999), ("docs", 1.5), ("score", 10**400), ("example_pass_rate", 10**400), ("example_pass_rate", float("nan")), ("example_pass_rate", "x"),
 ])
 def test_compare_exits_2_on_a_non_numeric_summary_value(tmp_path, monkeypatch, capsys, which, field, value):
     bad = tmp_path / "bad.json"

From 62a4d94a69335cfaf65fbe1a84404bdd61054255 Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Mon, 26 Oct 2026 15:01:13 -0500
Subject: [PATCH 23/49] fix(cli): reject malformed cell fields and check
 regression threshold keys

repo report and repo compare exit 1, naming the file, when a cell has a non-numeric, bool or overflowing grade.score, score_max, agent.num_turns or token count, or a skills block without the keys the summary reads. A null stays valid where a run leaves a value unknown.

A non-finite cost_usd now rejects its cell instead of being skipped, so the test covering it expects the exit 1 message.

The compare --spec threshold tests use the keys the regression gate reads, and cover accepted zero and integer values.

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/repos/report.py                 | 31 +++++++++++++++----
 dokimi/tests/test_repo_compare.py             | 21 +++++++++----
 .../tests/test_repo_report_unreadable_cell.py |  7 +++++
 dokimi/tests/test_run_compare_errors.py       | 26 +++++++++++++---
 4 files changed, 68 insertions(+), 17 deletions(-)

diff --git a/dokimi/dokimi/repos/report.py b/dokimi/dokimi/repos/report.py
index 3d7287e34..733cde557 100644
--- a/dokimi/dokimi/repos/report.py
+++ b/dokimi/dokimi/repos/report.py
@@ -71,31 +71,50 @@ def _planned_cells(plan_path: Path) -> set[str]:
 
 
 _AGENT_KEYS = ("timed_out", "is_error", "over_budget", "cost_usd", "num_turns")
+_AGENT_NUMBERS = ("cost_usd", "num_turns", *_TOKENS)
 
 
-def _valid_cost(cost: object) -> bool:
-    """True for a missing cost or a finite, non-bool number."""
-    if cost is None:
+def _optional_number(value: object) -> bool:
+    """True for None or a finite, non-bool number; an int too big for a float is not."""
+    if value is None:
         return True
-    if isinstance(cost, bool) or not isinstance(cost, (int, float)):
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
         return False
     try:
-        return math.isfinite(float(cost))
+        return math.isfinite(float(value))
     except OverflowError:
         return False
 
 
+def _skills_usable(skills: object) -> bool:
+    """True when ``skills`` is absent, or a mapping holding the keys :func:`summarize` reads."""
+    if skills is None:
+        return True
+    if not isinstance(skills, dict):
+        return False
+    if not skills or not skills.get("measured", True):
+        return True
+    needed = ["skill_invoked"] + (["skill_selected_correct"] if skills.get("expected_skill") else [])
+    return all(k in skills for k in needed)
+
+
 def _check_cell(path: Path, rec: dict[str, Any]) -> None:
     """Raise a :class:`CellFileError` unless ``rec`` has the fields the report and compare read."""
     grade = rec.get("grade")
     needed = ("status", "score", "success") if "score_task" in rec else ("status", "score_only", "success")
     if not isinstance(grade, dict) or any(k not in grade for k in needed):
         raise CellFileError(f"{path} has no usable `grade`")
+    if not _optional_number(grade.get("score")):
+        raise CellFileError(f"{path} has no usable `grade`")
     agent = rec.get("agent")
     if not isinstance(agent, dict) or any(k not in agent for k in _AGENT_KEYS):
         raise CellFileError(f"{path} has no usable `agent`")
-    if not _valid_cost(agent["cost_usd"]):
+    if not all(_optional_number(agent.get(k)) for k in _AGENT_NUMBERS):
         raise CellFileError(f"{path} has no usable `agent`")
+    if not _optional_number(rec.get("score_max")):
+        raise CellFileError(f"{path} has no usable `score_max`")
+    if not _skills_usable(rec.get("skills")):
+        raise CellFileError(f"{path} has no usable `skills`")
     if "score_task" in rec and not isinstance(rec["score_task"], bool):
         raise CellFileError(f"{path} has no usable `score_task`")
     for key in ("task", "arm", "model"):
diff --git a/dokimi/tests/test_repo_compare.py b/dokimi/tests/test_repo_compare.py
index 9a46eba5d..f1fe6298e 100644
--- a/dokimi/tests/test_repo_compare.py
+++ b/dokimi/tests/test_repo_compare.py
@@ -234,7 +234,7 @@ def test_score_headroom_is_measured_against_score_max(tmp_path):
     assert _row(compare([e], "guidance:none"), "score")["headroom"] is None
 
 
-@pytest.mark.parametrize("turns,expected", [(None, None), (1, None), (2, "floor"), (5, "floor"), ("1", None)])
+@pytest.mark.parametrize("turns,expected", [(None, None), (1, None), (2, "floor"), (5, "floor")])
 def test_an_agent_outage_is_unmeasured_not_a_floor(tmp_path, turns, expected):
     b = tmp_path / "b1"
     for i in range(3):
@@ -736,19 +736,28 @@ def test_an_empty_arm_list_is_refused_not_an_empty_comparison(tmp_path, capsys,
 
 
 @pytest.mark.parametrize("bad", [float("inf"), float("-inf"), float("nan")])
-def test_non_finite_values_are_not_measurements(tmp_path, bad):
+def test_non_finite_metrics_are_not_measurements(tmp_path, bad):
     b = tmp_path / "b1"
     for i in range(4):
         _cell(b, f"fix_{i}", "guidance:none", 1, cost=0.1, metrics={"m": bad})
-        _cell(b, f"fix_{i}", "guidance:raw", 1, cost=bad if i == 0 else 0.2, metrics={"m": 1})
+        _cell(b, f"fix_{i}", "guidance:raw", 1, cost=0.2, metrics={"m": 1})
     s = compare([b], "guidance:none", metrics=("m",))
     assert not [r for r in s["comparisons"] if r["measure"] == "metric:m"]  # no task has a finite value in both arms
-    cost = _row(s, "cost_usd")
-    assert [t["task"] for t in cost["tasks"]] == ["fix_1", "fix_2", "fix_3"]  # fix_0's cost is not a number
-    assert cost["up"] + cost["down"] + cost["tied"] == cost["n_tasks"] == 3
     json.dumps(s, allow_nan=False)
 
 
+@pytest.mark.parametrize("bad", [float("inf"), float("-inf"), float("nan")])
+def test_a_non_finite_cost_rejects_the_cell_with_a_message(tmp_path, capsys, bad):
+    b = tmp_path / "b1"
+    _cell(b, "fix_0", "guidance:none", 1, cost=bad)
+    _cell(b, "fix_0", "guidance:raw", 1)
+    args = build_parser().parse_args(["repo", "compare", str(b), "--baseline", "guidance:none"])
+    with pytest.raises(SystemExit) as exc:
+        args.func(args)
+    err = capsys.readouterr().err
+    assert exc.value.code == 1 and "has no usable `agent`" in err and "Traceback" not in err
+
+
 def test_a_batch_given_as_dot_keeps_its_directory_name(tmp_path, monkeypatch):
     b = tmp_path / "b1"
     _cell(b, "fix_a", "guidance:none", 1, success=False)
diff --git a/dokimi/tests/test_repo_report_unreadable_cell.py b/dokimi/tests/test_repo_report_unreadable_cell.py
index b138e2a63..3bd2bb11b 100644
--- a/dokimi/tests/test_repo_report_unreadable_cell.py
+++ b/dokimi/tests/test_repo_report_unreadable_cell.py
@@ -112,6 +112,13 @@ def test_a_result_file_without_a_usable_grade_exits_1(tmp_path, capsys, argv, co
     pytest.param("grade", {"status": "ok", "score_only": False}, id="grade-without-success"),
     pytest.param("agent", {**_GOOD["agent"], "cost_usd": "0.1"}, id="cost-a-string"),
     pytest.param("agent", {**_GOOD["agent"], "cost_usd": True}, id="cost-a-bool"),
+    pytest.param("grade", {**_GOOD["grade"], "score": True}, id="grade-score-a-bool"),
+    pytest.param("grade", {**_GOOD["grade"], "score": 10**400}, id="grade-score-overflows-a-float"),
+    pytest.param("score_max", True, id="score-max-a-bool"),
+    pytest.param("agent", {**_GOOD["agent"], "num_turns": 10**400}, id="num-turns-overflows"),
+    pytest.param("agent", {**_GOOD["agent"], "input_tokens": True}, id="input-tokens-a-bool"),
+    pytest.param("skills", [], id="skills-not-a-mapping"),
+    pytest.param("skills", {"measured": True, "expected_skill": "s"}, id="skills-no-skill-invoked"),
 ]
 
 
diff --git a/dokimi/tests/test_run_compare_errors.py b/dokimi/tests/test_run_compare_errors.py
index 3d610f9b6..d258783fc 100644
--- a/dokimi/tests/test_run_compare_errors.py
+++ b/dokimi/tests/test_run_compare_errors.py
@@ -62,13 +62,17 @@ def test_compare_exits_2_on_a_bad_snapshot(tmp_path, monkeypatch, capsys, which,
 @pytest.mark.parametrize(
     "content",
     [None, "a: [", "- a\n", "[" * 100000, "thresholds:\n", "thresholds: []\n", "thresholds: {regressions: 3}\n",
-     "thresholds: {regressions: }\n", "thresholds: {regressions: {score: null}}\n",
-     "thresholds: {regressions: {score: [1]}}\n", "thresholds: {regressions: {score: .nan}}\n",
-     "thresholds: {regressions: {score: .inf}}\n", "thresholds: {regressions: {score: -1}}\n",
-     "thresholds: {regressions: {score: true}}\n", "thresholds: {regressions: {score: " + "9" * 400 + "}}\n"],
+     "thresholds: {regressions: }\n", "thresholds: {regressions: {score_drop_warn: null}}\n",
+     "thresholds: {regressions: {score_drop_warn: [1]}}\n",
+     "thresholds: {regressions: {score_drop_warn: .nan}}\n",
+     "thresholds: {regressions: {score_drop_warn: .inf}}\n",
+     "thresholds: {regressions: {score_drop_warn: -1}}\n",
+     "thresholds: {regressions: {score_drop_warn: true}}\n",
+     "thresholds: {regressions: {score_drop_warn: " + "9" * 400 + "}}\n",
+     "thresholds: {regressions: {score_drop_warn: 0.1, score_drop_critical: .nan}}\n"],
     ids=["missing", "malformed", "not-a-mapping", "too-deep", "thresholds-empty", "thresholds-list",
          "regressions-scalar", "regressions-empty", "value-null", "value-list", "value-nan",
-         "value-inf", "value-negative", "value-bool", "value-huge-int"])
+         "value-inf", "value-negative", "value-bool", "value-huge-int", "second-value-nan"])
 def test_compare_exits_2_on_a_bad_spec(tmp_path, monkeypatch, capsys, content):
     spec = tmp_path / "spec.yaml"
     _write(spec, content)
@@ -80,6 +84,18 @@ def test_compare_exits_2_on_a_bad_spec(tmp_path, monkeypatch, capsys, content):
     assert err.startswith("Error: ") and str(spec) in err and "Traceback" not in err
 
 
+@pytest.mark.parametrize("value", ["0", "0.05", "3"])
+def test_compare_accepts_a_non_negative_regression_threshold(tmp_path, monkeypatch, value):
+    spec = tmp_path / "spec.yaml"
+    spec.write_text(f"thresholds: {{regressions: {{score_drop_warn: {value}}}}}\n", encoding="utf-8")
+    good = _snapshot(tmp_path / "good.json")
+    argv = ["dokimi", "compare", "--base", good, "--candidate", good, "--spec", str(spec),
+            "--out-json", str(tmp_path / "c.json"), "--out-md", str(tmp_path / "c.md")]
+    monkeypatch.setattr("sys.argv", argv)
+    cli.main()
+    assert (tmp_path / "c.json").is_file()
+
+
 _SPEC_MIN_DOCS = (
     "weights:\n  coverage: 0.4\n  freshness_lite: 0.3\n  readability: 0.3\n"
     "metrics:\n  freshness_lite:\n    max_age_days: 365\n  readability:\n    grade_max: 12\n"

From 6e782be91c9b2350d710025a45e02660dd709729 Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Mon, 26 Oct 2026 15:11:52 -0500
Subject: [PATCH 24/49] fix(cli): treat an unreadable batch pin as a message,
 not a traceback

_pin_batch reads _eval/batch.json whenever the path exists and reports any read failure, including a directory or a permission error, as an unreadable pin naming the file.

The agent-key test lists the keys it checks directly, so removing one from the report's constant fails the test. The grade check gets a status-only case.

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/commands/repo_run.py               |  4 ++--
 dokimi/tests/test_repo_report_unreadable_cell.py |  4 ++--
 dokimi/tests/test_repo_run.py                    | 13 +++++++++++++
 3 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/dokimi/dokimi/commands/repo_run.py b/dokimi/dokimi/commands/repo_run.py
index 96177031f..e00a61185 100644
--- a/dokimi/dokimi/commands/repo_run.py
+++ b/dokimi/dokimi/commands/repo_run.py
@@ -125,10 +125,10 @@ def _uses_builtin_graders(content: Path, manifest: RepoManifest) -> bool:
 
 def _pin_batch(batch_dir: Path, pin: dict[str, str]) -> None:
     path = batch_dir / "_eval" / "batch.json"
-    if path.is_file():
+    if path.exists():
         try:
             old = json.loads(path.read_text(encoding="utf-8"))
-        except (ValueError, RecursionError) as exc:  # JSONDecodeError, bad UTF-8, deep nesting
+        except (OSError, ValueError, RecursionError) as exc:  # unreadable, bad JSON or UTF-8, deep nesting
             old = exc
         if not isinstance(old, dict):
             raise ValueError(f"batch {batch_dir.name} has an unreadable pin ({path}); use another --batch")
diff --git a/dokimi/tests/test_repo_report_unreadable_cell.py b/dokimi/tests/test_repo_report_unreadable_cell.py
index 3bd2bb11b..79a741453 100644
--- a/dokimi/tests/test_repo_report_unreadable_cell.py
+++ b/dokimi/tests/test_repo_report_unreadable_cell.py
@@ -10,7 +10,6 @@
 import pytest
 
 from dokimi.cli import build_parser
-from dokimi.repos.report import _AGENT_KEYS
 
 
 def _batch(tmp_path, name: str, files: dict[str, bytes]):
@@ -82,6 +81,7 @@ def test_report_still_reads_an_empty_batch(tmp_path, capsys):
     pytest.param(b'{"grade": null}', id="null"),
     pytest.param(b'{"grade": []}', id="list"),
     pytest.param(b'{"grade": {"score_only": false}, "task": "t", "arm": "a", "model": "m"}', id="no-status"),
+    pytest.param(b'{"grade": {"status": "ok"}}', id="status-only"),
 ]
 
 
@@ -164,7 +164,7 @@ def test_an_unreadable_result_file_exits_1_naming_the_file(tmp_path, capsys):
     assert f"cannot read {batch / 't__a__m__r1' / 'result.json'}" in err and "Traceback" not in err
 
 
-@pytest.mark.parametrize("key", _AGENT_KEYS)
+@pytest.mark.parametrize("key", ["timed_out", "is_error", "over_budget", "cost_usd", "num_turns"])
 def test_a_result_file_missing_an_agent_key_exits_1(tmp_path, capsys, key):
     agent = {k: v for k, v in _GOOD["agent"].items() if k != key}
     batch = _batch(tmp_path, "b1", {"t__a__m__r1/result.json": json.dumps({**_GOOD, "agent": agent}).encode()})
diff --git a/dokimi/tests/test_repo_run.py b/dokimi/tests/test_repo_run.py
index b2daf365c..e30ad5761 100644
--- a/dokimi/tests/test_repo_run.py
+++ b/dokimi/tests/test_repo_run.py
@@ -867,6 +867,19 @@ def test_a_batch_whose_pin_is_unreadable_exits_1_naming_the_file(setup, capsys,
     assert err.startswith("✗ ") and "_eval/batch.json" in err and "use another --batch" in err
 
 
+def test_a_batch_whose_pin_is_a_directory_exits_1_naming_the_file(setup, capsys):
+    _run(setup, "--arm", "guidance:raw", "--task", "fix_add")
+    pin = setup["out"] / "b1/_eval/batch.json"
+    pin.unlink()
+    pin.mkdir()
+    capsys.readouterr()
+    with pytest.raises(SystemExit) as exc:
+        _run(setup, "--arm", "guidance:none", "--task", "fix_add")
+    assert exc.value.code == 1
+    err = capsys.readouterr().err
+    assert err.startswith("✗ ") and "_eval/batch.json" in err and "use another --batch" in err
+
+
 def test_graders_digest_follows_the_grader_sources(tmp_path, monkeypatch):
     from dokimi.repos import graders
 

From d583f548ba08a6b266ded798127872f12b2a7e30 Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Mon, 26 Oct 2026 15:34:38 -0500
Subject: [PATCH 25/49] fix(repos): bound numeric cell fields and cover
 accepted skills blocks

Numbers larger than 1e300 are rejected so sums over a batch stay finite. The same check now covers num_turns, token counts and score. The tests cover accepted skills blocks through repo report, a skills block with an expected skill but no skill_selected_correct, and a skills block without skill_invoked.

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/repos/report.py                 |  7 ++++--
 .../tests/test_repo_report_unreadable_cell.py | 25 +++++++++++++++++++
 2 files changed, 30 insertions(+), 2 deletions(-)

diff --git a/dokimi/dokimi/repos/report.py b/dokimi/dokimi/repos/report.py
index 733cde557..71bf78128 100644
--- a/dokimi/dokimi/repos/report.py
+++ b/dokimi/dokimi/repos/report.py
@@ -74,14 +74,17 @@ def _planned_cells(plan_path: Path) -> set[str]:
 _AGENT_NUMBERS = ("cost_usd", "num_turns", *_TOKENS)
 
 
+_SAFE_MAGNITUDE = 1e300  # a sum of a batch's values stays a finite float
+
+
 def _optional_number(value: object) -> bool:
-    """True for None or a finite, non-bool number; an int too big for a float is not."""
+    """True for None or a finite, non-bool number whose magnitude keeps sums of a batch finite."""
     if value is None:
         return True
     if isinstance(value, bool) or not isinstance(value, (int, float)):
         return False
     try:
-        return math.isfinite(float(value))
+        return math.isfinite(float(value)) and abs(float(value)) <= _SAFE_MAGNITUDE
     except OverflowError:
         return False
 
diff --git a/dokimi/tests/test_repo_report_unreadable_cell.py b/dokimi/tests/test_repo_report_unreadable_cell.py
index 79a741453..38a261fc8 100644
--- a/dokimi/tests/test_repo_report_unreadable_cell.py
+++ b/dokimi/tests/test_repo_report_unreadable_cell.py
@@ -119,6 +119,11 @@ def test_a_result_file_without_a_usable_grade_exits_1(tmp_path, capsys, argv, co
     pytest.param("agent", {**_GOOD["agent"], "input_tokens": True}, id="input-tokens-a-bool"),
     pytest.param("skills", [], id="skills-not-a-mapping"),
     pytest.param("skills", {"measured": True, "expected_skill": "s"}, id="skills-no-skill-invoked"),
+    pytest.param("skills", {"measured": True}, id="skills-measured-no-skill-invoked"),
+    pytest.param("skills", {"measured": True, "expected_skill": "s", "skill_invoked": True},
+                 id="skills-expected-no-selected-correct"),
+    pytest.param("agent", {**_GOOD["agent"], "cost_usd": 10**308}, id="cost-at-10-to-the-308"),
+    pytest.param("agent", {**_GOOD["agent"], "num_turns": 10**308}, id="num-turns-at-10-to-the-308"),
 ]
 
 
@@ -137,6 +142,26 @@ def test_a_result_file_without_usable_agent_task_arm_or_model_exits_1(tmp_path,
     assert "Traceback" not in err
 
 
+_SKILLS_ACCEPTED = [
+    pytest.param({"measured": False}, {}, id="unmeasured"),
+    pytest.param({"measured": True, "skill_invoked": True}, {"skill_invoked": 1}, id="measured-no-expected-skill"),
+    pytest.param({"measured": True, "expected_skill": "s", "skill_invoked": True, "skill_selected_correct": True},
+                 {"skill_expected": 1, "skill_selected_correct": 1}, id="measured-expected-skill"),
+]
+
+
+@pytest.mark.parametrize("skills, shown", _SKILLS_ACCEPTED)
+def test_a_valid_skills_block_is_accepted_by_repo_report(tmp_path, capsys, skills, shown):
+    rec = {**_GOOD, "skills": skills}
+    batch = _batch(tmp_path, "b1", {"t__a__m__r1/result.json": json.dumps(rec).encode()})
+    _run("report", str(batch), "--json")
+    row = json.loads(capsys.readouterr().out)["rows"][0]
+    if skills["measured"]:
+        assert {k: row[k] for k in shown} == shown
+    else:
+        assert "skill_measured" not in row
+
+
 def test_a_result_file_with_a_score_task_and_no_score_only_is_accepted(tmp_path, capsys):
     rec = {**_GOOD, "score_task": False, "grade": {"status": "ok", "score": 1.0, "success": True},
            "agent": {**_GOOD["agent"], "cost_usd": None}}

From 40a759e406858e63d0397509ab4de6e13be55510 Mon Sep 17 00:00:00 2001
From: Anton Volkov <antonwolfy@gmail.com>
Date: Mon, 26 Oct 2026 21:03:13 +0000
Subject: [PATCH 26/49] fix(repos): lower the cell magnitude bound so variance
 stays finite

Signed-off-by: Anton Volkov <antonwolfy@gmail.com>
---
 dokimi/dokimi/repos/report.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dokimi/dokimi/repos/report.py b/dokimi/dokimi/repos/report.py
index 71bf78128..82bed15c4 100644
--- a/dokimi/dokimi/repos/report.py
+++ b/dokimi/dokimi/repos/report.py
@@ -74,7 +74,7 @@ def _planned_cells(plan_path: Path) -> set[str]:
 _AGENT_NUMBERS = ("cost_usd", "num_turns", *_TOKENS)
 
 
-_SAFE_MAGNITUDE = 1e300  # a sum of a batch's values stays a finite float
+_SAFE_MAGNITUDE = 1e150  # sums stay finite and squared per-task deltas (variance) do too
 
 
 def _optional_number(value: object) -> bool:

From dec3b34236342aa711eb907d9fd07e907746cc1b Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Mon, 26 Oct 2026 16:14:08 -0500
Subject: [PATCH 27/49] fix(repos): reject score-only cells without a score and
 pin the bound at 1e150

A cell with score_only true and no score is now rejected with a message instead of raising KeyError in repo report and repo compare. The magnitude bound is 1e150, so squared per-task deltas in the variance stay finite. Tests cover the bound on both sides, each token key, an empty skills block and a skills block without measured.

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/repos/report.py                 | 23 +++++---
 .../tests/test_repo_report_unreadable_cell.py | 53 ++++++++++++++++++-
 2 files changed, 68 insertions(+), 8 deletions(-)

diff --git a/dokimi/dokimi/repos/report.py b/dokimi/dokimi/repos/report.py
index 82bed15c4..ad4f60386 100644
--- a/dokimi/dokimi/repos/report.py
+++ b/dokimi/dokimi/repos/report.py
@@ -74,11 +74,12 @@ def _planned_cells(plan_path: Path) -> set[str]:
 _AGENT_NUMBERS = ("cost_usd", "num_turns", *_TOKENS)
 
 
-_SAFE_MAGNITUDE = 1e150  # sums stay finite and squared per-task deltas (variance) do too
+# keeps per-task deltas squared (variance in eval/stats.py mean_ci) and sums finite
+_SAFE_MAGNITUDE = 1e150
 
 
 def _optional_number(value: object) -> bool:
-    """True for None or a finite, non-bool number whose magnitude keeps sums of a batch finite."""
+    """True for None or a finite, non-bool number small enough that its squared deltas and sums stay finite."""
     if value is None:
         return True
     if isinstance(value, bool) or not isinstance(value, (int, float)):
@@ -101,13 +102,21 @@ def _skills_usable(skills: object) -> bool:
     return all(k in skills for k in needed)
 
 
+def _grade_usable(rec: dict[str, Any]) -> bool:
+    """True when ``grade`` holds the keys :func:`summarize` reads for the cell's kind."""
+    grade = rec.get("grade")
+    if not isinstance(grade, dict) or any(k not in grade for k in ("status", "success")):
+        return False
+    if "score_task" in rec:
+        return "score" in grade
+    if "score_only" not in grade:
+        return False
+    return "score" in grade or not grade["score_only"]
+
+
 def _check_cell(path: Path, rec: dict[str, Any]) -> None:
     """Raise a :class:`CellFileError` unless ``rec`` has the fields the report and compare read."""
-    grade = rec.get("grade")
-    needed = ("status", "score", "success") if "score_task" in rec else ("status", "score_only", "success")
-    if not isinstance(grade, dict) or any(k not in grade for k in needed):
-        raise CellFileError(f"{path} has no usable `grade`")
-    if not _optional_number(grade.get("score")):
+    if not _grade_usable(rec) or not _optional_number(rec["grade"].get("score")):
         raise CellFileError(f"{path} has no usable `grade`")
     agent = rec.get("agent")
     if not isinstance(agent, dict) or any(k not in agent for k in _AGENT_KEYS):
diff --git a/dokimi/tests/test_repo_report_unreadable_cell.py b/dokimi/tests/test_repo_report_unreadable_cell.py
index 38a261fc8..90d7925ea 100644
--- a/dokimi/tests/test_repo_report_unreadable_cell.py
+++ b/dokimi/tests/test_repo_report_unreadable_cell.py
@@ -6,10 +6,12 @@
 from __future__ import annotations
 
 import json
+import math
 
 import pytest
 
 from dokimi.cli import build_parser
+from dokimi.repos.report import _SAFE_MAGNITUDE
 
 
 def _batch(tmp_path, name: str, files: dict[str, bytes]):
@@ -123,6 +125,17 @@ def test_a_result_file_without_a_usable_grade_exits_1(tmp_path, capsys, argv, co
     pytest.param("skills", {"measured": True, "expected_skill": "s", "skill_invoked": True},
                  id="skills-expected-no-selected-correct"),
     pytest.param("agent", {**_GOOD["agent"], "cost_usd": 10**308}, id="cost-at-10-to-the-308"),
+    pytest.param("agent", {**_GOOD["agent"], "cost_usd": -1e308}, id="cost-at-minus-1e308"),
+    pytest.param("agent", {**_GOOD["agent"], "cost_usd": math.nextafter(_SAFE_MAGNITUDE, math.inf)},
+                 id="cost-just-above-the-bound"),
+    pytest.param("agent", {**_GOOD["agent"], "cost_usd": -math.nextafter(_SAFE_MAGNITUDE, math.inf)},
+                 id="cost-just-below-the-negative-bound"),
+    pytest.param("agent", {**_GOOD["agent"], "output_tokens": "1"}, id="output-tokens-a-string"),
+    pytest.param("agent", {**_GOOD["agent"], "cache_read_tokens": True}, id="cache-read-tokens-a-bool"),
+    pytest.param("agent", {**_GOOD["agent"], "cache_write_tokens": "1"}, id="cache-write-tokens-a-string"),
+    pytest.param("grade", {"status": "ok", "score_only": True, "success": True}, id="score-only-without-score"),
+    pytest.param("skills", {"expected_skill": "s", "skill_invoked": True},
+                 id="skills-without-measured-no-selected-correct"),
     pytest.param("agent", {**_GOOD["agent"], "num_turns": 10**308}, id="num-turns-at-10-to-the-308"),
 ]
 
@@ -144,6 +157,8 @@ def test_a_result_file_without_usable_agent_task_arm_or_model_exits_1(tmp_path,
 
 _SKILLS_ACCEPTED = [
     pytest.param({"measured": False}, {}, id="unmeasured"),
+    pytest.param({}, {}, id="empty"),
+    pytest.param({"skill_invoked": True}, {"skill_invoked": 1}, id="no-measured-key"),
     pytest.param({"measured": True, "skill_invoked": True}, {"skill_invoked": 1}, id="measured-no-expected-skill"),
     pytest.param({"measured": True, "expected_skill": "s", "skill_invoked": True, "skill_selected_correct": True},
                  {"skill_expected": 1, "skill_selected_correct": 1}, id="measured-expected-skill"),
@@ -156,12 +171,48 @@ def test_a_valid_skills_block_is_accepted_by_repo_report(tmp_path, capsys, skill
     batch = _batch(tmp_path, "b1", {"t__a__m__r1/result.json": json.dumps(rec).encode()})
     _run("report", str(batch), "--json")
     row = json.loads(capsys.readouterr().out)["rows"][0]
-    if skills["measured"]:
+    if skills.get("measured", True):
         assert {k: row[k] for k in shown} == shown
     else:
         assert "skill_measured" not in row
 
 
+_NUMBERS_ACCEPTED = [pytest.param(_SAFE_MAGNITUDE, id="at-the-bound"), pytest.param(-_SAFE_MAGNITUDE, id="at-minus-the-bound")]
+
+
+@pytest.mark.parametrize("cost", _NUMBERS_ACCEPTED)
+def test_a_cost_at_the_bound_is_accepted(tmp_path, capsys, cost):
+    rec = {**_GOOD, "agent": {**_GOOD["agent"], "cost_usd": cost}}
+    batch = _batch(tmp_path, "b1", {"t__a__m__r1/result.json": json.dumps(rec).encode()})
+    _run("report", str(batch), "--json")
+    assert json.loads(capsys.readouterr().out)["rows"]
+
+
+def _three_task_batch(tmp_path, cost_a, cost_b):
+    files = {}
+    for task in ("t1", "t2", "t3"):
+        for arm, cost in (("a", cost_a), ("b", cost_b)):
+            rec = {**_GOOD, "task": task, "arm": arm, "agent": {**_GOOD["agent"], "cost_usd": cost}}
+            files[f"{task}__{arm}__m__r1/result.json"] = json.dumps(rec).encode()
+    return _batch(tmp_path, "b1", files)
+
+
+@pytest.mark.parametrize("cmd", [("report",), ("compare", "--baseline", "a")], ids=["report", "compare"])
+def test_deltas_that_would_overflow_when_squared_are_rejected_with_a_message(tmp_path, capsys, cmd):
+    batch = _three_task_batch(tmp_path, 1e300, -1e300)
+    with pytest.raises(SystemExit) as exc:
+        _run(cmd[0], str(batch), *cmd[1:])
+    assert exc.value.code == 1
+    assert "has no usable `agent`" in capsys.readouterr().err
+
+
+@pytest.mark.parametrize("cmd", [("report",), ("compare", "--baseline", "a")], ids=["report", "compare"])
+def test_deltas_at_the_bound_compare_without_overflow(tmp_path, capsys, cmd):
+    batch = _three_task_batch(tmp_path, 1e150, -1e150)
+    _run(cmd[0], str(batch), *cmd[1:], "--json")
+    assert json.loads(capsys.readouterr().out)
+
+
 def test_a_result_file_with_a_score_task_and_no_score_only_is_accepted(tmp_path, capsys):
     rec = {**_GOOD, "score_task": False, "grade": {"status": "ok", "score": 1.0, "success": True},
            "agent": {**_GOOD["agent"], "cost_usd": None}}

From 276795ddc4f27e07a01451e21560362a9ed954e4 Mon Sep 17 00:00:00 2001
From: Anton Volkov <antonwolfy@gmail.com>
Date: Mon, 26 Oct 2026 21:33:23 +0000
Subject: [PATCH 28/49] test(repos): build the bound-edge cells from
 _SAFE_MAGNITUDE

Signed-off-by: Anton Volkov <antonwolfy@gmail.com>
---
 dokimi/tests/test_repo_report_unreadable_cell.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dokimi/tests/test_repo_report_unreadable_cell.py b/dokimi/tests/test_repo_report_unreadable_cell.py
index 90d7925ea..d40dd68b9 100644
--- a/dokimi/tests/test_repo_report_unreadable_cell.py
+++ b/dokimi/tests/test_repo_report_unreadable_cell.py
@@ -208,7 +208,7 @@ def test_deltas_that_would_overflow_when_squared_are_rejected_with_a_message(tmp
 
 @pytest.mark.parametrize("cmd", [("report",), ("compare", "--baseline", "a")], ids=["report", "compare"])
 def test_deltas_at_the_bound_compare_without_overflow(tmp_path, capsys, cmd):
-    batch = _three_task_batch(tmp_path, 1e150, -1e150)
+    batch = _three_task_batch(tmp_path, _SAFE_MAGNITUDE, -_SAFE_MAGNITUDE)
     _run(cmd[0], str(batch), *cmd[1:], "--json")
     assert json.loads(capsys.readouterr().out)
 

From 731db3919c8bad0b4b157f82eb12481d98d2231c Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Mon, 26 Oct 2026 18:27:09 -0500
Subject: [PATCH 29/49] fix(cli): report unexpected cell shapes and unreadable
 question sets without a traceback

repo report and repo compare exit 1 with a message when a cell field has a
shape the field checks do not cover. arms run exits 1 when --questions is a
directory or deeply nested JSON. A fixed-value case pins the magnitude bound
from above.

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/commands/arms.py                |  4 ++--
 dokimi/dokimi/commands/repo_run.py            | 10 ++++++++++
 dokimi/tests/test_arms_output_errors.py       | 15 +++++++++++++++
 .../tests/test_repo_report_unreadable_cell.py | 19 +++++++++++++++++--
 4 files changed, 44 insertions(+), 4 deletions(-)

diff --git a/dokimi/dokimi/commands/arms.py b/dokimi/dokimi/commands/arms.py
index ccf910004..90863ce02 100644
--- a/dokimi/dokimi/commands/arms.py
+++ b/dokimi/dokimi/commands/arms.py
@@ -88,7 +88,7 @@ def cmd_arms_run(args: argparse.Namespace) -> None:
         output = _build_arms_output(args, specs, questions)
     except EmptyQuestionError:
         raise  # a bad input record, not an arm setup error: cli.main reports it (exit 2)
-    except (ValueError, FileNotFoundError) as exc:
+    except (ValueError, OSError, RecursionError) as exc:
         print(f"Error building arms/plugins: {exc}", file=sys.stderr)
         sys.exit(1)
 
@@ -484,7 +484,7 @@ def _refuse_external_trigger_set(path: str) -> None:
 
     try:
         data = json.loads(Path(path).read_text(encoding="utf-8"))
-    except (OSError, ValueError):
+    except (OSError, ValueError, RecursionError):
         return  # the questions loader reports it
     fmt = detect_trigger_format(data)
     if fmt and fmt != FORMAT_QUESTIONS:
diff --git a/dokimi/dokimi/commands/repo_run.py b/dokimi/dokimi/commands/repo_run.py
index e00a61185..4d8131e4f 100644
--- a/dokimi/dokimi/commands/repo_run.py
+++ b/dokimi/dokimi/commands/repo_run.py
@@ -331,6 +331,10 @@ def _harness_bin(name: str) -> Path:
     return Path(found).resolve()
 
 
+# what a cell field of the wrong shape raises past the field checks in repos/report.py
+_MALFORMED_CELL = (KeyError, TypeError, AttributeError, OverflowError, ValueError)
+
+
 def cmd_repo_report(args: argparse.Namespace) -> None:
     """Per (task, arm, model): gated success, grader failures, missing cells, cost, turns."""
     from dokimi.repos.report import CellFileError, format_report, summarize
@@ -344,6 +348,9 @@ def cmd_repo_report(args: argparse.Namespace) -> None:
     except CellFileError as exc:
         print(f"✗ {exc}", file=sys.stderr)
         sys.exit(1)
+    except _MALFORMED_CELL as exc:
+        print(f"✗ {batch_dir}: a cell file has an unexpected shape ({type(exc).__name__}: {exc})", file=sys.stderr)
+        sys.exit(1)
     print(json.dumps(summary, indent=2) if args.json else format_report(summary))
 
 
@@ -363,6 +370,9 @@ def cmd_repo_compare(args: argparse.Namespace) -> None:
     except (CompareError, CellFileError) as exc:
         print(f"✗ {exc}", file=sys.stderr)
         sys.exit(1)
+    except _MALFORMED_CELL as exc:
+        print(f"✗ a cell file has an unexpected shape ({type(exc).__name__}: {exc})", file=sys.stderr)
+        sys.exit(1)
     print(json.dumps(summary, indent=2) if args.json else format_compare(summary))
 
 
diff --git a/dokimi/tests/test_arms_output_errors.py b/dokimi/tests/test_arms_output_errors.py
index 8bce80ebb..f4a8393ca 100644
--- a/dokimi/tests/test_arms_output_errors.py
+++ b/dokimi/tests/test_arms_output_errors.py
@@ -156,3 +156,18 @@ def test_matrix_run_exits_1_naming_an_unwritable_delta_or_rollup(tmp_path, monke
     assert exc.value.code == 1
     err = capsys.readouterr().err
     assert f"Error: cannot write {bad}" in err and "Traceback" not in err
+
+
+@pytest.mark.parametrize("kind", ["directory", "deep"])
+def test_arms_run_exits_1_on_an_unreadable_questions_file(tmp_path, capsys, kind):
+    questions = tmp_path / "q.json"
+    if kind == "directory":
+        questions.mkdir()
+    else:
+        questions.write_text("[" * 100000, encoding="utf-8")
+    args = build_parser().parse_args(["arms", "run", "--product", "p", "--arms", "a:spec.yaml",
+                                      "--questions", str(questions)])
+    with pytest.raises(SystemExit) as exc:
+        args.func(args)
+    assert exc.value.code == 1
+    assert capsys.readouterr().err.startswith("Error building arms/plugins:")
diff --git a/dokimi/tests/test_repo_report_unreadable_cell.py b/dokimi/tests/test_repo_report_unreadable_cell.py
index d40dd68b9..ddd1c9098 100644
--- a/dokimi/tests/test_repo_report_unreadable_cell.py
+++ b/dokimi/tests/test_repo_report_unreadable_cell.py
@@ -197,9 +197,10 @@ def _three_task_batch(tmp_path, cost_a, cost_b):
     return _batch(tmp_path, "b1", files)
 
 
+@pytest.mark.parametrize("cost", [1e154, 1e300])
 @pytest.mark.parametrize("cmd", [("report",), ("compare", "--baseline", "a")], ids=["report", "compare"])
-def test_deltas_that_would_overflow_when_squared_are_rejected_with_a_message(tmp_path, capsys, cmd):
-    batch = _three_task_batch(tmp_path, 1e300, -1e300)
+def test_deltas_that_would_overflow_when_squared_are_rejected_with_a_message(tmp_path, capsys, cmd, cost):
+    batch = _three_task_batch(tmp_path, cost, -cost)
     with pytest.raises(SystemExit) as exc:
         _run(cmd[0], str(batch), *cmd[1:])
     assert exc.value.code == 1
@@ -304,3 +305,17 @@ def test_a_score_task_result_file_without_grade_success_exits_1(tmp_path, capsys
         _run("report", str(batch))
     assert exc.value.code == 1
     assert "has no usable `grade`" in capsys.readouterr().err
+
+
+@pytest.mark.parametrize("cmd", [("report",), ("compare", "--baseline", "a")], ids=["report", "compare"])
+def test_a_cell_shape_the_checks_miss_exits_1_without_a_traceback(tmp_path, capsys, monkeypatch, cmd):
+    from dokimi.repos import report
+
+    monkeypatch.setattr(report, "_check_cell", lambda path, rec: None)
+    rec = {**_GOOD, "grade": "x"}
+    batch = _batch(tmp_path, "b1", {"t__a__m__r1/result.json": json.dumps(rec).encode(),
+                                    "t__b__m__r1/result.json": json.dumps({**rec, "arm": "b"}).encode()})
+    with pytest.raises(SystemExit) as exc:
+        _run(cmd[0], str(batch), *cmd[1:])
+    assert exc.value.code == 1
+    assert "unexpected shape" in capsys.readouterr().err

From 3782fa33c04fdd47220113eafa48f30ac7f9dc0f Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Tue, 27 Oct 2026 08:34:33 -0500
Subject: [PATCH 30/49] test(repo): pin each cell-shape error caught by repo
 report and compare

Parametrize over the five exceptions in _MALFORMED_CELL, and drop two
readers from the changelog's list of those that still raise.

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 ...20000_vchamarthi_cli-input-run-arms-repo.md |  2 +-
 .../tests/test_repo_report_unreadable_cell.py  | 18 ++++++++++++++++++
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-run-arms-repo.md b/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-run-arms-repo.md
index c371cbfb1..23c7219bc 100644
--- a/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-run-arms-repo.md
+++ b/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-run-arms-repo.md
@@ -1,3 +1,3 @@
 ### Fixed
 
-- **`run`, `compare`, `arms`, `tasks`, `explore` and `repo`** exit with a message on the unreadable, non-UTF-8 or corrupt spec, snapshot, result, plan, task and question files their tests cover, or an unwritable output, instead of a traceback. Some other readers still raise, for example `repo scan --manifest-file`, `repo scan --static`, `repo reach` and `repo budget --harness`, `arms run --questions` and `tasks run --tasks`. `run` and `compare` exit 2 on a bad spec or snapshot (a non-numeric summary value, or fewer documents than the manifest's `min_docs`) and exit 1 when an output cannot be written, as `arms` does. `repo report` and `repo compare` exit 1 on a result file without a usable `grade`, `agent`, `task`, `arm` or `model`. `tasks` error messages use the `Error:` prefix.
+- **`run`, `compare`, `arms`, `tasks`, `explore` and `repo`** exit with a message on the unreadable, non-UTF-8 or corrupt spec, snapshot, result, plan, task and question files their tests cover, or an unwritable output, instead of a traceback. Some other readers still raise, for example `repo scan --manifest-file`, `repo scan --static`, `repo reach` and `tasks run --tasks`. `run` and `compare` exit 2 on a bad spec or snapshot (a non-numeric summary value, or fewer documents than the manifest's `min_docs`) and exit 1 when an output cannot be written, as `arms` does. `repo report` and `repo compare` exit 1 on a result file without a usable `grade`, `agent`, `task`, `arm` or `model`. `tasks` error messages use the `Error:` prefix.
diff --git a/dokimi/tests/test_repo_report_unreadable_cell.py b/dokimi/tests/test_repo_report_unreadable_cell.py
index ddd1c9098..ff71b8235 100644
--- a/dokimi/tests/test_repo_report_unreadable_cell.py
+++ b/dokimi/tests/test_repo_report_unreadable_cell.py
@@ -319,3 +319,21 @@ def test_a_cell_shape_the_checks_miss_exits_1_without_a_traceback(tmp_path, caps
         _run(cmd[0], str(batch), *cmd[1:])
     assert exc.value.code == 1
     assert "unexpected shape" in capsys.readouterr().err
+
+
+@pytest.mark.parametrize("cmd", [("report",), ("compare", "--baseline", "a")], ids=["report", "compare"])
+@pytest.mark.parametrize("exc", [KeyError("k"), TypeError("t"), AttributeError("a"), OverflowError("o"), ValueError("v")],
+                         ids=lambda e: type(e).__name__)
+def test_each_cell_shape_error_exits_1_without_a_traceback(tmp_path, capsys, monkeypatch, cmd, exc):
+    from dokimi.repos import compare, report
+
+    def boom(batch_dir):
+        raise exc
+
+    monkeypatch.setattr(report, "load_cells", boom)
+    monkeypatch.setattr(compare, "load_cells", boom)
+    batch = _batch(tmp_path, "b1", {"t__a__m__r1/result.json": json.dumps(_GOOD).encode()})
+    with pytest.raises(SystemExit) as raised:
+        _run(cmd[0], str(batch), *cmd[1:])
+    assert raised.value.code == 1
+    assert f"unexpected shape ({type(exc).__name__}" in capsys.readouterr().err

From d0b374591601d80ba6ef56a2ebd8d30a17d783ae Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Tue, 27 Oct 2026 13:19:12 -0500
Subject: [PATCH 31/49] fix(repo): record a non-integer agent.num_turns as null

- repo run stores num_turns only when the harness reports an integer, so repo report and repo compare accept the cell

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 .../20261025_120000_vchamarthi_cli-input-run-arms-repo.md   | 2 +-
 dokimi/dokimi/repos/events.py                               | 3 ++-
 dokimi/tests/test_repo_reach.py                             | 6 ++++++
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-run-arms-repo.md b/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-run-arms-repo.md
index 23c7219bc..4decbabc6 100644
--- a/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-run-arms-repo.md
+++ b/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-run-arms-repo.md
@@ -1,3 +1,3 @@
 ### Fixed
 
-- **`run`, `compare`, `arms`, `tasks`, `explore` and `repo`** exit with a message on the unreadable, non-UTF-8 or corrupt spec, snapshot, result, plan, task and question files their tests cover, or an unwritable output, instead of a traceback. Some other readers still raise, for example `repo scan --manifest-file`, `repo scan --static`, `repo reach` and `tasks run --tasks`. `run` and `compare` exit 2 on a bad spec or snapshot (a non-numeric summary value, or fewer documents than the manifest's `min_docs`) and exit 1 when an output cannot be written, as `arms` does. `repo report` and `repo compare` exit 1 on a result file without a usable `grade`, `agent`, `task`, `arm` or `model`. `tasks` error messages use the `Error:` prefix.
+- **`run`, `compare`, `arms`, `tasks`, `explore` and `repo`** exit with a message on the unreadable, non-UTF-8 or corrupt spec, snapshot, result, plan, task and question files their tests cover, or an unwritable output, instead of a traceback. Some other readers still raise, for example `repo scan --manifest-file`, `repo scan --static`, `repo reach` and `tasks run --tasks`. `run` and `compare` exit 2 on a bad spec or snapshot (a non-numeric summary value, or fewer documents than the manifest's `min_docs`) and exit 1 when an output cannot be written, as `arms` does. `repo report` and `repo compare` exit 1 on a result file without a usable `grade`, `agent`, `task`, `arm` or `model`. `tasks` error messages use the `Error:` prefix. `repo run` records `agent.num_turns` as null when the harness reports anything but an integer, so the batch stays reportable.
diff --git a/dokimi/dokimi/repos/events.py b/dokimi/dokimi/repos/events.py
index 10bca0166..b4ada7456 100644
--- a/dokimi/dokimi/repos/events.py
+++ b/dokimi/dokimi/repos/events.py
@@ -161,7 +161,8 @@ def _apply(out: Transcript, rec: dict[str, Any], t: float | None,
     elif kind == "result":
         out.answer = str(rec.get("result") or "").strip()
         out.cost_usd = rec.get("total_cost_usd")
-        out.num_turns = rec.get("num_turns")
+        turns = rec.get("num_turns")
+        out.num_turns = turns if isinstance(turns, int) and not isinstance(turns, bool) else None
         out.is_error = bool(rec.get("is_error"))
         out.result_subtype = rec.get("subtype")
         out.subagent_calls = out.subagent_calls or 0
diff --git a/dokimi/tests/test_repo_reach.py b/dokimi/tests/test_repo_reach.py
index 3e774169c..eb7d95670 100644
--- a/dokimi/tests/test_repo_reach.py
+++ b/dokimi/tests/test_repo_reach.py
@@ -958,3 +958,9 @@ def test_reach_resalts_when_a_guidance_file_already_holds_its_token(tmp_path, mo
                         profile=load_profile(_profile(tmp_path)), model="m", out_dir=tmp_path / "out",
                         work_dir=tmp_path, salt="s", kinds=("root",))
     assert payload["salt"] == "s-1"
+
+
+@pytest.mark.parametrize("turns, expected", [(3, 3), ("3", None), (True, None), (2.5, None), (None, None)])
+def test_num_turns_is_an_int_or_none(turns, expected):
+    raw = json.dumps({"type": "result", "result": "ok", "num_turns": turns})
+    assert convert("claude-stream-json", raw).num_turns == expected

From f71c4909a6d0136c1f4002cde9f23fc3dd43dd0c Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Wed, 28 Oct 2026 09:41:48 -0500
Subject: [PATCH 32/49] fix(repo): count a finished run as started whatever its
 num_turns holds

repo run tested num_turns for None to tell that the agent started. A result record with a non-integer num_turns now reads as None, so a finished run with no tool events skipped the skill uptake measurement. Transcript.result_seen carries the started signal.

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/repos/events.py        |  2 ++
 dokimi/dokimi/repos/runner.py        |  4 ++--
 dokimi/tests/test_repo_skill_arms.py | 11 +++++++++++
 3 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/dokimi/dokimi/repos/events.py b/dokimi/dokimi/repos/events.py
index b4ada7456..9e06874d5 100644
--- a/dokimi/dokimi/repos/events.py
+++ b/dokimi/dokimi/repos/events.py
@@ -51,6 +51,7 @@ class Transcript:
     events: list[dict[str, Any]] = field(default_factory=list)
     cost_usd: float | None = None
     num_turns: int | None = None
+    result_seen: bool = False  # the harness wrote its final result record, so the agent started
     input_tokens: int | None = None
     output_tokens: int | None = None
     cache_read_tokens: int | None = None
@@ -161,6 +162,7 @@ def _apply(out: Transcript, rec: dict[str, Any], t: float | None,
     elif kind == "result":
         out.answer = str(rec.get("result") or "").strip()
         out.cost_usd = rec.get("total_cost_usd")
+        out.result_seen = True
         turns = rec.get("num_turns")
         out.num_turns = turns if isinstance(turns, int) and not isinstance(turns, bool) else None
         out.is_error = bool(rec.get("is_error"))
diff --git a/dokimi/dokimi/repos/runner.py b/dokimi/dokimi/repos/runner.py
index d8a6ae423..8f0ed6631 100644
--- a/dokimi/dokimi/repos/runner.py
+++ b/dokimi/dokimi/repos/runner.py
@@ -290,7 +290,7 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path,
         meta["harness_version"] = transcript.harness_version or opts.harness_version
         meta["model_resolved"] = transcript.model_resolved()  # an alias can map `model` to another one
         _write_json(rd / "meta.json", meta)
-        if transcript.events or transcript.num_turns is not None:  # else the agent never started
+        if transcript.events or transcript.result_seen:  # else the agent never started
             events = transcript.events
         cost = transcript.cost_usd
         agent.update(
@@ -301,7 +301,7 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path,
             cache_write_tokens=transcript.cache_write_tokens, usage_scope=transcript.usage_scope,
             subagent_calls=transcript.subagent_calls, plugins=transcript.plugins,
             over_budget=cost is not None and cost > caps["budget_usd"], trace_errors=transcript.errors[:5])
-        if meta.get("skills") and (transcript.events or transcript.num_turns is not None):  # else it never started
+        if meta.get("skills") and (transcript.events or transcript.result_seen):  # else it never started
             skills = skill_usage(transcript.events, [s["name"] for s in meta["skills"]], task.expected_skill)
         graded = run_entry(ctx, "grade", task, rd, grade_network, container_name(rd.name, "grade"))
         grade = outcome(task, rd, graded)
diff --git a/dokimi/tests/test_repo_skill_arms.py b/dokimi/tests/test_repo_skill_arms.py
index 0a91b9ff1..4b8142684 100644
--- a/dokimi/tests/test_repo_skill_arms.py
+++ b/dokimi/tests/test_repo_skill_arms.py
@@ -125,6 +125,17 @@ def test_a_task_without_expected_skill_reports_no_selection(setup, monkeypatch):
     assert "skill selected correct" not in format_report(summarize(setup["out"] / "b1"))
 
 
+def test_a_result_with_a_non_integer_num_turns_and_no_tool_events_still_measures_skill_uptake(setup, monkeypatch):
+    # The agent finished (a result record) but made no tool call and reported "3" turns: it started, so uptake is 0, not unmeasured.
+    raw = json.dumps({"type": "result", "result": "ok", "num_turns": "3"})
+    monkeypatch.setattr("dokimi.repos.runner.convert", lambda trace, _raw, workdir=None: convert(trace, raw, workdir))
+    _run(setup, "--arm", "skill:skills/lint", "--task", "fix_add")
+    rd = next((setup["out"] / "b1").glob("fix_add__*"))
+    result = json.loads((rd / "result.json").read_text())
+    assert result["agent"]["num_turns"] is None
+    assert result["skills"]["measured"] is True and result["skills"]["skills_used"] == []
+
+
 def test_catalog_arm_counts_the_expected_skill_picked_first(setup, monkeypatch):
     monkeypatch.setenv("FAKE_SKILLS", "lint,format")  # the agent picks the right skill first
     _run(setup, "--arm", "guidance:none+skill-catalog:skills", "--task", "fix_lint")

From 10151bda635fc41939a224f1d220b5dc78511a74 Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Wed, 28 Oct 2026 09:41:48 -0500
Subject: [PATCH 33/49] fix(cli): catch deeply nested and non-UTF-8 repo inputs
 and a missing tasks run task

repo scan, validate, reach, budget and run exit 1 with a message on a deeply nested --manifest-file, --static or --harness file and on a non-UTF-8 or deeply nested .agent-eval/repo-eval.yaml. tasks run exits with an Error: message on a task that is missing or unreadable. The changelog fragment names what is covered.

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 ...0000_vchamarthi_cli-input-run-arms-repo.md |  2 +-
 dokimi/dokimi/commands/repo.py                |  4 +-
 dokimi/dokimi/commands/tasks.py               |  4 +-
 dokimi/dokimi/repos/harness_profile.py        |  2 +-
 dokimi/dokimi/repos/manifest.py               |  2 +
 dokimi/tests/test_repo_deep_input.py          | 80 +++++++++++++++++++
 .../test_repo_harness_profile_unreadable.py   | 12 +++
 .../tests/test_repo_report_unreadable_cell.py |  1 +
 dokimi/tests/test_tasks_command.py            | 17 ++++
 9 files changed, 118 insertions(+), 6 deletions(-)
 create mode 100644 dokimi/tests/test_repo_deep_input.py

diff --git a/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-run-arms-repo.md b/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-run-arms-repo.md
index 4decbabc6..836bb9e27 100644
--- a/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-run-arms-repo.md
+++ b/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-run-arms-repo.md
@@ -1,3 +1,3 @@
 ### Fixed
 
-- **`run`, `compare`, `arms`, `tasks`, `explore` and `repo`** exit with a message on the unreadable, non-UTF-8 or corrupt spec, snapshot, result, plan, task and question files their tests cover, or an unwritable output, instead of a traceback. Some other readers still raise, for example `repo scan --manifest-file`, `repo scan --static`, `repo reach` and `tasks run --tasks`. `run` and `compare` exit 2 on a bad spec or snapshot (a non-numeric summary value, or fewer documents than the manifest's `min_docs`) and exit 1 when an output cannot be written, as `arms` does. `repo report` and `repo compare` exit 1 on a result file without a usable `grade`, `agent`, `task`, `arm` or `model`. `tasks` error messages use the `Error:` prefix. `repo run` records `agent.num_turns` as null when the harness reports anything but an integer, so the batch stays reportable.
+- **`run`, `compare`, `arms`, `tasks`, `explore` and `repo`** exit with a message on the unreadable, non-UTF-8 or corrupt spec, snapshot, result, plan, task and question files their tests cover, or an unwritable output, instead of a traceback. This covers a deeply nested `--manifest-file`, `--static` or `--harness` file, a non-UTF-8 `.agent-eval/repo-eval.yaml`, and a `tasks run --tasks` name that is missing or unreadable. `run` and `compare` exit 2 on a bad spec or snapshot (a non-numeric summary value, or fewer documents than the manifest's `min_docs`) and exit 1 when an output cannot be written, as `arms` does. `repo report` and `repo compare` exit 1 on a result file without a usable `grade`, `agent`, `task`, `arm` or `model`. `tasks` error messages use the `Error:` prefix. `repo run` records `agent.num_turns` as null when the harness reports anything but an integer, so the batch stays reportable; a finished run with such a value still counts as started.
diff --git a/dokimi/dokimi/commands/repo.py b/dokimi/dokimi/commands/repo.py
index b27f19088..6f608334e 100644
--- a/dokimi/dokimi/commands/repo.py
+++ b/dokimi/dokimi/commands/repo.py
@@ -83,7 +83,7 @@ def cmd_repo_scan(args: argparse.Namespace) -> None:
         tree = open_source(RepoSource.parse(args.source), _cache_dir(args))
         manifest = _load_manifest(tree, args, guidance_default=True)
         static = json.loads(Path(args.static).read_text(encoding="utf-8")) if args.static else None
-    except (SourceError, ManifestError, OSError, ValueError) as exc:  # ValueError: bad JSON or UTF-8
+    except (SourceError, ManifestError, OSError, ValueError, RecursionError) as exc:  # ValueError: bad JSON or UTF-8
         print(f"✗ {exc}", file=sys.stderr)
         sys.exit(1)
     payload = scan_repo(tree, manifest, static)
@@ -314,7 +314,7 @@ def _load_manifest(tree, args: argparse.Namespace, *, guidance_default: bool = F
         return load_manifest(tree, path)
     try:
         raw = yaml.safe_load(Path(args.manifest_file).read_text(encoding="utf-8"))
-    except (OSError, ValueError, yaml.YAMLError) as exc:  # ValueError: not UTF-8
+    except (OSError, ValueError, RecursionError, yaml.YAMLError) as exc:  # ValueError: not UTF-8
         raise ManifestError(args.manifest_file, [str(exc)]) from exc
     return parse_manifest(raw, path)
 
diff --git a/dokimi/dokimi/commands/tasks.py b/dokimi/dokimi/commands/tasks.py
index 49f3da24a..40a7bd5b3 100644
--- a/dokimi/dokimi/commands/tasks.py
+++ b/dokimi/dokimi/commands/tasks.py
@@ -132,8 +132,8 @@ def _select_tasks(args: argparse.Namespace) -> list:
         else:
             task_names = [t.strip() for t in args.tasks.split(",") if t.strip()]
             tasks = [load_task(t, root=root) for t in task_names]
-    except ValueError as exc:
-        raise SystemExit(str(exc)) from None
+    except (ValueError, OSError, RecursionError) as exc:  # a missing or unreadable task, deeply nested YAML
+        raise SystemExit(f"Error: {exc}") from None
     if not tasks:
         raise SystemExit("No tasks selected. Pass --tasks <name[,name]> or --all.")
     return tasks
diff --git a/dokimi/dokimi/repos/harness_profile.py b/dokimi/dokimi/repos/harness_profile.py
index c13170583..97a32ba4d 100644
--- a/dokimi/dokimi/repos/harness_profile.py
+++ b/dokimi/dokimi/repos/harness_profile.py
@@ -210,7 +210,7 @@ def load_profile(name_or_path: str | Path) -> HarnessProfile:
         raw = yaml.safe_load(path.read_text(encoding="utf-8"))
     except yaml.YAMLError as exc:
         raise ProfileError(f"{path}: not valid YAML: {exc}") from exc
-    except (OSError, UnicodeDecodeError) as exc:
+    except (OSError, UnicodeDecodeError, RecursionError) as exc:  # RecursionError: deeply nested `[`
         raise ProfileError(f"{path}: cannot read: {exc}") from exc
     return parse_profile(raw, str(path))
 
diff --git a/dokimi/dokimi/repos/manifest.py b/dokimi/dokimi/repos/manifest.py
index 931a9eccf..adcce1e14 100644
--- a/dokimi/dokimi/repos/manifest.py
+++ b/dokimi/dokimi/repos/manifest.py
@@ -224,6 +224,8 @@ def load_manifest(tree: RepoTree, path: str = DEFAULT_MANIFEST_PATH) -> RepoMani
         raw = yaml.safe_load(tree.read_text(path))
     except yaml.YAMLError as exc:
         raise ManifestError(path, [f"not valid YAML: {exc}"]) from exc
+    except (UnicodeDecodeError, RecursionError) as exc:  # not UTF-8, deeply nested `[`
+        raise ManifestError(path, [f"cannot read: {exc}"]) from exc
     return parse_manifest(raw, path)
 
 
diff --git a/dokimi/tests/test_repo_deep_input.py b/dokimi/tests/test_repo_deep_input.py
new file mode 100644
index 000000000..8966d8961
--- /dev/null
+++ b/dokimi/tests/test_repo_deep_input.py
@@ -0,0 +1,80 @@
+# Copyright (C) 2026 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+"""`repo` commands exit 1 with a message on deeply nested or non-UTF-8 manifest, static and harness files."""
+
+from __future__ import annotations
+
+import pytest
+
+from dokimi.cli import build_parser
+from dokimi.repos.harness_profile import ProfileError, load_profile
+from dokimi.repos.manifest import ManifestError, load_manifest
+from dokimi.repos.source import RepoSource, open_source
+
+DEEP = "[" * 100000
+
+
+def _run(*argv: str) -> None:
+    args = build_parser().parse_args(["repo", *argv])
+    args.func(args)
+
+
+def _exits_1_naming(capsys, path, *argv: str) -> None:
+    with pytest.raises(SystemExit) as exc:
+        _run(*argv)
+    assert exc.value.code == 1
+    assert str(path) in capsys.readouterr().err
+
+
+@pytest.fixture
+def repo(tmp_path):
+    root = tmp_path / "repo"
+    (root / ".agent-eval").mkdir(parents=True)
+    return root
+
+
+@pytest.mark.parametrize("command", [["scan"], ["validate"], ["reach", "--model", "m", "--dry-run"]])
+def test_a_deeply_nested_manifest_file_exits_1_naming_it(repo, tmp_path, capsys, command):
+    manifest = tmp_path / "deep.yaml"
+    manifest.write_text(DEEP, encoding="utf-8")
+    _exits_1_naming(capsys, manifest, command[0], str(repo), "--manifest-file", str(manifest), *command[1:])
+
+
+def test_a_deeply_nested_static_file_exits_1(repo, tmp_path, capsys):
+    static = tmp_path / "static.json"
+    static.write_text(DEEP, encoding="utf-8")
+    manifest = tmp_path / "m.yaml"
+    manifest.write_text("schema: repo_manifest.v1\nid: toy\nguidance:\n  globs: ['AGENTS.md']\n", encoding="utf-8")
+    with pytest.raises(SystemExit) as exc:
+        _run("scan", str(repo), "--manifest-file", str(manifest), "--static", str(static))
+    assert exc.value.code == 1
+    assert capsys.readouterr().err.startswith("✗ ")
+
+
+@pytest.mark.parametrize("command", [["budget"], ["reach", "--model", "m", "--dry-run"]])
+def test_a_deeply_nested_harness_file_exits_1_naming_it(repo, tmp_path, capsys, command):
+    profile = tmp_path / "profile.yaml"
+    profile.write_text(DEEP, encoding="utf-8")
+    _exits_1_naming(capsys, profile, command[0], str(repo), "--harness", str(profile), *command[1:])
+
+
+def test_load_profile_reports_a_deeply_nested_file(tmp_path):
+    profile = tmp_path / "profile.yaml"
+    profile.write_text(DEEP, encoding="utf-8")
+    with pytest.raises(ProfileError, match="profile.yaml"):
+        load_profile(profile)
+
+
+@pytest.mark.parametrize("content", [b"\xff\xfe\x00bad", DEEP.encode()])
+def test_load_manifest_reports_a_repo_manifest_it_cannot_read(repo, content):
+    (repo / ".agent-eval" / "repo-eval.yaml").write_bytes(content)
+    tree = open_source(RepoSource.parse(str(repo)), repo.parent / "cache")
+    with pytest.raises(ManifestError, match="cannot read"):
+        load_manifest(tree)
+
+
+@pytest.mark.parametrize("command", [["validate"], ["reach", "--model", "m", "--dry-run"]])
+def test_a_non_utf8_repo_manifest_exits_1(repo, capsys, command):
+    (repo / ".agent-eval" / "repo-eval.yaml").write_bytes(b"\xff\xfe\x00bad")
+    _exits_1_naming(capsys, "repo-eval.yaml", command[0], str(repo), *command[1:])
diff --git a/dokimi/tests/test_repo_harness_profile_unreadable.py b/dokimi/tests/test_repo_harness_profile_unreadable.py
index c82592fa8..ab2ebe2ac 100644
--- a/dokimi/tests/test_repo_harness_profile_unreadable.py
+++ b/dokimi/tests/test_repo_harness_profile_unreadable.py
@@ -36,6 +36,18 @@ def test_load_profile_names_the_file_it_cannot_read(tmp_path, kind):
         load_profile(path)
 
 
+def test_load_profile_names_a_file_whose_read_raises_oserror(tmp_path, monkeypatch):
+    path = tmp_path / "profile.yaml"
+    path.write_text("name: x", encoding="utf-8")
+
+    def deny(self, *args, **kwargs):
+        raise PermissionError("denied")
+
+    monkeypatch.setattr("pathlib.Path.read_text", deny)
+    with pytest.raises(ProfileError, match="profile.yaml.*denied"):
+        load_profile(path)
+
+
 @pytest.mark.parametrize("kind", ["non-utf8", "unreadable"])
 def test_repo_budget_exits_1_on_a_harness_file_it_cannot_read(tmp_path, capsys, kind):
     path = _bad_profile(tmp_path, kind)
diff --git a/dokimi/tests/test_repo_report_unreadable_cell.py b/dokimi/tests/test_repo_report_unreadable_cell.py
index ff71b8235..e85696ccd 100644
--- a/dokimi/tests/test_repo_report_unreadable_cell.py
+++ b/dokimi/tests/test_repo_report_unreadable_cell.py
@@ -118,6 +118,7 @@ def test_a_result_file_without_a_usable_grade_exits_1(tmp_path, capsys, argv, co
     pytest.param("grade", {**_GOOD["grade"], "score": 10**400}, id="grade-score-overflows-a-float"),
     pytest.param("score_max", True, id="score-max-a-bool"),
     pytest.param("agent", {**_GOOD["agent"], "num_turns": 10**400}, id="num-turns-overflows"),
+    pytest.param("agent", {**_GOOD["agent"], "num_turns": "3"}, id="num-turns-a-string"),
     pytest.param("agent", {**_GOOD["agent"], "input_tokens": True}, id="input-tokens-a-bool"),
     pytest.param("skills", [], id="skills-not-a-mapping"),
     pytest.param("skills", {"measured": True, "expected_skill": "s"}, id="skills-no-skill-invoked"),
diff --git a/dokimi/tests/test_tasks_command.py b/dokimi/tests/test_tasks_command.py
index d413d47d4..2b61f390e 100644
--- a/dokimi/tests/test_tasks_command.py
+++ b/dokimi/tests/test_tasks_command.py
@@ -211,3 +211,20 @@ def test_tasks_run_with_some_cells_errored_keeps_exit_0_and_counts(tmp_path, mon
     out, err = capsys.readouterr()
     assert code == 0
     assert "codex: 0/0 pass (n/a), 1 errored" in out and "incomplete" not in err
+
+
+def test_tasks_run_with_a_missing_task_exits_with_a_message(tmp_path):
+    args = build_parser().parse_args([
+        "tasks", "run", "--tasks", "no-such-task", "--tasks-root", str(tmp_path), "--harnesses", "codex",
+        "--model", "m", "--dry-run",
+    ])
+    with pytest.raises(SystemExit) as excinfo:
+        args.func(args)
+    assert str(excinfo.value).startswith("Error:") and "no-such-task" in str(excinfo.value)
+
+
+def test_tasks_inspect_of_a_missing_path_exits_with_an_error_prefix(tmp_path):
+    args = build_parser().parse_args(["tasks", "inspect", str(tmp_path / "nope")])
+    with pytest.raises(SystemExit) as excinfo:
+        args.func(args)
+    assert str(excinfo.value).startswith("Error:")

From b8b52782d8c49c7d520d7c9308136da464e26806 Mon Sep 17 00:00:00 2001
From: Alexander Rybkin <alexander.rybkin@intel.com>
Date: Wed, 28 Oct 2026 22:33:12 -0700
Subject: [PATCH 34/49] fix(repo): record a non-finite grader score as a grader
 failure

score_value passed NaN, inf and 1e200 from grade.json into result.json,
and the reader check this PR adds then refused the whole batch in
repo report and repo compare. A score the reader would refuse is now
no measurement: the cell is a grader failure, counted as 0.

Signed-off-by: Alexander Rybkin <alexander.rybkin@intel.com>
---
 ...0000_vchamarthi_cli-input-run-arms-repo.md |  2 +-
 dokimi/dokimi/repos/grading.py                | 10 ++++++-
 dokimi/dokimi/repos/report.py                 |  3 +-
 dokimi/tests/test_repo_run.py                 | 29 +++++++++++++++++++
 4 files changed, 41 insertions(+), 3 deletions(-)

diff --git a/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-run-arms-repo.md b/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-run-arms-repo.md
index 836bb9e27..62a052465 100644
--- a/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-run-arms-repo.md
+++ b/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-run-arms-repo.md
@@ -1,3 +1,3 @@
 ### Fixed
 
-- **`run`, `compare`, `arms`, `tasks`, `explore` and `repo`** exit with a message on the unreadable, non-UTF-8 or corrupt spec, snapshot, result, plan, task and question files their tests cover, or an unwritable output, instead of a traceback. This covers a deeply nested `--manifest-file`, `--static` or `--harness` file, a non-UTF-8 `.agent-eval/repo-eval.yaml`, and a `tasks run --tasks` name that is missing or unreadable. `run` and `compare` exit 2 on a bad spec or snapshot (a non-numeric summary value, or fewer documents than the manifest's `min_docs`) and exit 1 when an output cannot be written, as `arms` does. `repo report` and `repo compare` exit 1 on a result file without a usable `grade`, `agent`, `task`, `arm` or `model`. `tasks` error messages use the `Error:` prefix. `repo run` records `agent.num_turns` as null when the harness reports anything but an integer, so the batch stays reportable; a finished run with such a value still counts as started.
+- **`run`, `compare`, `arms`, `tasks`, `explore` and `repo`** exit with a message on the unreadable, non-UTF-8 or corrupt spec, snapshot, result, plan, task and question files their tests cover, or an unwritable output, instead of a traceback. This covers a deeply nested `--manifest-file`, `--static` or `--harness` file, a non-UTF-8 `.agent-eval/repo-eval.yaml`, and a `tasks run --tasks` name that is missing or unreadable. `run` and `compare` exit 2 on a bad spec or snapshot (a non-numeric summary value, or fewer documents than the manifest's `min_docs`) and exit 1 when an output cannot be written, as `arms` does. `repo report` and `repo compare` exit 1 on a result file without a usable `grade`, `agent`, `task`, `arm` or `model`. `tasks` error messages use the `Error:` prefix. `repo run` records `agent.num_turns` as null when the harness reports anything but an integer, so the batch stays reportable; a finished run with such a value still counts as started. A grader score that is NaN, infinite or beyond 1e150 is recorded as a grader failure, so `repo report` and `repo compare` still read the batch.
diff --git a/dokimi/dokimi/repos/grading.py b/dokimi/dokimi/repos/grading.py
index e37043f1b..f6b879b1d 100644
--- a/dokimi/dokimi/repos/grading.py
+++ b/dokimi/dokimi/repos/grading.py
@@ -53,6 +53,7 @@
 from .workspace import Task
 
 GRADE_SCHEMA = "repo_grade.v1"
+SCORE_BOUND = 1e150  # the largest |score| `repo report` reads: squared deltas and sums stay finite
 _PLACEHOLDER_RE = re.compile(r"\{([a-z_]+)\}")
 CONTAINER_PATHS = {"run_dir": RUN_DIR, "eval_dir": CONTENT_DIR, "source_dir": SOURCE_DIR}
 GRADE_HOME = "grade-home"  # under {run_dir}, which the agent's container never mounts
@@ -260,7 +261,14 @@ def score_value(task: Task, grade: dict[str, Any] | None) -> float | None:
     if not task.score or grade is None:
         return None
     value = (grade.get("metrics") or {}).get(task.score["metric"])
-    return float(value) if isinstance(value, (int, float)) and not isinstance(value, bool) else None
+    if not isinstance(value, (int, float)) or isinstance(value, bool):
+        return None
+    try:
+        score = float(value)
+    except OverflowError:
+        return None
+    # NaN (no comparison holds), inf or a huge score is no measurement: `repo report` refuses such a cell
+    return score if abs(score) <= SCORE_BOUND else None
 
 
 def outcome(task: Task, run_dir: Path, result: ContainerResult, error: str | None = None) -> dict[str, Any]:
diff --git a/dokimi/dokimi/repos/report.py b/dokimi/dokimi/repos/report.py
index b6c862181..b6eee1ad6 100644
--- a/dokimi/dokimi/repos/report.py
+++ b/dokimi/dokimi/repos/report.py
@@ -43,6 +43,7 @@
 
 import math
 
+from .grading import SCORE_BOUND
 from .process_counts import process_line, process_means
 from .runner import UNSUPPORTED_FILE, slug
 
@@ -79,7 +80,7 @@ def _planned_cells(plan_path: Path) -> set[str]:
 
 
 # keeps per-task deltas squared (variance in eval/stats.py mean_ci) and sums finite
-_SAFE_MAGNITUDE = 1e150
+_SAFE_MAGNITUDE = SCORE_BOUND
 
 
 def _optional_number(value: object) -> bool:
diff --git a/dokimi/tests/test_repo_run.py b/dokimi/tests/test_repo_run.py
index d8818e04e..14097a844 100644
--- a/dokimi/tests/test_repo_run.py
+++ b/dokimi/tests/test_repo_run.py
@@ -948,3 +948,32 @@ def test_graders_digest_covers_main_and_pairs_names_with_contents(tmp_path, monk
     assert graders.graders_digest() == base
     (tmp_path / "main.py").rename(tmp_path / "other.py")  # same bytes under another name
     assert graders.graders_digest() != base
+
+
+@pytest.mark.parametrize("bad", [float("nan"), float("inf"), 1e200, 10**400], ids=["nan", "inf", "huge", "overflow"])
+def test_a_grader_score_that_is_no_measurement_is_a_grader_failure_and_the_batch_stays_reportable(
+        setup, monkeypatch, capsys, bad):
+    import dokimi.repos.grading as grading
+    read_grade = grading.read_grade
+
+    def bad_recall(run_dir):  # the grader of one cell writes `bad` as its score metric
+        path = run_dir / "grade.json"
+        if run_dir.name == "review_add__guidance-none__m-1__r1" and path.is_file():
+            doc = json.loads(path.read_text())
+            doc["metrics"]["recall"] = bad
+            path.write_text(json.dumps(doc))
+        return read_grade(run_dir)
+
+    monkeypatch.setattr(grading, "read_grade", bad_recall)
+    _run(setup, "--arm", "guidance:none,guidance:raw", "--task", "fix_add", "--task", "review_add")
+    batch = setup["out"] / "b1"
+    grade = json.loads((batch / "review_add__guidance-none__m-1__r1/result.json").read_text())["grade"]
+    assert grade["status"] == "grader_failure" and grade["score"] is None
+    assert grade["reasons"] == ["grade.json has no numeric metrics.recall (task.json `score`)"]
+    rows = {(r["task"], r["arm"]): r for r in _report(setup)["rows"]}
+    assert rows[("review_add", "guidance:none")]["grader_failures"] == 1
+    assert rows[("review_add", "guidance:none")]["score"] == 0.0
+    capsys.readouterr()
+    args = build_parser().parse_args(["repo", "compare", str(batch), "--baseline", "guidance:none"])
+    args.func(args)
+    assert "review_add" in capsys.readouterr().out

From 883f35640b83e38b9f5cdfe1de74768e652a20cf Mon Sep 17 00:00:00 2001
From: Alexander Rybkin <alexander.rybkin@intel.com>
Date: Wed, 28 Oct 2026 22:33:37 -0700
Subject: [PATCH 35/49] test(repo): pin that a finished run with no tool call
 is counted

The started check sat on two lines and only the skills one was tested:
putting the process line back to num_turns left result.json process
null with the suite green. Compute it once and assert process too.

Signed-off-by: Alexander Rybkin <alexander.rybkin@intel.com>
---
 dokimi/dokimi/repos/runner.py        | 5 +++--
 dokimi/tests/test_repo_skill_arms.py | 1 +
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/dokimi/dokimi/repos/runner.py b/dokimi/dokimi/repos/runner.py
index 549f6494c..42f18926f 100644
--- a/dokimi/dokimi/repos/runner.py
+++ b/dokimi/dokimi/repos/runner.py
@@ -292,7 +292,8 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path,
         meta["harness_version"] = transcript.harness_version or opts.harness_version
         meta["model_resolved"] = transcript.model_resolved()  # an alias can map `model` to another one
         _write_json(rd / "meta.json", meta)
-        if transcript.events or transcript.result_seen:  # else the agent never started
+        started = bool(transcript.events) or transcript.result_seen  # else the agent never started
+        if started:
             events = transcript.events
         cost = transcript.cost_usd
         agent.update(
@@ -305,7 +306,7 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path,
             over_budget=cost is not None and cost > caps["budget_usd"], trace_errors=transcript.errors[:5])
         if events is not None:  # after the cost is recorded: a metric that fails must not cost the cell its record
             process = _process_or_null(ctx, events, agent["trace_errors"])
-        if meta.get("skills") and (transcript.events or transcript.result_seen):  # else it never started
+        if meta.get("skills") and started:
             skills = skill_usage(transcript.events, [s["name"] for s in meta["skills"]], task.expected_skill)
         graded = run_entry(ctx, "grade", task, rd, grade_network, container_name(rd.name, "grade"))
         grade = outcome(task, rd, graded)
diff --git a/dokimi/tests/test_repo_skill_arms.py b/dokimi/tests/test_repo_skill_arms.py
index 4b8142684..e0ea32d76 100644
--- a/dokimi/tests/test_repo_skill_arms.py
+++ b/dokimi/tests/test_repo_skill_arms.py
@@ -134,6 +134,7 @@ def test_a_result_with_a_non_integer_num_turns_and_no_tool_events_still_measures
     result = json.loads((rd / "result.json").read_text())
     assert result["agent"]["num_turns"] is None
     assert result["skills"]["measured"] is True and result["skills"]["skills_used"] == []
+    assert result["process"] is not None  # its tool calls are counted (zero), not left unmeasured
 
 
 def test_catalog_arm_counts_the_expected_skill_picked_first(setup, monkeypatch):

From a5af9bc3646e253c6c37749b29d465dbeedc82b3 Mon Sep 17 00:00:00 2001
From: Alexander Rybkin <alexander.rybkin@intel.com>
Date: Wed, 28 Oct 2026 22:34:31 -0700
Subject: [PATCH 36/49] docs(run): say which exit code an empty root gives with
 --strict

The README promised exit 1 under --strict when no document was
measured, but with a golden_manifest (spec.v1.yaml has one) that is a
min_docs violation and now exits 2. Say so, and pin 0 docs and --strict.

Signed-off-by: Alexander Rybkin <alexander.rybkin@intel.com>
---
 dokimi/README.md                        |  2 +-
 dokimi/tests/test_run_compare_errors.py | 11 +++++++----
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/dokimi/README.md b/dokimi/README.md
index 499e2da55..cc3979fb2 100644
--- a/dokimi/README.md
+++ b/dokimi/README.md
@@ -202,7 +202,7 @@ The default static benchmark is configured in `benchmarks/spec.v1.yaml` and curr
 | `readability` | Estimates text readability with a Flesch-Kincaid-style score. |
 | `example_pass_rate` | Executes supported fenced code examples in isolation. |
 
-Weights are normalized across enabled metrics. The benchmark also supports soft gates, hard gates, critical bands, and regression thresholds. Hard gates and critical bands are enforced only when `--strict` is passed. `--strict` also exits 1 when no document was measured.
+Weights are normalized across enabled metrics. The benchmark also supports soft gates, hard gates, critical bands, and regression thresholds. Hard gates and critical bands are enforced only when `--strict` is passed. With `--strict`, a spec without a `golden_manifest` also exits 1 when no document was measured. A spec with one, as `spec.v1.yaml` has, exits 2 with or without `--strict` when fewer documents than its `min_docs` match, none included.
 
 ## LLM and treatment evaluation workflow
 
diff --git a/dokimi/tests/test_run_compare_errors.py b/dokimi/tests/test_run_compare_errors.py
index d258783fc..2f5dcb233 100644
--- a/dokimi/tests/test_run_compare_errors.py
+++ b/dokimi/tests/test_run_compare_errors.py
@@ -103,12 +103,15 @@ def test_compare_accepts_a_non_negative_regression_threshold(tmp_path, monkeypat
 )
 
 
-def test_run_exits_2_when_the_root_has_fewer_docs_than_min_docs(tmp_path, monkeypatch, capsys):
-    (tmp_path / "docs").mkdir()
-    (tmp_path / "docs" / "a.md").write_text("# A\n\ntext\n", encoding="utf-8")
+@pytest.mark.parametrize("docs", [0, 1])
+@pytest.mark.parametrize("strict", [[], ["--strict"]], ids=["plain", "strict"])
+def test_run_exits_2_when_the_root_has_fewer_docs_than_min_docs(tmp_path, monkeypatch, capsys, docs, strict):
+    (tmp_path / "docs").mkdir()  # none measured is a min_docs violation too, as the README says
+    for i in range(docs):
+        (tmp_path / "docs" / f"{i}.md").write_text("# A\n\ntext\n", encoding="utf-8")
     spec = tmp_path / "spec.yaml"
     spec.write_text(_SPEC_MIN_DOCS, encoding="utf-8")
-    code = _main(monkeypatch, "run", "--root", str(tmp_path), "--spec", str(spec),
+    code = _main(monkeypatch, "run", *strict, "--root", str(tmp_path), "--spec", str(spec),
                  "--out-json", str(tmp_path / "o.json"), "--out-md", str(tmp_path / "o.md"))
     assert code == 2
     err = capsys.readouterr().err

From 333d6305685c59d6281d19e86fdd206e4978578d Mon Sep 17 00:00:00 2001
From: Anton Volkov <antonwolfy@gmail.com>
Date: Thu, 29 Oct 2026 12:07:27 +0000
Subject: [PATCH 37/49] Test that the grader score bound is inclusive at its
 edge

Signed-off-by: Anton Volkov <antonwolfy@gmail.com>
---
 dokimi/tests/test_repo_runner.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/dokimi/tests/test_repo_runner.py b/dokimi/tests/test_repo_runner.py
index afe48e21a..efbf6b7c5 100644
--- a/dokimi/tests/test_repo_runner.py
+++ b/dokimi/tests/test_repo_runner.py
@@ -279,6 +279,20 @@ def test_score_only_grade_without_a_numeric_score_metric_is_grader_failure(tmp_p
     assert "no numeric metrics.recall" in got["reasons"][-1]
 
 
+def test_the_score_bound_is_inclusive_at_its_edge(tmp_path):
+    import math
+
+    from dokimi.repos.grading import SCORE_BOUND, score_value
+    task = Task("rev", {"score": {"metric": "m", "max": 1}}, tmp_path)
+
+    def read(value):
+        return score_value(task, {"metrics": {"m": value}})
+
+    assert read(SCORE_BOUND) == SCORE_BOUND and read(-SCORE_BOUND) == -SCORE_BOUND
+    assert read(math.nextafter(SCORE_BOUND, math.inf)) is None
+    assert read(math.nextafter(-SCORE_BOUND, -math.inf)) is None
+
+
 # ------------------------------------------------------------------ tasks check, through the CLI
 
 

From f128b5206095c0c7e15d8d7a5742cda9db0c7a26 Mon Sep 17 00:00:00 2001
From: Anton Volkov <antonwolfy@gmail.com>
Date: Thu, 29 Oct 2026 12:08:14 +0000
Subject: [PATCH 38/49] Word the score-bound grader failure for non-finite and
 huge scores

Signed-off-by: Anton Volkov <antonwolfy@gmail.com>
---
 dokimi/dokimi/repos/grading.py | 3 ++-
 dokimi/tests/test_repo_run.py  | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/dokimi/dokimi/repos/grading.py b/dokimi/dokimi/repos/grading.py
index f6b879b1d..c4d0174e6 100644
--- a/dokimi/dokimi/repos/grading.py
+++ b/dokimi/dokimi/repos/grading.py
@@ -311,7 +311,8 @@ def _grade_reasons(task: Task, grade: dict[str, Any] | None, failed: bool) -> li
         return ["grade.json is score-only (pass: null) but task.json has no `score`"]
     if not failed and task.score and score_value(task, grade) is None:
         # a missing score would leave the mean, while a grader failure counts as 0
-        return [f"grade.json has no numeric metrics.{task.score['metric']} (task.json `score`)"]
+        return ["grade.json has no numeric metrics." + task.score["metric"]
+                + f" that is finite and within +-{SCORE_BOUND:g} (task.json `score`)"]
     return []
 
 
diff --git a/dokimi/tests/test_repo_run.py b/dokimi/tests/test_repo_run.py
index dd34fb21e..4179ac835 100644
--- a/dokimi/tests/test_repo_run.py
+++ b/dokimi/tests/test_repo_run.py
@@ -978,7 +978,8 @@ def bad_recall(run_dir):  # the grader of one cell writes `bad` as its score met
     batch = setup["out"] / "b1"
     grade = json.loads((batch / "review_add__guidance-none__m-1__r1/result.json").read_text())["grade"]
     assert grade["status"] == "grader_failure" and grade["score"] is None
-    assert grade["reasons"] == ["grade.json has no numeric metrics.recall (task.json `score`)"]
+    assert grade["reasons"] == [
+        "grade.json has no numeric metrics.recall that is finite and within +-1e+150 (task.json `score`)"]
     rows = {(r["task"], r["arm"]): r for r in _report(setup)["rows"]}
     assert rows[("review_add", "guidance:none")]["grader_failures"] == 1
     assert rows[("review_add", "guidance:none")]["score"] == 0.0

From 08b97784d0d193827b9ee2add18a8653de857ef1 Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Thu, 29 Oct 2026 11:05:14 -0500
Subject: [PATCH 39/49] fix(arms): matrix run exits 1 when a cell cannot read
 its inputs

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/commands/arms.py          |  4 ++--
 dokimi/tests/test_arms_output_errors.py | 30 +++++++++++++++++++++++++
 2 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/dokimi/dokimi/commands/arms.py b/dokimi/dokimi/commands/arms.py
index 90863ce02..0198feb0d 100644
--- a/dokimi/dokimi/commands/arms.py
+++ b/dokimi/dokimi/commands/arms.py
@@ -123,7 +123,7 @@ def cmd_arms_matrix_run(args: argparse.Namespace) -> None:
             cell_args = argparse.Namespace(**vars(args))
             _apply_matrix_cell(cell_args, cell)
             _check_sampling_support(cell_args)
-    except (ValueError, FileNotFoundError) as exc:
+    except (ValueError, OSError, RecursionError) as exc:
         print(f"Error loading matrix run inputs: {exc}", file=sys.stderr)
         sys.exit(1)
 
@@ -145,7 +145,7 @@ def cmd_arms_matrix_run(args: argparse.Namespace) -> None:
             output = _build_arms_output(cell_args, specs, questions)
         except EmptyQuestionError:
             raise  # a bad input record, not a cell error: cli.main reports it (exit 2)
-        except (ValueError, FileNotFoundError) as exc:
+        except (ValueError, OSError, RecursionError) as exc:
             print(f"Error running matrix cell {cell.name!r}: {exc}", file=sys.stderr)
             sys.exit(1)
         _save_arms_output(output, out_json, out_md)
diff --git a/dokimi/tests/test_arms_output_errors.py b/dokimi/tests/test_arms_output_errors.py
index f4a8393ca..989be7a56 100644
--- a/dokimi/tests/test_arms_output_errors.py
+++ b/dokimi/tests/test_arms_output_errors.py
@@ -171,3 +171,33 @@ def test_arms_run_exits_1_on_an_unreadable_questions_file(tmp_path, capsys, kind
         args.func(args)
     assert exc.value.code == 1
     assert capsys.readouterr().err.startswith("Error building arms/plugins:")
+
+
+@pytest.mark.parametrize("kind", ["directory", "deep"])
+def test_matrix_run_exits_1_on_an_unreadable_questions_file(tmp_path, capsys, kind):
+    args = _matrix_run(tmp_path, tmp_path / "out")
+    questions = Path(args.questions)
+    questions.unlink()
+    if kind == "directory":
+        questions.mkdir()
+    else:
+        questions.write_text("[" * 100000, encoding="utf-8")
+    with pytest.raises(SystemExit) as exc:
+        args.func(args)
+    assert exc.value.code == 1
+    assert capsys.readouterr().err.startswith("Error loading matrix run inputs:")
+
+
+@pytest.mark.parametrize("error", [PermissionError("denied"), RecursionError("deep")], ids=lambda e: type(e).__name__)
+def test_matrix_run_exits_1_when_a_cell_cannot_read_its_inputs(tmp_path, monkeypatch, capsys, error):
+    from dokimi.commands import arms
+
+    def build(*a, **k):
+        raise error
+
+    monkeypatch.setattr(arms, "_build_arms_output", build)
+    args = _matrix_run(tmp_path, tmp_path / "out")
+    with pytest.raises(SystemExit) as exc:
+        args.func(args)
+    assert exc.value.code == 1
+    assert capsys.readouterr().err.startswith("Error running matrix cell 'c1':")

From 1f946af9e91c884bfad03c6acc11947fd466f095 Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Thu, 29 Oct 2026 11:05:15 -0500
Subject: [PATCH 40/49] fix(repo): name an unreadable manifest and a bad
 --static file

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/commands/repo.py       | 13 +++++++++++--
 dokimi/dokimi/repos/manifest.py      |  2 +-
 dokimi/tests/test_repo_deep_input.py | 17 ++++++++++++++++-
 3 files changed, 28 insertions(+), 4 deletions(-)

diff --git a/dokimi/dokimi/commands/repo.py b/dokimi/dokimi/commands/repo.py
index 6f608334e..2ca0d6d3e 100644
--- a/dokimi/dokimi/commands/repo.py
+++ b/dokimi/dokimi/commands/repo.py
@@ -11,6 +11,7 @@
 import re
 import sys
 from pathlib import Path
+from typing import Any
 
 
 def _normalize_whitespace(text: str, max_len: int | None = 100) -> str:
@@ -73,6 +74,14 @@ def cmd_repo_validate(args: argparse.Namespace) -> None:
         sys.exit(1)
 
 
+def _read_static(path: str) -> Any:
+    try:
+        return json.loads(Path(path).read_text(encoding="utf-8"))
+    except (OSError, ValueError, RecursionError) as exc:  # ValueError: bad JSON or UTF-8
+        print(f"✗ --static {path}: {exc}", file=sys.stderr)
+        sys.exit(1)
+
+
 def cmd_repo_scan(args: argparse.Namespace) -> None:
     """L0: check every mechanically checkable claim in the guidance against the same tree."""
     from dokimi.artifacts import save_artifact, stamp
@@ -82,10 +91,10 @@ def cmd_repo_scan(args: argparse.Namespace) -> None:
     try:
         tree = open_source(RepoSource.parse(args.source), _cache_dir(args))
         manifest = _load_manifest(tree, args, guidance_default=True)
-        static = json.loads(Path(args.static).read_text(encoding="utf-8")) if args.static else None
-    except (SourceError, ManifestError, OSError, ValueError, RecursionError) as exc:  # ValueError: bad JSON or UTF-8
+    except (SourceError, ManifestError, OSError, ValueError, RecursionError) as exc:
         print(f"✗ {exc}", file=sys.stderr)
         sys.exit(1)
+    static = _read_static(args.static) if args.static else None
     payload = scan_repo(tree, manifest, static)
     if args.out:
         try:
diff --git a/dokimi/dokimi/repos/manifest.py b/dokimi/dokimi/repos/manifest.py
index 86db3b50b..dcebf37d5 100644
--- a/dokimi/dokimi/repos/manifest.py
+++ b/dokimi/dokimi/repos/manifest.py
@@ -228,7 +228,7 @@ def load_manifest(tree: RepoTree, path: str = DEFAULT_MANIFEST_PATH) -> RepoMani
         raw = yaml.safe_load(tree.read_text(path))
     except yaml.YAMLError as exc:
         raise ManifestError(path, [f"not valid YAML: {exc}"]) from exc
-    except (UnicodeDecodeError, RecursionError) as exc:  # not UTF-8, deeply nested `[`
+    except (OSError, UnicodeDecodeError, RecursionError) as exc:  # unreadable, not UTF-8, deeply nested `[`
         raise ManifestError(path, [f"cannot read: {exc}"]) from exc
     return parse_manifest(raw, path)
 
diff --git a/dokimi/tests/test_repo_deep_input.py b/dokimi/tests/test_repo_deep_input.py
index 8966d8961..8ba8548dd 100644
--- a/dokimi/tests/test_repo_deep_input.py
+++ b/dokimi/tests/test_repo_deep_input.py
@@ -49,7 +49,8 @@ def test_a_deeply_nested_static_file_exits_1(repo, tmp_path, capsys):
     with pytest.raises(SystemExit) as exc:
         _run("scan", str(repo), "--manifest-file", str(manifest), "--static", str(static))
     assert exc.value.code == 1
-    assert capsys.readouterr().err.startswith("✗ ")
+    err = capsys.readouterr().err
+    assert err.startswith("✗ ") and str(static) in err
 
 
 @pytest.mark.parametrize("command", [["budget"], ["reach", "--model", "m", "--dry-run"]])
@@ -78,3 +79,17 @@ def test_load_manifest_reports_a_repo_manifest_it_cannot_read(repo, content):
 def test_a_non_utf8_repo_manifest_exits_1(repo, capsys, command):
     (repo / ".agent-eval" / "repo-eval.yaml").write_bytes(b"\xff\xfe\x00bad")
     _exits_1_naming(capsys, "repo-eval.yaml", command[0], str(repo), *command[1:])
+
+
+def test_load_manifest_reports_a_repo_manifest_it_has_no_permission_to_read(repo, monkeypatch):
+    from dokimi.repos.source import WorkTree
+
+    (repo / ".agent-eval" / "repo-eval.yaml").write_text("id: toy\n", encoding="utf-8")
+
+    def denied(self, path):
+        raise PermissionError(13, "Permission denied", path)
+
+    monkeypatch.setattr(WorkTree, "read_text", denied)
+    tree = open_source(RepoSource.parse(str(repo)), repo.parent / "cache")
+    with pytest.raises(ManifestError, match="cannot read: .*Permission denied"):
+        load_manifest(tree)

From 3676296d9b84ad597186bfff95343c00c5688d4f Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Thu, 29 Oct 2026 11:05:15 -0500
Subject: [PATCH 41/49] fix(explore): reject a falsy non-string repo_root and
 name a task file with a bad timestamp

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/commands/explore.py       |  8 ++------
 dokimi/dokimi/explore/tasks.py          |  4 ++--
 dokimi/tests/test_explore_task_files.py | 14 ++++++++++++++
 3 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/dokimi/dokimi/commands/explore.py b/dokimi/dokimi/commands/explore.py
index 8fa7c000c..c958300d2 100644
--- a/dokimi/dokimi/commands/explore.py
+++ b/dokimi/dokimi/commands/explore.py
@@ -121,13 +121,9 @@ def cmd_explore_run(args: argparse.Namespace) -> None:
 
     if args.out_md:
         from dokimi.report.exploration_report import render_explore_report
+        from dokimi.utils import write_text_or_exit
 
-        try:
-            Path(args.out_md).parent.mkdir(parents=True, exist_ok=True)
-            Path(args.out_md).write_text(render_explore_report(output), encoding="utf-8")
-        except OSError as exc:
-            print(f"Error: cannot write {args.out_md}: {exc}", file=sys.stderr)
-            raise SystemExit(1) from exc
+        write_text_or_exit(Path(args.out_md), render_explore_report(output))
         print(f"Saved exploration report: {args.out_md}")
 
     for arm, stats in output["summary"]["per_arm"].items():
diff --git a/dokimi/dokimi/explore/tasks.py b/dokimi/dokimi/explore/tasks.py
index 09b98bb6d..7f1b030dc 100644
--- a/dokimi/dokimi/explore/tasks.py
+++ b/dokimi/dokimi/explore/tasks.py
@@ -113,7 +113,7 @@ def _read_task_yaml(path: Path) -> dict:
         data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
     except OSError as exc:
         raise ValueError(f"Explore task {path} cannot be read: {exc}") from exc
-    except (UnicodeDecodeError, yaml.YAMLError, RecursionError) as exc:
+    except (ValueError, yaml.YAMLError, RecursionError) as exc:  # ValueError: bad UTF-8, an invalid YAML timestamp
         raise ValueError(f"Explore task {path} is not valid UTF-8 YAML: {exc}") from exc
     if not isinstance(data, dict):
         raise ValueError(f"Explore task {path} must hold a YAML mapping")
@@ -138,7 +138,7 @@ def _task_from_data(
     if not references:
         raise ValueError(f"Explore task {path} has no references")
     repo_root = data.get("repo_root")
-    if repo_root and not isinstance(repo_root, str):
+    if repo_root is not None and not isinstance(repo_root, str):
         raise ValueError(f"Explore task {path} repo_root must be a path string")
     if repo_root:
         resolved_root = (path.parent / repo_root).resolve()
diff --git a/dokimi/tests/test_explore_task_files.py b/dokimi/tests/test_explore_task_files.py
index c298420c2..00a9e5ddc 100644
--- a/dokimi/tests/test_explore_task_files.py
+++ b/dokimi/tests/test_explore_task_files.py
@@ -32,6 +32,11 @@ def _exit_message(argv) -> str:
         (b"id: x\nquery: q\nreferences: [a.py]\n", "references must be a list"),
         (b"id: x\nquery: q\nreferences:\n  - lines: [1]\n", "references must be a list"),
         (b"id: x\nquery: q\nreferences:\n  - path: a.py\nrepo_root: 5\n", "repo_root must be"),
+        pytest.param(b"id: x\nquery: q\nreferences:\n  - path: a.py\nrepo_root: 0\n", "repo_root must be",
+                     id="falsy-root"),
+        pytest.param(b"id: x\nquery: q\nreferences:\n  - path: a.py\nrepo_root: []\n", "repo_root must be",
+                     id="empty-list-root"),
+        pytest.param(b"id: x\ncreated: 2020-13-45\n", "is not valid UTF-8 YAML", id="bad-timestamp"),
     ],
 )
 def test_a_malformed_task_file_names_the_file(tmp_path, argv_tail, content, message):
@@ -66,3 +71,12 @@ def deny(self, *args, **kwargs):
     monkeypatch.setattr(Path, "read_text", deny)
     out = _exit_message(["explore", "list", "--tasks-root", str(tmp_path)])
     assert f"Explore task {unreadable} cannot be read" in out
+
+
+def test_selecting_a_task_by_id_fails_on_a_malformed_sibling_naming_it(tmp_path):
+    (tmp_path / "ok.yaml").write_text(GOOD, encoding="utf-8")
+    bad = tmp_path / "bad.yaml"
+    bad.write_text("id: [unclosed\n", encoding="utf-8")  # it might declare the id asked for
+    out = _exit_message(["explore", "run", "--tasks", "ok", "--tasks-root", str(tmp_path)])
+    assert str(bad) in out
+    assert "is not valid UTF-8 YAML" in out

From 142aa94046414d798627dac51ee5b683e79550e0 Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Thu, 29 Oct 2026 11:05:16 -0500
Subject: [PATCH 42/49] fix(repo): report a cell field of the wrong shape per
 cell and read a stored unmeasurable score as a grader failure

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/commands/repo_run.py            |  10 --
 dokimi/dokimi/repos/compare.py                |  29 ++---
 dokimi/dokimi/repos/report.py                 | 108 ++++++++++++------
 dokimi/tests/test_repo_compare.py             |   5 +-
 .../tests/test_repo_report_unreadable_cell.py |  78 ++++++++++---
 5 files changed, 151 insertions(+), 79 deletions(-)

diff --git a/dokimi/dokimi/commands/repo_run.py b/dokimi/dokimi/commands/repo_run.py
index c1b5803c1..585a7a544 100644
--- a/dokimi/dokimi/commands/repo_run.py
+++ b/dokimi/dokimi/commands/repo_run.py
@@ -382,10 +382,6 @@ def _harness_bin(name: str) -> Path:
     return Path(found).resolve()
 
 
-# what a cell field of the wrong shape raises past the field checks in repos/report.py
-_MALFORMED_CELL = (KeyError, TypeError, AttributeError, OverflowError, ValueError)
-
-
 def cmd_repo_report(args: argparse.Namespace) -> None:
     """Per (task, arm, model): gated success, grader failures, missing cells, cost, turns."""
     from dokimi.repos.report import CellFileError, format_report, summarize
@@ -399,9 +395,6 @@ def cmd_repo_report(args: argparse.Namespace) -> None:
     except CellFileError as exc:
         print(f"✗ {exc}", file=sys.stderr)
         sys.exit(1)
-    except _MALFORMED_CELL as exc:
-        print(f"✗ {batch_dir}: a cell file has an unexpected shape ({type(exc).__name__}: {exc})", file=sys.stderr)
-        sys.exit(1)
     print(json.dumps(summary, indent=2) if args.json else format_report(summary))
 
 
@@ -422,9 +415,6 @@ def cmd_repo_compare(args: argparse.Namespace) -> None:
     except (CompareError, CellFileError) as exc:
         print(f"✗ {exc}", file=sys.stderr)
         sys.exit(1)
-    except _MALFORMED_CELL as exc:
-        print(f"✗ a cell file has an unexpected shape ({type(exc).__name__}: {exc})", file=sys.stderr)
-        sys.exit(1)
     print(json.dumps(summary, indent=2) if args.json else format_compare(summary))
 
 
diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py
index 510a5af5e..e30fe09df 100644
--- a/dokimi/dokimi/repos/compare.py
+++ b/dokimi/dokimi/repos/compare.py
@@ -87,8 +87,8 @@
 from dokimi.eval.stats import adjust_pvalues, bootstrap_ci, mean_ci
 
 from .process_counts import process_order, process_values
-from .report import (load_cells, mismatch_warnings, model_matches, model_mismatch, resolved_models, unmeasured_note,
-                     uptake_counts)
+from .report import (load_cells, mismatch_warnings, model_matches, model_mismatch, reading_cells, resolved_models,
+                     unmeasured_note, uptake_counts)
 
 #: Largest number of non-zero deltas whose sign-flip distribution is enumerated (2**16 patterns).
 EXACT_MAX = 16
@@ -588,24 +588,25 @@ def _task_values(cells: list[dict[str, Any]], metrics: tuple[str, ...], pass_k:
         groups[(rec["model"], rec["task"])].append(rec)
     out = {}
     for key, recs in groups.items():
-        score_task = any(r["score_task"] for r in recs)
-        found = {"cost_usd": [r["agent"].get("cost_usd") for r in recs],
-                 "turns": [r["agent"].get("num_turns") for r in recs],
-                 **{f"metric:{name}": [r["metrics"].get(name) for r in recs] for name in metrics}}
-        found.update(_process_lists(recs))
-        if score_task:
-            found["score"] = [0.0 if r["grade"]["status"] == "grader_failure" else (r["grade"]["score"] or 0.0)
-                              for r in recs]
-        else:
-            found["gate"] = [float(bool(r["grade"]["success"])) for r in recs]
+        with reading_cells([f"{r['batch']}/{r['cell']}/result.json" for r in recs]):
+            score_task = any(r["score_task"] for r in recs)
+            found = {"cost_usd": [r["agent"].get("cost_usd") for r in recs],
+                     "turns": [r["agent"].get("num_turns") for r in recs],
+                     **{f"metric:{name}": [r["metrics"].get(name) for r in recs] for name in metrics}}
+            found.update(_process_lists(recs))
+            if score_task:
+                found["score"] = [0.0 if r["grade"]["status"] == "grader_failure" else (r["grade"]["score"] or 0.0)
+                                  for r in recs]
+            else:
+                found["gate"] = [float(bool(r["grade"]["success"])) for r in recs]
+            # unmeasured: a grader failure, or an agent outage (see _agent_outage); either way its 0 is not a measurement
+            grader_failed = any(r["grade"]["status"] == "grader_failure" or _agent_outage(r) for r in recs)
         nums = {measure: _finite(vals) for measure, vals in found.items()}
         score_max_vals = [r.get("score_max") for r in recs]
         maxes = set(_finite(score_max_vals))  # cells of two eval trees may differ
         # If any cell has unknown max, or maxes differ, top is None; otherwise take the single value
         has_unknown_max = any(v is None for v in score_max_vals)
         top = (maxes.pop() if len(maxes) == 1 and not has_unknown_max else None) if score_task else 1.0
-        # unmeasured: a grader failure, or an agent outage (see _agent_outage); either way its 0 is not a measurement
-        grader_failed = any(r["grade"]["status"] == "grader_failure" or _agent_outage(r) for r in recs)
         n = {measure: len(v) for measure, v in nums.items()}
         values = {measure: (sum(v) / len(v) if v else None) for measure, v in nums.items()}
         if pass_k and not score_task:
diff --git a/dokimi/dokimi/repos/report.py b/dokimi/dokimi/repos/report.py
index b6eee1ad6..3244f035b 100644
--- a/dokimi/dokimi/repos/report.py
+++ b/dokimi/dokimi/repos/report.py
@@ -38,6 +38,8 @@
 import json
 import re
 from collections import defaultdict
+from collections.abc import Iterator
+from contextlib import contextmanager
 from pathlib import Path
 from typing import Any
 
@@ -114,11 +116,40 @@ def _grade_usable(rec: dict[str, Any]) -> bool:
         return False
     if "score_task" in rec:
         return "score" in grade
-    if "score_only" not in grade:
+    if not isinstance(grade.get("score_only"), bool):
         return False
     return "score" in grade or not grade["score_only"]
 
 
+def _is_number(value: object) -> bool:
+    return isinstance(value, (int, float)) and not isinstance(value, bool)
+
+
+def _fail_unmeasurable_score(rec: dict[str, Any]) -> None:
+    """Read a stored NaN, infinite or out-of-bound ``grade.score`` as the grader failure grading now records."""
+    grade = rec.get("grade")
+    if isinstance(grade, dict) and _is_number(grade.get("score")) and not _optional_number(grade["score"]):
+        reasons = grade.get("reasons")
+        if not isinstance(reasons, list):
+            reasons = []
+        grade.update(status="grader_failure", success=False, score=None,
+                     reasons=[*reasons, f"score {grade['score']!r} is not finite or exceeds {SCORE_BOUND:g}"])
+
+
+# what a cell field of the wrong shape raises past _check_cell
+_MALFORMED_CELL = (KeyError, TypeError, AttributeError, OverflowError, ValueError)
+
+
+@contextmanager
+def reading_cells(files: list[str]) -> Iterator[None]:
+    """Re-raise a field of the wrong shape, read inside the block, as a :class:`CellFileError` naming ``files``."""
+    try:
+        yield
+    except _MALFORMED_CELL as exc:
+        raise CellFileError(f"a field of {', '.join(files)} has an unexpected shape "
+                            f"({type(exc).__name__}: {exc})") from exc
+
+
 def _check_cell(path: Path, rec: dict[str, Any]) -> None:
     """Raise a :class:`CellFileError` unless ``rec`` has the fields the report and compare read."""
     if not _grade_usable(rec) or not _optional_number(rec["grade"].get("score")):
@@ -147,11 +178,13 @@ def load_cells(batch_dir: Path) -> tuple[list[dict[str, Any]], list[str]]:
     finished = []
     for path in sorted(batch_dir.glob("*/result.json")):
         rec = _read_object(path)
+        _fail_unmeasurable_score(rec)
         _check_cell(path, rec)
         rec["cell"] = path.parent.name
         # The row's kind is the task's, not what its cells' grades say. The runner records it from the
         # eval dir's task.json; the cell's copy is in the run dir, which the grader could rewrite.
-        rec["score_task"] = bool(rec["score_task"] if "score_task" in rec else rec["grade"]["score_only"])
+        with reading_cells([str(path)]):
+            rec["score_task"] = bool(rec["score_task"] if "score_task" in rec else rec["grade"]["score_only"])
         finished.append(rec)
     done = {r["cell"] for r in finished}
     unsupported = {p.parent.name for p in batch_dir.glob(f"*/{UNSUPPORTED_FILE}")}
@@ -183,41 +216,42 @@ def summarize(batch_dir: Path) -> dict[str, Any]:
     rows = []
     for key in sorted(groups):
         recs = groups[key]
-        n = len(recs)
-        failures = sum(r["grade"]["status"] == "grader_failure" for r in recs)
-        score_only = any(r["score_task"] for r in recs)
-        row: dict[str, Any] = {"task": key[0], "arm": key[1], "model": key[2], "n": n,
-                               "grader_failures": failures,
-                               "agent_timeouts": sum(bool(r["agent"]["timed_out"]) for r in recs),
-                               "agent_errors": sum(bool(r["agent"]["is_error"]) for r in recs),
-                               "over_budget": sum(bool(r["agent"]["over_budget"]) for r in recs),
-                               "cost_unknown": sum(r["agent"]["cost_usd"] is None for r in recs),
-                               "missing": missing_by.get((key[0], _cell_arm(key[1]), _cell_model(key[2])), 0),
-                               "cost_usd": _mean([r["agent"]["cost_usd"] for r in recs]),
-                               "total_cost_usd": round(sum(r["agent"]["cost_usd"] or 0 for r in recs), 6),
-                               "turns": _mean([r["agent"]["num_turns"] for r in recs]),
-                               "process": process_means(recs),
-                               **{k: _mean([r["agent"].get(k) for r in recs]) for k in _TOKENS},
-                               "networks": sorted({str(r["agent"].get("network")) for r in recs}),
-                               "model_resolved": sorted({m for r in recs for m in resolved_models(r)})}
-        uptake = uptake_counts(recs)
-        if uptake is not None:
-            row["uptake"] = uptake
-        if score_only:
-            scores = [0.0 if r["grade"]["status"] == "grader_failure" else r["grade"]["score"] for r in recs]
-            row.update(kind="score", score=_mean([0.0 if s is None else s for s in scores]))
-        else:
-            wins = sum(bool(r["grade"]["success"]) for r in recs)
-            row.update(kind="gate", successes=wins, success_rate=round(wins / n, 4), ci95=wilson(wins, n))
-        used = [r["skills"] for r in recs if r.get("skills") and r["skills"].get("measured", True)]
-        if used:  # cells whose trace was unreadable are not counted: no rate from a missing measurement
-            row["skill_measured"] = len(used)
-            row["skill_invoked"] = sum(bool(u["skill_invoked"]) for u in used)
-            expecting = [u for u in used if u.get("expected_skill")]  # a task naming no skill has nothing to select
-            if expecting:
-                row["skill_expected"] = len(expecting)
-                row["skill_selected_correct"] = sum(bool(u["skill_selected_correct"]) for u in expecting)
-        rows.append(row)
+        with reading_cells([str(batch_dir / r["cell"] / "result.json") for r in recs]):
+            n = len(recs)
+            failures = sum(r["grade"]["status"] == "grader_failure" for r in recs)
+            score_only = any(r["score_task"] for r in recs)
+            row: dict[str, Any] = {"task": key[0], "arm": key[1], "model": key[2], "n": n,
+                                   "grader_failures": failures,
+                                   "agent_timeouts": sum(bool(r["agent"]["timed_out"]) for r in recs),
+                                   "agent_errors": sum(bool(r["agent"]["is_error"]) for r in recs),
+                                   "over_budget": sum(bool(r["agent"]["over_budget"]) for r in recs),
+                                   "cost_unknown": sum(r["agent"]["cost_usd"] is None for r in recs),
+                                   "missing": missing_by.get((key[0], _cell_arm(key[1]), _cell_model(key[2])), 0),
+                                   "cost_usd": _mean([r["agent"]["cost_usd"] for r in recs]),
+                                   "total_cost_usd": round(sum(r["agent"]["cost_usd"] or 0 for r in recs), 6),
+                                   "turns": _mean([r["agent"]["num_turns"] for r in recs]),
+                                   "process": process_means(recs),
+                                   **{k: _mean([r["agent"].get(k) for r in recs]) for k in _TOKENS},
+                                   "networks": sorted({str(r["agent"].get("network")) for r in recs}),
+                                   "model_resolved": sorted({m for r in recs for m in resolved_models(r)})}
+            uptake = uptake_counts(recs)
+            if uptake is not None:
+                row["uptake"] = uptake
+            if score_only:
+                scores = [0.0 if r["grade"]["status"] == "grader_failure" else r["grade"]["score"] for r in recs]
+                row.update(kind="score", score=_mean([0.0 if s is None else s for s in scores]))
+            else:
+                wins = sum(bool(r["grade"]["success"]) for r in recs)
+                row.update(kind="gate", successes=wins, success_rate=round(wins / n, 4), ci95=wilson(wins, n))
+            used = [r["skills"] for r in recs if r.get("skills") and r["skills"].get("measured", True)]
+            if used:  # cells whose trace was unreadable are not counted: no rate from a missing measurement
+                row["skill_measured"] = len(used)
+                row["skill_invoked"] = sum(bool(u["skill_invoked"]) for u in used)
+                expecting = [u for u in used if u.get("expected_skill")]  # a task naming no skill has nothing to select
+                if expecting:
+                    row["skill_expected"] = len(expecting)
+                    row["skill_selected_correct"] = sum(bool(u["skill_selected_correct"]) for u in expecting)
+            rows.append(row)
     return {"batch": str(batch_dir), "cells": len(cells), "missing": missing, "rows": rows,
             "unsupported": load_unsupported(batch_dir),
             "total_cost_usd": round(sum(r["agent"]["cost_usd"] or 0 for r in cells), 6),
diff --git a/dokimi/tests/test_repo_compare.py b/dokimi/tests/test_repo_compare.py
index f1fe6298e..7cc208e22 100644
--- a/dokimi/tests/test_repo_compare.py
+++ b/dokimi/tests/test_repo_compare.py
@@ -746,8 +746,8 @@ def test_non_finite_metrics_are_not_measurements(tmp_path, bad):
     json.dumps(s, allow_nan=False)
 
 
-@pytest.mark.parametrize("bad", [float("inf"), float("-inf"), float("nan")])
-def test_a_non_finite_cost_rejects_the_cell_with_a_message(tmp_path, capsys, bad):
+@pytest.mark.parametrize("bad", [float("inf"), float("-inf"), float("nan"), "1", True])
+def test_an_unusable_cost_rejects_the_cell_naming_it(tmp_path, capsys, bad):
     b = tmp_path / "b1"
     _cell(b, "fix_0", "guidance:none", 1, cost=bad)
     _cell(b, "fix_0", "guidance:raw", 1)
@@ -756,6 +756,7 @@ def test_a_non_finite_cost_rejects_the_cell_with_a_message(tmp_path, capsys, bad
         args.func(args)
     err = capsys.readouterr().err
     assert exc.value.code == 1 and "has no usable `agent`" in err and "Traceback" not in err
+    assert "fix_0__guidance-none__m-1__r1" in err and "guidance-raw" not in err
 
 
 def test_a_batch_given_as_dot_keeps_its_directory_name(tmp_path, monkeypatch):
diff --git a/dokimi/tests/test_repo_report_unreadable_cell.py b/dokimi/tests/test_repo_report_unreadable_cell.py
index e85696ccd..d55965193 100644
--- a/dokimi/tests/test_repo_report_unreadable_cell.py
+++ b/dokimi/tests/test_repo_report_unreadable_cell.py
@@ -28,6 +28,10 @@ def _run(*argv: str) -> None:
     args.func(args)
 
 
+def _no_constant(name: str) -> None:
+    raise ValueError(f"{name} in the JSON output")
+
+
 _CORRUPT = [
     pytest.param(b"{not json", id="not-json"),
     pytest.param(b"\xff\xfe", id="not-utf8"),
@@ -115,7 +119,8 @@ def test_a_result_file_without_a_usable_grade_exits_1(tmp_path, capsys, argv, co
     pytest.param("agent", {**_GOOD["agent"], "cost_usd": "0.1"}, id="cost-a-string"),
     pytest.param("agent", {**_GOOD["agent"], "cost_usd": True}, id="cost-a-bool"),
     pytest.param("grade", {**_GOOD["grade"], "score": True}, id="grade-score-a-bool"),
-    pytest.param("grade", {**_GOOD["grade"], "score": 10**400}, id="grade-score-overflows-a-float"),
+    pytest.param("grade", {**_GOOD["grade"], "score": "1"}, id="grade-score-a-string"),
+    pytest.param("grade", {**_GOOD["grade"], "score_only": "false", "score": 1.0}, id="grade-score-only-a-string"),
     pytest.param("score_max", True, id="score-max-a-bool"),
     pytest.param("agent", {**_GOOD["agent"], "num_turns": 10**400}, id="num-turns-overflows"),
     pytest.param("agent", {**_GOOD["agent"], "num_turns": "3"}, id="num-turns-a-string"),
@@ -172,8 +177,10 @@ def test_a_valid_skills_block_is_accepted_by_repo_report(tmp_path, capsys, skill
     batch = _batch(tmp_path, "b1", {"t__a__m__r1/result.json": json.dumps(rec).encode()})
     _run("report", str(batch), "--json")
     row = json.loads(capsys.readouterr().out)["rows"][0]
-    if skills.get("measured", True):
+    if skills and skills.get("measured", True):
         assert {k: row[k] for k in shown} == shown
+        if "skill_expected" not in shown:
+            assert "skill_expected" not in row
     else:
         assert "skill_measured" not in row
 
@@ -212,7 +219,7 @@ def test_deltas_that_would_overflow_when_squared_are_rejected_with_a_message(tmp
 def test_deltas_at_the_bound_compare_without_overflow(tmp_path, capsys, cmd):
     batch = _three_task_batch(tmp_path, _SAFE_MAGNITUDE, -_SAFE_MAGNITUDE)
     _run(cmd[0], str(batch), *cmd[1:], "--json")
-    assert json.loads(capsys.readouterr().out)
+    assert json.loads(capsys.readouterr().out, parse_constant=_no_constant)
 
 
 def test_a_result_file_with_a_score_task_and_no_score_only_is_accepted(tmp_path, capsys):
@@ -308,33 +315,72 @@ def test_a_score_task_result_file_without_grade_success_exits_1(tmp_path, capsys
     assert "has no usable `grade`" in capsys.readouterr().err
 
 
-@pytest.mark.parametrize("cmd", [("report",), ("compare", "--baseline", "a")], ids=["report", "compare"])
-def test_a_cell_shape_the_checks_miss_exits_1_without_a_traceback(tmp_path, capsys, monkeypatch, cmd):
+def _two_arm_batch(tmp_path, rec):
+    return _batch(tmp_path, "b1", {"t__a__m__r1/result.json": json.dumps(rec).encode(),
+                                   "t__b__m__r1/result.json": json.dumps({**rec, "arm": "b"}).encode()})
+
+
+_CMDS = pytest.mark.parametrize("cmd", [("report",), ("compare", "--baseline", "a")], ids=["report", "compare"])
+
+
+@_CMDS
+def test_a_cell_shape_the_checks_miss_exits_1_naming_the_cell(tmp_path, capsys, monkeypatch, cmd):
     from dokimi.repos import report
 
     monkeypatch.setattr(report, "_check_cell", lambda path, rec: None)
-    rec = {**_GOOD, "grade": "x"}
-    batch = _batch(tmp_path, "b1", {"t__a__m__r1/result.json": json.dumps(rec).encode(),
-                                    "t__b__m__r1/result.json": json.dumps({**rec, "arm": "b"}).encode()})
+    batch = _two_arm_batch(tmp_path, {**_GOOD, "grade": "x"})
     with pytest.raises(SystemExit) as exc:
         _run(cmd[0], str(batch), *cmd[1:])
     assert exc.value.code == 1
-    assert "unexpected shape" in capsys.readouterr().err
+    err = capsys.readouterr().err
+    assert "t__a__m__r1" in err and "result.json" in err and "unexpected shape" in err
 
 
-@pytest.mark.parametrize("cmd", [("report",), ("compare", "--baseline", "a")], ids=["report", "compare"])
+@_CMDS
 @pytest.mark.parametrize("exc", [KeyError("k"), TypeError("t"), AttributeError("a"), OverflowError("o"), ValueError("v")],
                          ids=lambda e: type(e).__name__)
-def test_each_cell_shape_error_exits_1_without_a_traceback(tmp_path, capsys, monkeypatch, cmd, exc):
+def test_each_cell_shape_error_exits_1_naming_the_cell(tmp_path, capsys, monkeypatch, cmd, exc):
     from dokimi.repos import compare, report
 
-    def boom(batch_dir):
+    def boom(recs):
         raise exc
 
-    monkeypatch.setattr(report, "load_cells", boom)
-    monkeypatch.setattr(compare, "load_cells", boom)
-    batch = _batch(tmp_path, "b1", {"t__a__m__r1/result.json": json.dumps(_GOOD).encode()})
+    monkeypatch.setattr(report, "process_means", boom)
+    monkeypatch.setattr(compare, "_process_lists", boom)
+    batch = _two_arm_batch(tmp_path, _GOOD)
     with pytest.raises(SystemExit) as raised:
         _run(cmd[0], str(batch), *cmd[1:])
     assert raised.value.code == 1
-    assert f"unexpected shape ({type(exc).__name__}" in capsys.readouterr().err
+    err = capsys.readouterr().err
+    assert "t__a__m__r1" in err and f"unexpected shape ({type(exc).__name__}" in err
+
+
+@pytest.mark.parametrize("target", ["pass_hat", "order_distance"])
+def test_an_error_in_compare_statistics_is_not_reported_as_a_bad_cell(tmp_path, monkeypatch, target):
+    from dokimi.repos import compare
+
+    def boom(*args):
+        raise KeyError("a defect, not a cell")
+
+    monkeypatch.setattr(compare, target, boom)
+    batch = _two_arm_batch(tmp_path, _GOOD)
+    with pytest.raises(KeyError, match="a defect, not a cell"):
+        _run("compare", str(batch), "--baseline", "a", "--pass-k", "1", "--path-stability")
+
+
+@_CMDS
+@pytest.mark.parametrize("score", ["NaN", "Infinity", "-Infinity", "1e200", pytest.param("9" * 400, id="400-digits")])
+def test_a_stored_unmeasurable_score_is_read_as_a_grader_failure(tmp_path, capsys, cmd, score):
+    rec = {**_GOOD, "score_task": True, "grade": {"status": "graded", "score_only": True, "success": None,
+                                                  "score": 0.5, "reasons": []}}
+    text = json.dumps(rec).replace("0.5", score)
+    batch = _batch(tmp_path, "b1", {"t__a__m__r1/result.json": text.encode(),
+                                    "t__b__m__r1/result.json": json.dumps({**rec, "arm": "b"}).encode()})
+    _run(cmd[0], str(batch), *cmd[1:], "--json")
+    out = json.loads(capsys.readouterr().out, parse_constant=_no_constant)
+    if cmd[0] == "report":
+        row = next(r for r in out["rows"] if r["arm"] == "a")
+        assert row["grader_failures"] == 1 and row["score"] == 0.0
+    else:
+        row = next(r for r in out["comparisons"] if r["measure"] == "score")
+        assert (row["baseline_mean"], row["treatment_mean"]) == (0.0, 0.5)

From 2ac7e40517ee1749a4fc9b8fff50af209d04dbbf Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Thu, 29 Oct 2026 11:05:16 -0500
Subject: [PATCH 43/49] fix(run): exit 2 only on a bad spec, snapshot or
 document selection

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/README.md                        |  2 +-
 dokimi/dokimi/commands/run.py           | 24 ++++++-----
 dokimi/dokimi/runner/compare.py         | 14 +++---
 dokimi/dokimi/runner/spec.py            | 18 +++++---
 dokimi/tests/test_run_compare_errors.py | 57 ++++++++++++++++++++++---
 5 files changed, 89 insertions(+), 26 deletions(-)

diff --git a/dokimi/README.md b/dokimi/README.md
index cc3979fb2..b673b2d6a 100644
--- a/dokimi/README.md
+++ b/dokimi/README.md
@@ -202,7 +202,7 @@ The default static benchmark is configured in `benchmarks/spec.v1.yaml` and curr
 | `readability` | Estimates text readability with a Flesch-Kincaid-style score. |
 | `example_pass_rate` | Executes supported fenced code examples in isolation. |
 
-Weights are normalized across enabled metrics. The benchmark also supports soft gates, hard gates, critical bands, and regression thresholds. Hard gates and critical bands are enforced only when `--strict` is passed. With `--strict`, a spec without a `golden_manifest` also exits 1 when no document was measured. A spec with one, as `spec.v1.yaml` has, exits 2 with or without `--strict` when fewer documents than its `min_docs` match, none included.
+Weights are normalized across enabled metrics. The benchmark also supports soft gates, hard gates, critical bands, and regression thresholds. Hard gates and critical bands are enforced only when `--strict` is passed. With `--strict`, a spec without a `golden_manifest` also exits 1 when no document was measured. A spec with one, as `spec.v1.yaml` has, exits 2 with or without `--strict` when fewer documents than its `min_docs` match, none included. An empty or absolute `include` pattern exits 2 as well. Exit 1 is not only the `--strict` verdict: a document that cannot be read, or an `--out-json` or `--out-md` path that cannot be written, also exits 1, with an `Error:` or `❌` line on stderr saying which.
 
 ## LLM and treatment evaluation workflow
 
diff --git a/dokimi/dokimi/commands/run.py b/dokimi/dokimi/commands/run.py
index 097bb52f8..46f48d699 100644
--- a/dokimi/dokimi/commands/run.py
+++ b/dokimi/dokimi/commands/run.py
@@ -13,12 +13,16 @@
 
 from dokimi.report.json_report import write_json
 from dokimi.report.markdown_report import write_compare_report, write_run_report
-from dokimi.runner.compare import compare_snapshots
+from dokimi.runner.compare import CompareInputError, compare_snapshots
 from dokimi.runner.run import DocLoadError, load_benchmark_spec, run_benchmark, save_snapshot
+from dokimi.runner.spec import DocSelectionError
 
 
 def _run_or_exit(root: Path, spec_path: Path) -> dict:
-    """Run the benchmark; a bad spec exits 2 (1 is the verdict code of --strict), a bad doc exits 1."""
+    """Run the benchmark; a bad spec or doc selection exits 2, an unreadable doc exits 1.
+
+    Exit 1 is also the verdict code of --strict, and an unwritable output exits 1 too (see _write_outputs).
+    """
     try:
         load_benchmark_spec(spec_path)  # fail on a spec that cannot be read, parsed or validated, before the run
     except (RuntimeError, ValueError) as exc:
@@ -29,7 +33,7 @@ def _run_or_exit(root: Path, spec_path: Path) -> dict:
     except DocLoadError as exc:
         print(f"❌ {exc}", file=sys.stderr)
         sys.exit(1)
-    except ValueError as exc:  # e.g. fewer documents than the manifest's min_docs
+    except DocSelectionError as exc:  # e.g. fewer documents than the manifest's min_docs
         print(f"Error: {exc}", file=sys.stderr)
         sys.exit(2)
 
@@ -91,24 +95,24 @@ def cmd_run(args: argparse.Namespace) -> None:
 
 
 def _load_compare_spec(spec_path: Path) -> dict | None:
-    """The `compare --spec` mapping; any unreadable or wrong-shape spec is a ``ValueError`` naming the file."""
+    """The `compare --spec` mapping; any unreadable or wrong-shape spec is a ``CompareInputError`` naming the file."""
     import yaml
 
     try:
         spec_data = yaml.safe_load(spec_path.read_text(encoding="utf-8"))
     except (OSError, ValueError, RecursionError, yaml.YAMLError) as exc:
-        raise ValueError(f"Invalid spec {spec_path}: {exc}") from exc
+        raise CompareInputError(f"Invalid spec {spec_path}: {exc}") from exc
     if spec_data is not None and not isinstance(spec_data, dict):
-        raise ValueError(f"Invalid spec {spec_path}: root must be a mapping")
+        raise CompareInputError(f"Invalid spec {spec_path}: root must be a mapping")
     thresholds = (spec_data or {}).get("thresholds", {})
     if not isinstance(thresholds, dict):
-        raise ValueError(f"Invalid spec {spec_path}: `thresholds` must be a mapping")
+        raise CompareInputError(f"Invalid spec {spec_path}: `thresholds` must be a mapping")
     regressions = thresholds.get("regressions", {})
     if not isinstance(regressions, dict):
-        raise ValueError(f"Invalid spec {spec_path}: `thresholds.regressions` must be a mapping")
+        raise CompareInputError(f"Invalid spec {spec_path}: `thresholds.regressions` must be a mapping")
     for name, value in regressions.items():
         if not _non_negative_number(value):
-            raise ValueError(
+            raise CompareInputError(
                 f"Invalid spec {spec_path}: `thresholds.regressions.{name}` must be a finite non-negative number, got {value!r}"
             )
     return spec_data
@@ -134,7 +138,7 @@ def cmd_compare(args: argparse.Namespace) -> None:
     try:
         spec_data = _load_compare_spec(spec_path) if spec_path else None
         result = compare_snapshots(base, cand, spec=spec_data)
-    except ValueError as exc:
+    except CompareInputError as exc:
         print(f"Error: {exc}", file=sys.stderr)
         sys.exit(2)
     _write_outputs(((write_json, result, out_json), (write_compare_report, result, out_md)))
diff --git a/dokimi/dokimi/runner/compare.py b/dokimi/dokimi/runner/compare.py
index fba63257e..8750471d7 100644
--- a/dokimi/dokimi/runner/compare.py
+++ b/dokimi/dokimi/runner/compare.py
@@ -16,26 +16,30 @@
 _REQUIRED_SUMMARY_KEYS = {"docs", "score", "coverage", "freshness_lite", "readability"}
 
 
+class CompareInputError(ValueError):
+    """A snapshot or spec that ``compare`` cannot read or that has the wrong shape."""
+
+
 def _load_snapshot(path: Path, role: str) -> Any:
     try:
         return json.loads(path.read_text(encoding="utf-8"))
     except (OSError, ValueError, RecursionError) as exc:  # ValueError covers JSONDecodeError and bad UTF-8
-        raise ValueError(f"Invalid {role} snapshot {path}: {exc}") from exc
+        raise CompareInputError(f"Invalid {role} snapshot {path}: {exc}") from exc
 
 
 def _check_summary(path: Path, payload: object, role: str) -> None:
     summary = payload.get("summary") if isinstance(payload, dict) else None
     if not isinstance(summary, dict):
-        raise ValueError(f"Snapshot missing summary object: {path}")
+        raise CompareInputError(f"Snapshot missing summary object: {path}")
     missing = _REQUIRED_SUMMARY_KEYS - set(summary.keys())
     # example_pass_rate is optional (may not be present in old snapshots)
     if missing:
-        raise ValueError(f"Snapshot {path} missing summary keys: {sorted(missing)}")
+        raise CompareInputError(f"Snapshot {path} missing summary keys: {sorted(missing)}")
     for key in sorted(_REQUIRED_SUMMARY_KEYS | {"example_pass_rate"}):
         value = summary.get(key)
         if key in summary and not _valid_number(key, value):
             kind = "an integer" if key == "docs" else "a finite number"
-            raise ValueError(f"Invalid {role} snapshot {path}: summary `{key}` must be {kind}, got {value!r}")
+            raise CompareInputError(f"Invalid {role} snapshot {path}: summary `{key}` must be {kind}, got {value!r}")
 
 
 def _valid_number(key: str, value: object) -> bool:
@@ -81,7 +85,7 @@ def compare_snapshots(base_path: Path, cand_path: Path, spec: dict | None = None
     # Add regression analysis if spec provided
     if spec is not None:
         if not isinstance(spec, dict):
-            raise ValueError("spec must be a dict (mapping)")
+            raise CompareInputError("spec must be a dict (mapping)")
         regressions = detect_regressions(diff, spec)
         result["regressions"] = {
             "score": {
diff --git a/dokimi/dokimi/runner/spec.py b/dokimi/dokimi/runner/spec.py
index 1d34b0999..1a3417490 100644
--- a/dokimi/dokimi/runner/spec.py
+++ b/dokimi/dokimi/runner/spec.py
@@ -25,6 +25,10 @@ class SpecValidationError(ValueError):
     """Raised when a spec fails JSON Schema validation."""
 
 
+class DocSelectionError(ValueError):
+    """Raised when a ``golden_manifest`` glob is unusable or matches a document count outside its bounds."""
+
+
 def is_full_spec(spec: dict) -> bool:
     """Return True if ``spec`` is a complete v1 spec (vs. a minimal/legacy one).
 
@@ -99,8 +103,8 @@ def select_docs(root: Path, manifest: dict) -> list[Path]:
     """Resolve and bound the document set from a ``golden_manifest``.
 
     Applies ``include``/``exclude`` globs (relative to ``root``) and enforces
-    ``min_docs``/``max_docs``. Raises ``ValueError`` if the resulting count
-    violates the manifest bounds.
+    ``min_docs``/``max_docs``. Raises :class:`DocSelectionError` if a pattern
+    is unusable or the resulting count violates the manifest bounds.
     """
     root = Path(root)
     root_resolved = root.resolve()
@@ -109,7 +113,11 @@ def select_docs(root: Path, manifest: dict) -> list[Path]:
 
     selected: set[Path] = set()
     for pattern in include:
-        for p in root.glob(pattern):
+        try:
+            matches = list(root.glob(pattern))
+        except (ValueError, NotImplementedError) as exc:  # an empty or absolute pattern
+            raise DocSelectionError(f"golden_manifest include pattern {pattern!r}: {exc}") from exc
+        for p in matches:
             # Reject matches that escape root (e.g. via ".." in a pattern).
             if p.is_file() and p.resolve().is_relative_to(root_resolved):
                 selected.add(p)
@@ -123,12 +131,12 @@ def select_docs(root: Path, manifest: dict) -> list[Path]:
     min_docs = manifest.get("min_docs")
     max_docs = manifest.get("max_docs")
     if min_docs is not None and n < min_docs:
-        raise ValueError(
+        raise DocSelectionError(
             f"golden_manifest violation: matched {n} document(s) under {root}, "
             f"fewer than min_docs={min_docs}. include={include} exclude={exclude}"
         )
     if max_docs is not None and n > max_docs:
-        raise ValueError(
+        raise DocSelectionError(
             f"golden_manifest violation: matched {n} document(s) under {root}, "
             f"more than max_docs={max_docs}. include={include} exclude={exclude}"
         )
diff --git a/dokimi/tests/test_run_compare_errors.py b/dokimi/tests/test_run_compare_errors.py
index 2f5dcb233..a09b1f314 100644
--- a/dokimi/tests/test_run_compare_errors.py
+++ b/dokimi/tests/test_run_compare_errors.py
@@ -40,7 +40,7 @@ def test_run_exits_2_on_a_bad_spec(tmp_path, monkeypatch, capsys, content):
     assert code == 2
     err = capsys.readouterr().err
     assert err.startswith("Error: ") and str(spec) in err
-    assert not (tmp_path / "o.json").exists()
+    assert not (tmp_path / "o.json").exists() and not (tmp_path / "o.md").exists()
 
 
 @pytest.mark.parametrize("which", ["--base", "--candidate"])
@@ -118,6 +118,34 @@ def test_run_exits_2_when_the_root_has_fewer_docs_than_min_docs(tmp_path, monkey
     assert err.startswith("Error: ") and "min_docs" in err and "Traceback" not in err
 
 
+@pytest.mark.parametrize("pattern", ["", "/etc/*"], ids=["empty", "absolute"])
+def test_run_exits_2_on_an_unusable_include_pattern(tmp_path, monkeypatch, capsys, pattern):
+    spec = tmp_path / "spec.yaml"
+    spec.write_text(_SPEC_MIN_DOCS.replace("'docs/**/*.md'", repr(pattern)), encoding="utf-8")
+    code = _main(monkeypatch, "run", "--root", str(tmp_path), "--spec", str(spec),
+                 "--out-json", str(tmp_path / "o.json"), "--out-md", str(tmp_path / "o.md"))
+    assert code == 2
+    err = capsys.readouterr().err
+    assert err.startswith("Error: ") and f"include pattern {pattern!r}" in err
+
+
+def test_run_does_not_report_a_metric_error_as_bad_input(tmp_path, monkeypatch):
+    from dokimi.runner import run
+
+    def boom(text):
+        raise ValueError("a defect, not an input")
+
+    monkeypatch.setattr(run.coverage, "score", boom)
+    (tmp_path / "docs").mkdir()
+    (tmp_path / "docs" / "a.md").write_text("# A\n\ntext\n", encoding="utf-8")
+    spec = tmp_path / "spec.yaml"
+    spec.write_text(_SPEC_MIN_DOCS.replace("min_docs: 3", "min_docs: 1"), encoding="utf-8")
+    monkeypatch.setattr("sys.argv", ["dokimi", "run", "--root", str(tmp_path), "--spec", str(spec),
+                                     "--out-json", str(tmp_path / "o.json"), "--out-md", str(tmp_path / "o.md")])
+    with pytest.raises(ValueError, match="a defect, not an input"):
+        cli.main()
+
+
 @pytest.mark.parametrize("which", ["--base", "--candidate"])
 @pytest.mark.parametrize("field,value", [
     ("docs", "1"), ("score", None), ("coverage", True), ("score", float("nan")), ("score", float("inf")),
@@ -136,20 +164,39 @@ def test_compare_exits_2_on_a_non_numeric_summary_value(tmp_path, monkeypatch, c
     assert err.startswith("Error: ") and f"Invalid {role} snapshot {bad}" in err and f"`{field}`" in err
 
 
+def test_compare_does_not_report_a_regression_check_error_as_bad_input(tmp_path, monkeypatch):
+    from dokimi.runner import compare
+
+    def boom(diff, spec):
+        raise ValueError("a defect, not an input")
+
+    monkeypatch.setattr(compare, "detect_regressions", boom)
+    good = _snapshot(tmp_path / "good.json")
+    spec = tmp_path / "spec.yaml"
+    spec.write_text("thresholds: {}\n", encoding="utf-8")
+    monkeypatch.setattr("sys.argv", ["dokimi", "compare", "--base", good, "--candidate", good, "--spec", str(spec),
+                                     "--out-json", str(tmp_path / "c.json"), "--out-md", str(tmp_path / "c.md")])
+    with pytest.raises(ValueError, match="a defect, not an input"):
+        cli.main()
+
+
 def _blocked(tmp_path):
     blocker = tmp_path / "blocker"
     blocker.write_text("x", encoding="utf-8")  # a regular file: nothing can be created below it
     return blocker
 
 
-def test_compare_exits_1_when_an_output_cannot_be_written(tmp_path, monkeypatch, capsys):
+@pytest.mark.parametrize("which", ["json", "md"])
+def test_compare_exits_1_when_an_output_cannot_be_written(tmp_path, monkeypatch, capsys, which):
     good = _snapshot(tmp_path / "good.json")
-    target = _blocked(tmp_path) / "c.json"
+    blocked = _blocked(tmp_path) / "out"
+    outs = {"json": str(tmp_path / "c.json"), "md": str(tmp_path / "c.md")}
+    outs[which] = str(blocked)
     code = _main(monkeypatch, "compare", "--base", good, "--candidate", good,
-                 "--out-json", str(target), "--out-md", str(tmp_path / "c.md"))
+                 "--out-json", outs["json"], "--out-md", outs["md"])
     assert code == 1
     err = capsys.readouterr().err
-    assert f"Error: cannot write {target}" in err and "Traceback" not in err
+    assert f"Error: cannot write {blocked}" in err and "Traceback" not in err
 
 
 @pytest.mark.parametrize("which", ["json", "md"])

From fcf3b10481c736b5e47762113463204d5b8fabef Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Thu, 29 Oct 2026 11:05:17 -0500
Subject: [PATCH 44/49] fix(repo): note a non-integer num_turns in
 agent.trace_errors

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/repos/events.py        | 9 +++++++--
 dokimi/tests/test_repo_reach.py      | 4 +++-
 dokimi/tests/test_repo_skill_arms.py | 1 +
 3 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/dokimi/dokimi/repos/events.py b/dokimi/dokimi/repos/events.py
index 8c19bdf65..f6c59f0fd 100644
--- a/dokimi/dokimi/repos/events.py
+++ b/dokimi/dokimi/repos/events.py
@@ -138,6 +138,12 @@ def _apply_result(block: dict[str, Any], by_id: dict[str, dict[str, Any]]) -> No
             result_of["output"], result_of["output_truncated"] = clip_output(output)
 
 
+def _set_num_turns(out: Transcript, turns: Any) -> None:
+    out.num_turns = turns if isinstance(turns, int) and not isinstance(turns, bool) else None
+    if out.num_turns is None and turns is not None:
+        out.errors.append(f"result num_turns {str(turns)[:50]!r} is not an integer; recorded as null")
+
+
 def _apply(out: Transcript, rec: dict[str, Any], t: float | None,
            by_id: dict[str, dict[str, Any]], workdir: str | None) -> None:
     kind = rec.get("type")
@@ -170,8 +176,7 @@ def _apply(out: Transcript, rec: dict[str, Any], t: float | None,
         out.answer = str(rec.get("result") or "").strip()
         out.cost_usd = rec.get("total_cost_usd")
         out.result_seen = True
-        turns = rec.get("num_turns")
-        out.num_turns = turns if isinstance(turns, int) and not isinstance(turns, bool) else None
+        _set_num_turns(out, rec.get("num_turns"))
         out.is_error = bool(rec.get("is_error"))
         out.result_subtype = rec.get("subtype")
         out.subagent_calls = out.subagent_calls or 0
diff --git a/dokimi/tests/test_repo_reach.py b/dokimi/tests/test_repo_reach.py
index eb7d95670..8c09fab39 100644
--- a/dokimi/tests/test_repo_reach.py
+++ b/dokimi/tests/test_repo_reach.py
@@ -963,4 +963,6 @@ def test_reach_resalts_when_a_guidance_file_already_holds_its_token(tmp_path, mo
 @pytest.mark.parametrize("turns, expected", [(3, 3), ("3", None), (True, None), (2.5, None), (None, None)])
 def test_num_turns_is_an_int_or_none(turns, expected):
     raw = json.dumps({"type": "result", "result": "ok", "num_turns": turns})
-    assert convert("claude-stream-json", raw).num_turns == expected
+    got = convert("claude-stream-json", raw)
+    assert got.num_turns == expected
+    assert any("num_turns" in e for e in got.errors) == (expected is None and turns is not None)
diff --git a/dokimi/tests/test_repo_skill_arms.py b/dokimi/tests/test_repo_skill_arms.py
index e0ea32d76..6825dc472 100644
--- a/dokimi/tests/test_repo_skill_arms.py
+++ b/dokimi/tests/test_repo_skill_arms.py
@@ -133,6 +133,7 @@ def test_a_result_with_a_non_integer_num_turns_and_no_tool_events_still_measures
     rd = next((setup["out"] / "b1").glob("fix_add__*"))
     result = json.loads((rd / "result.json").read_text())
     assert result["agent"]["num_turns"] is None
+    assert any("num_turns '3' is not an integer" in e for e in result["agent"]["trace_errors"])
     assert result["skills"]["measured"] is True and result["skills"]["skills_used"] == []
     assert result["process"] is not None  # its tool calls are counted (zero), not left unmeasured
 

From c6ee9c354ca27ab8b9971175c25264d55f4bbcbd Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Thu, 29 Oct 2026 11:05:18 -0500
Subject: [PATCH 45/49] test(repo): compare the batch path in a
 platform-independent form

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/tests/test_repo_run.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dokimi/tests/test_repo_run.py b/dokimi/tests/test_repo_run.py
index 4179ac835..bc1eb25ad 100644
--- a/dokimi/tests/test_repo_run.py
+++ b/dokimi/tests/test_repo_run.py
@@ -911,7 +911,7 @@ def test_a_batch_whose_pin_is_unreadable_exits_1_naming_the_file(setup, capsys,
         _run(setup, "--arm", "guidance:none", "--task", "fix_add")
     assert exc.value.code == 1
     err = capsys.readouterr().err
-    assert err.startswith("✗ ") and "_eval/batch.json" in err and "use another --batch" in err
+    assert err.startswith("✗ ") and str(Path("_eval") / "batch.json") in err and "use another --batch" in err
 
 
 def test_a_batch_whose_pin_is_a_directory_exits_1_naming_the_file(setup, capsys):
@@ -924,7 +924,7 @@ def test_a_batch_whose_pin_is_a_directory_exits_1_naming_the_file(setup, capsys)
         _run(setup, "--arm", "guidance:none", "--task", "fix_add")
     assert exc.value.code == 1
     err = capsys.readouterr().err
-    assert err.startswith("✗ ") and "_eval/batch.json" in err and "use another --batch" in err
+    assert err.startswith("✗ ") and str(Path("_eval") / "batch.json") in err and "use another --batch" in err
 
 
 def test_graders_digest_follows_the_grader_sources(tmp_path, monkeypatch):

From dd7d4e667c5c6d087f719072a7dbcc482936ff53 Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Thu, 29 Oct 2026 11:05:18 -0500
Subject: [PATCH 46/49] docs(changelog): list the fixed bad-input cases

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 .../20261025_120000_vchamarthi_cli-input-run-arms-repo.md    | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-run-arms-repo.md b/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-run-arms-repo.md
index 62a052465..bef0a5418 100644
--- a/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-run-arms-repo.md
+++ b/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-run-arms-repo.md
@@ -1,3 +1,6 @@
 ### Fixed
 
-- **`run`, `compare`, `arms`, `tasks`, `explore` and `repo`** exit with a message on the unreadable, non-UTF-8 or corrupt spec, snapshot, result, plan, task and question files their tests cover, or an unwritable output, instead of a traceback. This covers a deeply nested `--manifest-file`, `--static` or `--harness` file, a non-UTF-8 `.agent-eval/repo-eval.yaml`, and a `tasks run --tasks` name that is missing or unreadable. `run` and `compare` exit 2 on a bad spec or snapshot (a non-numeric summary value, or fewer documents than the manifest's `min_docs`) and exit 1 when an output cannot be written, as `arms` does. `repo report` and `repo compare` exit 1 on a result file without a usable `grade`, `agent`, `task`, `arm` or `model`. `tasks` error messages use the `Error:` prefix. `repo run` records `agent.num_turns` as null when the harness reports anything but an integer, so the batch stays reportable; a finished run with such a value still counts as started. A grader score that is NaN, infinite or beyond 1e150 is recorded as a grader failure, so `repo report` and `repo compare` still read the batch.
+- **`run`, `compare`, `arms`, `tasks`, `explore` and `repo`** exit with a message instead of a traceback on the unreadable, non-UTF-8 or corrupt spec, snapshot, result, plan, task and question files their tests cover, and on an unwritable output. This covers a deeply nested `--manifest-file`, `--static` or `--harness` file, an unreadable or non-UTF-8 `.agent-eval/repo-eval.yaml`, a `tasks run --tasks` name that matches no task, and an `explore` task file with an invalid YAML timestamp or a `repo_root` that is not a string.
+- `run` and `compare` exit 2 on a bad spec or snapshot (a non-numeric summary value, an empty or absolute `golden_manifest` include pattern, or fewer documents than its `min_docs`) and exit 1 when an output cannot be written, as `arms` does. An error inside a metric is no longer reported as bad input. `tasks` error messages use the `Error:` prefix.
+- `repo report` and `repo compare` exit 1 on a result file without a usable `grade`, `agent`, `task`, `arm` or `model`, naming the file. A grader score that is NaN, infinite or beyond 1e150 is recorded as a grader failure, and a batch that already stores one is read the same way.
+- `repo run` records `agent.num_turns` as null when the harness reports anything but an integer, and notes it in `agent.trace_errors`; a finished run with such a value still counts as started.

From 13dd4201bce2059eb60a9e1e8b7397334c4bc55e Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Thu, 29 Oct 2026 12:15:22 -0500
Subject: [PATCH 47/49] fix(repo): leave a process counter or grade metric
 beyond the bound out of the means

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 ...0000_vchamarthi_cli-input-run-arms-repo.md |  4 +-
 dokimi/dokimi/repos/compare.py                | 11 +++--
 dokimi/dokimi/repos/process_counts.py         | 21 ++++++----
 .../tests/test_repo_report_unreadable_cell.py | 41 +++++++++++++++++++
 4 files changed, 62 insertions(+), 15 deletions(-)

diff --git a/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-run-arms-repo.md b/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-run-arms-repo.md
index bef0a5418..3d0d9e2d3 100644
--- a/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-run-arms-repo.md
+++ b/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-run-arms-repo.md
@@ -1,6 +1,6 @@
 ### Fixed
 
-- **`run`, `compare`, `arms`, `tasks`, `explore` and `repo`** exit with a message instead of a traceback on the unreadable, non-UTF-8 or corrupt spec, snapshot, result, plan, task and question files their tests cover, and on an unwritable output. This covers a deeply nested `--manifest-file`, `--static` or `--harness` file, an unreadable or non-UTF-8 `.agent-eval/repo-eval.yaml`, a `tasks run --tasks` name that matches no task, and an `explore` task file with an invalid YAML timestamp or a `repo_root` that is not a string.
+- **`run`, `compare`, `arms`, `tasks`, `explore` and `repo`** exit with a message instead of a traceback on the unreadable, non-UTF-8 or corrupt spec, snapshot, result, plan, task and question files their tests cover, and on an unwritable output. This covers a deeply nested `--manifest-file`, `--static` or `--harness` file, an unreadable or non-UTF-8 `.agent-eval/repo-eval.yaml`, a `tasks run --tasks` name that matches no task, and an `explore` task file with an invalid YAML timestamp, a bad `lines` range or a `repo_root` that is not a string.
 - `run` and `compare` exit 2 on a bad spec or snapshot (a non-numeric summary value, an empty or absolute `golden_manifest` include pattern, or fewer documents than its `min_docs`) and exit 1 when an output cannot be written, as `arms` does. An error inside a metric is no longer reported as bad input. `tasks` error messages use the `Error:` prefix.
-- `repo report` and `repo compare` exit 1 on a result file without a usable `grade`, `agent`, `task`, `arm` or `model`, naming the file. A grader score that is NaN, infinite or beyond 1e150 is recorded as a grader failure, and a batch that already stores one is read the same way.
+- `repo report` and `repo compare` exit 1 on a result file without a usable `grade`, `agent`, `task`, `arm` or `model`, naming the file. A grader score that is NaN, infinite or beyond 1e150 is recorded as a grader failure, and a batch that already stores one is read the same way. A `process` counter or `grade.json` metric that is not finite or beyond 1e150 is left out of the means, as a non-number is, instead of ending in an overflow traceback.
 - `repo run` records `agent.num_turns` as null when the harness reports anything but an integer, and notes it in `agent.trace_errors`; a finished run with such a value still counts as started.
diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py
index e30fe09df..5307cf0be 100644
--- a/dokimi/dokimi/repos/compare.py
+++ b/dokimi/dokimi/repos/compare.py
@@ -77,7 +77,6 @@
 
 import itertools
 import json
-import math
 import random
 from collections import defaultdict
 from pathlib import Path
@@ -86,7 +85,7 @@
 from dokimi.context.consistency import Run, order_distance, pass_hat, run_from_events
 from dokimi.eval.stats import adjust_pvalues, bootstrap_ci, mean_ci
 
-from .process_counts import process_order, process_values
+from .process_counts import as_measure, process_order, process_values
 from .report import (load_cells, mismatch_warnings, model_matches, model_mismatch, reading_cells, resolved_models,
                      unmeasured_note, uptake_counts)
 
@@ -655,7 +654,7 @@ def _grade_metrics(path: Path, names: tuple[str, ...]) -> dict[str, float]:
     """The requested numeric or boolean ``metrics`` of a cell's ``grade.json``; absent ones are left out."""
     try:
         grade = json.loads(path.read_text(encoding="utf-8"))
-    except (OSError, ValueError):
+    except (OSError, ValueError, RecursionError):
         return {}
     found = grade.get("metrics") if isinstance(grade, dict) else None
     if not isinstance(found, dict):
@@ -665,13 +664,13 @@ def _grade_metrics(path: Path, names: tuple[str, ...]) -> dict[str, float]:
         value = found.get(name)
         if isinstance(value, bool):
             out[name] = 1.0 if value else 0.0
-        elif isinstance(value, (int, float)) and math.isfinite(value):  # NaN and inf are not measurements
-            out[name] = float(value)
+        elif (num := as_measure(value)) is not None:  # NaN, inf and a huge number are not measurements
+            out[name] = num
     return out
 
 
 def _finite(values: list[Any]) -> list[float]:
-    return [float(v) for v in values if isinstance(v, (int, float)) and not isinstance(v, bool) and math.isfinite(v)]
+    return [num for v in values if (num := as_measure(v)) is not None]
 
 
 def _process_lists(recs: list[dict[str, Any]]) -> dict[str, list[float | None]]:
diff --git a/dokimi/dokimi/repos/process_counts.py b/dokimi/dokimi/repos/process_counts.py
index 9fc7be959..e6e070f09 100644
--- a/dokimi/dokimi/repos/process_counts.py
+++ b/dokimi/dokimi/repos/process_counts.py
@@ -8,7 +8,7 @@
 each manifest ``process_metrics`` counter. A cell whose ``process`` is absent
 (written before it existed) or ``null`` (its trace could not be read) has none
 and is left out of a mean, not counted as zero; so is a value that is not a
-finite number.
+finite number within ``SCORE_BOUND``.
 """
 
 from __future__ import annotations
@@ -17,6 +17,8 @@
 from collections import defaultdict
 from typing import Any
 
+from .grading import SCORE_BOUND
+
 
 #: ``result.json`` ``process`` keys that are counts (``guidance_read`` is a list of paths, counted).
 PROCESS_KEYS = ("tool_calls", "commands", "failed_commands", "denied_commands", "guidance_read")
@@ -31,11 +33,11 @@ def process_values(rec: dict[str, Any]) -> dict[str, float]:
     for key in PROCESS_KEYS:
         value = found.get(key)
         count = len(value) if key == "guidance_read" and isinstance(value, list) else value
-        if (num := _count(count)) is not None:
+        if (num := as_measure(count)) is not None:
             out[f"process:{key}"] = num
     counters = found.get("counters")
     for name, value in (counters.items() if isinstance(counters, dict) else ()):
-        if (num := _count(value)) is not None:
+        if (num := as_measure(value)) is not None:
             out[f"counter:{name}"] = num
     return out
 
@@ -55,10 +57,15 @@ def process_order(key: str) -> tuple[int, str]:
     return (PROCESS_KEYS.index(name), "") if name in PROCESS_KEYS else (len(PROCESS_KEYS), key)
 
 
-def _count(value: Any) -> float | None:
-    """A finite number as a float; ``None`` for anything else (a bool, a string, NaN)."""
-    ok = isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value)
-    return float(value) if ok else None
+def as_measure(value: Any) -> float | None:
+    """A finite number within ``SCORE_BOUND`` as a float; ``None`` for anything else (a bool, a string, NaN, 1e200)."""
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        return None
+    try:
+        num = float(value)
+    except OverflowError:  # an int too large for a float
+        return None
+    return num if math.isfinite(num) and abs(num) <= SCORE_BOUND else None
 
 
 def process_line(means: dict[str, float] | None, width: int) -> list[str]:
diff --git a/dokimi/tests/test_repo_report_unreadable_cell.py b/dokimi/tests/test_repo_report_unreadable_cell.py
index d55965193..c9924b78b 100644
--- a/dokimi/tests/test_repo_report_unreadable_cell.py
+++ b/dokimi/tests/test_repo_report_unreadable_cell.py
@@ -384,3 +384,44 @@ def test_a_stored_unmeasurable_score_is_read_as_a_grader_failure(tmp_path, capsy
     else:
         row = next(r for r in out["comparisons"] if r["measure"] == "score")
         assert (row["baseline_mean"], row["treatment_mean"]) == (0.0, 0.5)
+
+
+
+def _huge_in_t0(huge, files_for):
+    """Three tasks, arms a and b; arm a's value in task t0 is ``huge``, every other value is the task index."""
+    files = {}
+    for i in range(3):
+        for arm in ("a", "b"):
+            value = "HUGE" if (i, arm) == (0, "a") else i
+            for rel, rec in files_for(f"t{i}", arm, value).items():
+                files[f"t{i}__{arm}__m__r1/{rel}"] = json.dumps(rec).replace('"HUGE"', huge).encode()
+    return files
+
+
+_HUGE = pytest.mark.parametrize("huge", ["1e200", pytest.param("9" * 400, id="400-digits")])
+
+
+@_CMDS
+@_HUGE
+def test_a_process_counter_beyond_the_bound_is_left_out(tmp_path, capsys, cmd, huge):
+    files = _huge_in_t0(huge, lambda task, arm, value: {"result.json": {
+        **_GOOD, "task": task, "arm": arm, "process": {"counters": {"x": value}}}})
+    _run(cmd[0], str(_batch(tmp_path, "b1", files)), *cmd[1:], "--json")
+    out = json.loads(capsys.readouterr().out, parse_constant=_no_constant)
+    if cmd[0] == "report":
+        rows = {r["task"]: r for r in out["rows"] if r["arm"] == "a"}
+        assert "counter:x" not in rows["t0"].get("process", {}) and rows["t1"]["process"]["counter:x"] == 1.0
+    else:
+        row = next(r for r in out["comparisons"] if r["measure"] == "counter:x")
+        assert [t["task"] for t in row["tasks"]] == ["t1", "t2"]
+
+
+@_HUGE
+def test_a_grade_metric_beyond_the_bound_is_left_out(tmp_path, capsys, huge):
+    files = _huge_in_t0(huge, lambda task, arm, value: {"result.json": {**_GOOD, "task": task, "arm": arm},
+                                                         "grade.json": {"metrics": {"m1": value}}})
+    files["t1__b__m__r1/grade.json"] = b"[" * 100000  # a grade.json too deep to read has no metrics
+    _run("compare", str(_batch(tmp_path, "b1", files)), "--baseline", "a", "--metric", "m1", "--json")
+    out = json.loads(capsys.readouterr().out, parse_constant=_no_constant)
+    row = next(r for r in out["comparisons"] if r["measure"] == "metric:m1")
+    assert [t["task"] for t in row["tasks"]] == ["t2"]

From 4c6f15b1c12281dba5c5726b293557ed8ba1dec7 Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Thu, 29 Oct 2026 12:15:22 -0500
Subject: [PATCH 48/49] fix(explore): name the task file when a reference's
 lines range is invalid

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/explore/tasks.py          | 11 +++++++----
 dokimi/tests/test_explore_task_files.py |  6 ++++++
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/dokimi/dokimi/explore/tasks.py b/dokimi/dokimi/explore/tasks.py
index 7f1b030dc..b8e723333 100644
--- a/dokimi/dokimi/explore/tasks.py
+++ b/dokimi/dokimi/explore/tasks.py
@@ -131,10 +131,13 @@ def _task_from_data(
         isinstance(ref, dict) and "path" in ref for ref in raw_refs
     ):
         raise ValueError(f"Explore task {path} references must be a list of entries with a 'path'")
-    references = tuple(
-        ExploreReference(path=str(ref["path"]), lines=_parse_lines(ref.get("lines")))
-        for ref in raw_refs
-    )
+    try:
+        references = tuple(
+            ExploreReference(path=str(ref["path"]), lines=_parse_lines(ref.get("lines")))
+            for ref in raw_refs
+        )
+    except ValueError as exc:
+        raise ValueError(f"Explore task {path}: {exc}") from exc
     if not references:
         raise ValueError(f"Explore task {path} has no references")
     repo_root = data.get("repo_root")
diff --git a/dokimi/tests/test_explore_task_files.py b/dokimi/tests/test_explore_task_files.py
index 00a9e5ddc..2270c42db 100644
--- a/dokimi/tests/test_explore_task_files.py
+++ b/dokimi/tests/test_explore_task_files.py
@@ -37,6 +37,12 @@ def _exit_message(argv) -> str:
         pytest.param(b"id: x\nquery: q\nreferences:\n  - path: a.py\nrepo_root: []\n", "repo_root must be",
                      id="empty-list-root"),
         pytest.param(b"id: x\ncreated: 2020-13-45\n", "is not valid UTF-8 YAML", id="bad-timestamp"),
+        pytest.param(b"id: x\nquery: q\nreferences:\n  - path: a.py\n    lines: [0]\n", "values must be >= 1",
+                     id="lines-zero"),
+        pytest.param(b"id: x\nquery: q\nreferences:\n  - path: a.py\n    lines: [1, 2, 3]\n",
+                     "must be [start, end] or [n]", id="lines-three"),
+        pytest.param(b"id: x\nquery: q\nreferences:\n  - path: a.py\n    lines: [x]\n", "must contain integers",
+                     id="lines-not-int"),
     ],
 )
 def test_a_malformed_task_file_names_the_file(tmp_path, argv_tail, content, message):

From 9e8067f7cbf08bcb0a0bc1b0c34072985d7b34d2 Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Thu, 29 Oct 2026 12:15:23 -0500
Subject: [PATCH 49/49] fix(tasks): prefix the harness validation exit with
 Error: and test a RecursionError from a task load

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/commands/tasks.py    |  2 +-
 dokimi/tests/test_tasks_command.py | 33 ++++++++++++++++++++++++++++++
 2 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/dokimi/dokimi/commands/tasks.py b/dokimi/dokimi/commands/tasks.py
index cd95e292c..0559f1ba6 100644
--- a/dokimi/dokimi/commands/tasks.py
+++ b/dokimi/dokimi/commands/tasks.py
@@ -218,7 +218,7 @@ def cmd_tasks_run(args: argparse.Namespace) -> None:
     try:
         runner.validate_harnesses()
     except ValueError as exc:
-        raise SystemExit(str(exc)) from None
+        raise SystemExit(f"Error: {exc}") from None
     try:
         output = runner.run()
     except PreflightError as exc:
diff --git a/dokimi/tests/test_tasks_command.py b/dokimi/tests/test_tasks_command.py
index 2b61f390e..9145951b4 100644
--- a/dokimi/tests/test_tasks_command.py
+++ b/dokimi/tests/test_tasks_command.py
@@ -223,6 +223,39 @@ def test_tasks_run_with_a_missing_task_exits_with_a_message(tmp_path):
     assert str(excinfo.value).startswith("Error:") and "no-such-task" in str(excinfo.value)
 
 
+@pytest.mark.parametrize("error", [RecursionError("maximum recursion depth exceeded"),
+                                   PermissionError(13, "Permission denied")], ids=lambda e: type(e).__name__)
+def test_tasks_run_with_an_unloadable_task_exits_with_a_message(tmp_path, monkeypatch, error):
+    import dokimi.harnesses
+
+    def unloadable(name, root=None):
+        raise error
+
+    monkeypatch.setattr(dokimi.harnesses, "load_task", unloadable)
+    args = build_parser().parse_args([
+        "tasks", "run", "--tasks", "t", "--tasks-root", str(tmp_path), "--harnesses", "codex", "--model", "m",
+        "--dry-run",
+    ])
+    with pytest.raises(SystemExit) as excinfo:
+        args.func(args)
+    assert str(excinfo.value) == f"Error: {error}"
+
+
+def test_tasks_run_with_an_unusable_harness_exits_with_an_error_prefix(tmp_path, monkeypatch):
+    from dokimi.harnesses import TaskSuiteRunner
+
+    def invalid(self):
+        raise ValueError("unknown harness: nope")
+
+    monkeypatch.setattr(TaskSuiteRunner, "validate_harnesses", invalid)
+    monkeypatch.setattr("dokimi.commands.tasks._select_tasks", lambda args: ["t"])
+    args = build_parser().parse_args(["tasks", "run", "--all", "--tasks-root", str(tmp_path), "--harnesses", "nope",
+                                      "--model", "m", "--dry-run"])
+    with pytest.raises(SystemExit) as excinfo:
+        args.func(args)
+    assert str(excinfo.value) == "Error: unknown harness: nope"
+
+
 def test_tasks_inspect_of_a_missing_path_exits_with_an_error_prefix(tmp_path):
     args = build_parser().parse_args(["tasks", "inspect", str(tmp_path / "nope")])
     with pytest.raises(SystemExit) as excinfo: