From 3eac00ceab075830a98d345ac18453bfde371389 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 18 Oct 2026 22:06:39 -0700 Subject: [PATCH 01/20] feat(repo): compare flags comparisons with no headroom A zero delta on a task set where both arms pass (or fail) every paired task reads as "no effect" but means the tasks had no room to move. Gate and score comparisons now carry headroom: ceiling, floor, near_ceiling (baseline at the top on all but at most one of 2+ tasks) or null, with a note in the text report. p values and Holm are unchanged. result.json records the task's score_max so score rows know their top. Signed-off-by: Nikolay Petrov --- ...0000_napetrov_feat-repo-compare-ceiling.md | 3 ++ dokimi/docs/repo-evaluation-contract.md | 6 ++- dokimi/dokimi/repos/compare.py | 45 +++++++++++++++- dokimi/dokimi/repos/runner.py | 3 +- dokimi/tests/test_repo_compare.py | 53 ++++++++++++++++++- dokimi/tests/test_repo_run.py | 1 + 6 files changed, 107 insertions(+), 4 deletions(-) create mode 100644 dokimi/changelog.d/20261018_120000_napetrov_feat-repo-compare-ceiling.md diff --git a/dokimi/changelog.d/20261018_120000_napetrov_feat-repo-compare-ceiling.md b/dokimi/changelog.d/20261018_120000_napetrov_feat-repo-compare-ceiling.md new file mode 100644 index 000000000..38d852c5b --- /dev/null +++ b/dokimi/changelog.d/20261018_120000_napetrov_feat-repo-compare-ceiling.md @@ -0,0 +1,3 @@ +### Added + +- **`repo compare` flags gate and score comparisons with no headroom** — each comparison gains `headroom`: `"ceiling"` when both arms are at the top (gate 1.0, the task's `score.max`) on every paired task, `"floor"` when both are at 0 on every one, `"near_ceiling"` when the baseline is at the top on all but at most one of 2+ paired tasks, else `null`; the text report prints a note under the comparison. A zero delta on such a task set is no headroom, not no effect. p values and Holm's correction are unchanged. `result.json` now records the task's `score_max`; score rows of batches run before this have no known top and are never flagged `ceiling`. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index f3fbc3989..bf63236c9 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1009,7 +1009,11 @@ python cli.py repo compare repo-runs/pilot --baseline guidance:none --metric roo task), the tasks up, down and tied, and a two-sided sign-flip permutation p (exact up to 16 changed tasks; at one cell per task and arm it equals McNemar's exact test). Holm's correction runs over the gate and score comparisons of one call; the - rest are descriptive. `BATCH/ARM` takes an arm from one batch only, so two + rest are descriptive. A gate or score comparison carries `headroom`: + `ceiling` (both arms at 1.0 or the task's `score.max` on every paired task), + `floor` (both at 0 on every one), `near_ceiling` (the baseline at the top on + all but at most one of 2+ tasks), else `null`, with a note in the text; a zero + delta there means no room to move, not no effect. `BATCH/ARM` takes an arm from one batch only, so two batches of the same arm (for example two revisions of the guidance) compare too; two selectors of the same cells are refused. A `--metric` no finished cell reports as a number or boolean is refused. Tasks left out are listed as diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index 48ed1f61f..2a600e607 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -28,6 +28,14 @@ the gate and score comparisons of one invocation; cost, turns and metrics are descriptive and keep their raw p. +A gate or score comparison also says whether its tasks had room to move (``headroom``), since a zero delta +on a task set nobody can fail, or nobody can pass, is not evidence of no effect. A task's top is 1.0 for +gate and the task's ``score.max`` for score (recorded in ``result.json``; a batch without it has no known +top, so its score rows are never at ceiling). ``ceiling``: both arms at the top on every paired task; +``floor``: both arms at 0 on every paired task; ``near_ceiling``: neither, at least 2 paired tasks, and the +baseline at the top on all but at most one, so at most one task can improve and a gain resting on one +task has a sign-flip p of 1. Otherwise ``None``. The flag is a note; p and Holm are unchanged. + Each selector's batches are reported with the stamp ``repo run`` pins a batch to (``tree``, ``eval_dir`` digest, ``image``) and the harness versions of its cells. A warning is printed, not an error, when a bare ``ARM`` pools cells of @@ -290,8 +298,28 @@ def _compare_model(treatment: str, model: str, base_vals: dict[tuple[str, str], else: pairs.append(pair) if pairs: + tops = [(base_vals[(model, p["task"])]["top"], treat_vals[(model, p["task"])]["top"]) for p in pairs] rows.append({"treatment": treatment, "model": model, "measure": measure, - **paired_summary(pairs, measure, seed=seed)}) + **paired_summary(pairs, measure, seed=seed), + "headroom": headroom(pairs, tops) if measure in PRIMARY else None}) + + +def headroom(pairs: list[dict[str, Any]], tops: list[tuple[float | None, float | None]]) -> str | None: + """``ceiling``, ``floor``, ``near_ceiling`` or ``None`` for one gate or score comparison (module docstring). + + ``tops`` holds each pair's (baseline, treatment) top value, ``None`` where it is not known. + """ + def at_top(value: float, top: float | None) -> bool: + return top is not None and value >= top - EPS + + base_top = [at_top(p["baseline"], b) for p, (b, _) in zip(pairs, tops, strict=True)] + if all(base_top) and all(at_top(p["treatment"], t) for p, (_, t) in zip(pairs, tops, strict=True)): + return "ceiling" + if all(abs(p["baseline"]) <= EPS and abs(p["treatment"]) <= EPS for p in pairs): + return "floor" + if len(pairs) >= 2 and base_top.count(False) <= 1: + return "near_ceiling" + return None def _unpaired(treatment: str, model: str, task: str, measure: str | None, reason: str) -> dict[str, Any]: @@ -409,6 +437,8 @@ def format_compare(summary: dict[str, Any]) -> str: if moved: lines.append(f" {r['measure']} changed: " + ", ".join( f"{t['task']} {t['delta']:+.2f}" for t in sorted(moved, key=lambda t: t["delta"]))) + if r["headroom"]: + lines.append(f" {r['measure']} {_headroom_note(r)}") if summary["unpaired"]: shown = summary["unpaired"][:10] lines.append("unpaired: " + ", ".join( @@ -422,6 +452,17 @@ def format_compare(summary: dict[str, Any]) -> str: return "\n".join(lines) +def _headroom_note(row: dict[str, Any]) -> str: + n, gate = row["n_tasks"], row["measure"] == "gate" + if row["headroom"] == "ceiling": + return (f"ceiling: both arms {'pass' if gate else 'reach score.max on'} all {n} paired tasks; " + "no headroom to show an effect") + if row["headroom"] == "floor": + return f"floor: both arms {'fail' if gate else 'score 0 on'} all {n} paired tasks; no headroom to show an effect" + return (f"near ceiling: the baseline {'passes' if gate else 'reaches score.max on'} all but at most one of " + f"{n} paired tasks; too few can improve to show a gain") + + def _format_header(summary: dict[str, Any]) -> list[str]: lines = [f"baseline {summary['baseline']}: {summary['cells']} finished cells in " f"{len(summary['batches'])} batch(es), {len(summary['missing'])} missing"] @@ -450,7 +491,9 @@ def _task_values(cells: list[dict[str, Any]], metrics: tuple[str, ...]) -> dict[ else: found["gate"] = [float(bool(r["grade"]["success"])) for r in recs] nums = {measure: _finite(vals) for measure, vals in found.items()} + maxes = set(_finite([r.get("score_max") for r in recs])) # cells of two eval trees may differ out[key] = {"kind": "score" if score_task else "gate", + "top": (maxes.pop() if len(maxes) == 1 else None) if score_task else 1.0, "n": {measure: len(v) for measure, v in nums.items()}, "values": {measure: (sum(v) / len(v) if v else None) for measure, v in nums.items()}} return out diff --git a/dokimi/dokimi/repos/runner.py b/dokimi/dokimi/repos/runner.py index ad7750eae..1fb146853 100644 --- a/dokimi/dokimi/repos/runner.py +++ b/dokimi/dokimi/repos/runner.py @@ -226,7 +226,8 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, record: dict[str, Any] = {"task": task.id, "family": task.family, "arm": arm, "model": opts.model, "model_resolved": meta.get("model_resolved"), "rep": rep, "harness": opts.profile.name, "harness_version": meta["harness_version"], - "base_date": meta["base_date"], "score_task": bool(task.score), "agent": agent, "grade": grade} + "base_date": meta["base_date"], "score_task": bool(task.score), + "score_max": task.score["max"] if task.score else None, "agent": agent, "grade": grade} if any(uptake_of): # an arm that added something to use: did the agent use it (#41 D4) record["uptake"] = arm_uptake(events, *uptake_of, WORKSPACE) try: diff --git a/dokimi/tests/test_repo_compare.py b/dokimi/tests/test_repo_compare.py index a63b9c740..388000e1e 100644 --- a/dokimi/tests/test_repo_compare.py +++ b/dokimi/tests/test_repo_compare.py @@ -21,7 +21,8 @@ def _cell(batch: Path, task: str, arm: str, rep: int, *, success: bool | None = True, score: float | None = None, failure: bool = False, cost: float | None = 0.1, turns: int = 10, metrics: dict | None = None, - model: str = "m-1", harness_version: str = "1.0", model_resolved: str | list[str] | None = None) -> None: + model: str = "m-1", harness_version: str = "1.0", model_resolved: str | list[str] | None = None, + score_max: float | None = 1.0) -> None: rd = batch / f"{task}__{arm.replace(':', '-')}__{model}__r{rep}" rd.mkdir(parents=True) score_task = score is not None @@ -30,6 +31,7 @@ def _cell(batch: Path, task: str, arm: str, rep: int, *, success: bool | None = "score": None if failure else score, "score_only": score_task and not failure, "reasons": []} rec = {"task": task, "arm": arm, "model": model, "rep": rep, "score_task": score_task, "grade": grade, "harness_version": harness_version, "model_resolved": model_resolved, + "score_max": score_max if score_task else None, "agent": {"cost_usd": cost, "num_turns": turns, "timed_out": False, "is_error": False, "over_budget": False, "network": "none"}} (rd / "meta.json").write_text("{}") @@ -171,6 +173,55 @@ def test_score_tasks_are_their_own_measure_and_a_grader_failure_scores_zero(tmp_ assert _row(s, "gate")["n_tasks"] == 1 # the gated task, not the score task +@pytest.mark.parametrize("base,treat,expected", [ + ((True, True, True), (True, True, True), "ceiling"), + ((False, False, False), (False, False, False), "floor"), + ((True, True, False), (True, True, True), "near_ceiling"), # one task can move up + ((True, True, True), (True, False, True), "near_ceiling"), # none can move up; a loss still shows + ((True, False, False), (True, True, True), None), + ((False, True, False), (False, True, False), None), # all tied, but not all at one end +]) +def test_gate_headroom_flags_a_task_set_that_cannot_move(tmp_path, base, treat, expected): + b = tmp_path / "b1" + for i, (was, now) in enumerate(zip(base, treat, strict=True)): + _cell(b, f"fix_{i}", "guidance:none", 1, success=was) + _cell(b, f"fix_{i}", "guidance:raw", 1, success=now) + s = compare([b], "guidance:none") + gate = _row(s, "gate") + assert gate["headroom"] == expected and _row(s, "cost_usd")["headroom"] is None + text = format_compare(s) + if expected == "ceiling": + assert gate["p"] is None and gate["p_holm"] is None # the flag is a note: p and Holm are unchanged + assert "gate ceiling: both arms pass all 3 paired tasks; no headroom to show an effect" in text + elif expected == "floor": + assert "gate floor: both arms fail all 3 paired tasks" in text + elif expected == "near_ceiling": + assert "gate near ceiling: the baseline passes all but at most one of 3 paired tasks" in text + else: + assert "ceiling" not in text and "floor" not in text + + +def test_score_headroom_is_measured_against_score_max(tmp_path): + b = tmp_path / "b1" + for i in range(2): + _cell(b, f"review_{i}", "guidance:none", 1, score=4.0, score_max=4.0) + _cell(b, f"review_{i}", "guidance:raw", 1, score=4.0, score_max=4.0) + _cell(b, f"old_{i}", "guidance:none", 1, score=1.0, score_max=None) # a batch from before score_max + _cell(b, f"old_{i}", "guidance:raw", 1, score=1.0, score_max=None) + assert _row(compare([b], "guidance:none"), "score")["headroom"] is None # old_* has no known top + for i in range(2): + for arm in ("guidance:none", "guidance:raw"): + (b / f"old_{i}__{arm.replace(':', '-')}__m-1__r1/result.json").unlink() + s = compare([b], "guidance:none") + assert _row(s, "score")["headroom"] == "ceiling" + assert "score ceiling: both arms reach score.max on all 2 paired tasks" in format_compare(s) + c = tmp_path / "b2" + for i in range(2): + _cell(c, f"review_{i}", "guidance:none", 1, score=0.0) + _cell(c, f"review_{i}", "guidance:raw", 1, score=0.0, failure=True) # a grader failure scores 0 + assert _row(compare([c], "guidance:none"), "score")["headroom"] == "floor" + + def test_unpaired_tasks_are_listed_not_compared(tmp_path): b = tmp_path / "b1" _cell(b, "fix_a", "guidance:none", 1) diff --git a/dokimi/tests/test_repo_run.py b/dokimi/tests/test_repo_run.py index 43161e66e..08dd8f6ae 100644 --- a/dokimi/tests/test_repo_run.py +++ b/dokimi/tests/test_repo_run.py @@ -127,6 +127,7 @@ def test_score_only_task_and_missing_cells(setup, monkeypatch, capsys): batch = setup["out"] / "b1" review = json.loads((batch / "review_add__guidance-raw__m-1__r1/result.json").read_text()) assert review["grade"]["score_only"] and review["grade"]["score"] == 1.0 and review["grade"]["success"] is None + assert review["score_max"] == 1.0 # the task's score.max, for `repo compare`'s headroom (batch / "fix_add__guidance-raw__m-1__r3/result.json").unlink() # crashed before it finished shutil.rmtree(batch / "fix_add__guidance-raw__m-1__r2") # never started (only in the plan) summary = _report(setup) From bfdc199c69e8f613b8528b132a40058f662d725a Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 19 Oct 2026 01:00:11 -0700 Subject: [PATCH 02/20] fix(repos/compare): keep top unknown when pooling cells with mixed score_max When legacy cells (score_max=None) and current cells (score_max=1.0) appear for the same task, do not report ceiling if legacy maxima are unknown. Previously, _finite() would drop None values, leaving only {1.0}, which incorrectly triggered the ceiling headroom flag. Now check if any cell has an unknown max before setting top. Fixes the case where both arms average 1.0 but some cells have unknown score_max values. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/compare.py | 8 ++++++-- dokimi/tests/test_repo_compare.py | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index 2a600e607..6cfb0ff1e 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -491,9 +491,13 @@ def _task_values(cells: list[dict[str, Any]], metrics: tuple[str, ...]) -> dict[ else: found["gate"] = [float(bool(r["grade"]["success"])) for r in recs] nums = {measure: _finite(vals) for measure, vals in found.items()} - maxes = set(_finite([r.get("score_max") for r in recs])) # cells of two eval trees may differ + score_max_vals = [r.get("score_max") for r in recs] + maxes = set(_finite(score_max_vals)) # cells of two eval trees may differ + # If any cell has unknown max, or maxes differ, top is None; otherwise take the single value + has_unknown_max = any(v is None for v in score_max_vals) + top = (maxes.pop() if len(maxes) == 1 and not has_unknown_max else None) if score_task else 1.0 out[key] = {"kind": "score" if score_task else "gate", - "top": (maxes.pop() if len(maxes) == 1 else None) if score_task else 1.0, + "top": top, "n": {measure: len(v) for measure, v in nums.items()}, "values": {measure: (sum(v) / len(v) if v else None) for measure, v in nums.items()}} return out diff --git a/dokimi/tests/test_repo_compare.py b/dokimi/tests/test_repo_compare.py index 388000e1e..332b8dbb9 100644 --- a/dokimi/tests/test_repo_compare.py +++ b/dokimi/tests/test_repo_compare.py @@ -222,6 +222,21 @@ def test_score_headroom_is_measured_against_score_max(tmp_path): assert _row(compare([c], "guidance:none"), "score")["headroom"] == "floor" +def test_score_headroom_unknown_when_pooling_legacy_and_current_cells(tmp_path): + """Regression: when legacy cells (score_max=None) and current cells (score_max=1.0) appear for the same task, + do not report ceiling if the legacy cells' maxima are unknown.""" + b = tmp_path / "b1" + for i in range(2): + # Current cells with known score_max + _cell(b, f"task_{i}", "guidance:none", 1, score=1.0, score_max=1.0) + _cell(b, f"task_{i}", "guidance:raw", 1, score=1.0, score_max=1.0) + # Legacy cells from before score_max tracking, same task IDs + _cell(b, f"task_{i}", "guidance:none", 2, score=1.0, score_max=None) + _cell(b, f"task_{i}", "guidance:raw", 2, score=1.0, score_max=None) + # Both arms reach 1.0, but legacy cells have unknown maxima: headroom should be None, not "ceiling" + assert _row(compare([b], "guidance:none"), "score")["headroom"] is None + + def test_unpaired_tasks_are_listed_not_compared(tmp_path): b = tmp_path / "b1" _cell(b, "fix_a", "guidance:none", 1) From f8217da3ce546ce6318c1f39d5cab26ea1886705 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 19 Oct 2026 07:05:40 -0700 Subject: [PATCH 03/20] fix(repos/compare): headroom no longer reads a collapse, a grader failure or mixed score.max as no signal near_ceiling needs the treatment nowhere below the baseline, floor ignores pairs with a grader failure, and pairs whose tops differ have no headroom. Pin score_max in result.json to the task's score.max with a max-4 task, and rewrap and complete the contract bullet. Signed-off-by: Nikolay Petrov --- dokimi/docs/repo-evaluation-contract.md | 13 ++++++++---- dokimi/dokimi/repos/compare.py | 28 +++++++++++++++++-------- dokimi/tests/test_repo_compare.py | 25 ++++++++++++++++++++-- dokimi/tests/test_repo_run.py | 12 +++++++++++ 4 files changed, 63 insertions(+), 15 deletions(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index bf63236c9..2c8af1d82 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1010,10 +1010,15 @@ python cli.py repo compare repo-runs/pilot --baseline guidance:none --metric roo changed tasks; at one cell per task and arm it equals McNemar's exact test). Holm's correction runs over the gate and score comparisons of one call; the rest are descriptive. A gate or score comparison carries `headroom`: - `ceiling` (both arms at 1.0 or the task's `score.max` on every paired task), - `floor` (both at 0 on every one), `near_ceiling` (the baseline at the top on - all but at most one of 2+ tasks), else `null`, with a note in the text; a zero - delta there means no room to move, not no effect. `BATCH/ARM` takes an arm from one batch only, so two + `ceiling` (both arms at 1.0 or the task's `score.max` on every paired + task), `floor` (both at 0 on every one, none of the zeros a grader + failure), `near_ceiling` (the baseline at the top on all but at most one of + 2+ tasks and the treatment nowhere below it), else `null`, with a note in + the text; a zero delta there means no room to move, not no effect. The + top of a score task is the `score.max` that `repo run` records in + `result.json` as `score_max`: a batch run before that has no known top, so + its score rows are never `ceiling`, and arms whose tops differ are on two + scales and get `null`. `BATCH/ARM` takes an arm from one batch only, so two batches of the same arm (for example two revisions of the guidance) compare too; two selectors of the same cells are refused. A `--metric` no finished cell reports as a number or boolean is refused. Tasks left out are listed as diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index 6cfb0ff1e..e7df3f1d6 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -32,9 +32,11 @@ on a task set nobody can fail, or nobody can pass, is not evidence of no effect. A task's top is 1.0 for gate and the task's ``score.max`` for score (recorded in ``result.json``; a batch without it has no known top, so its score rows are never at ceiling). ``ceiling``: both arms at the top on every paired task; -``floor``: both arms at 0 on every paired task; ``near_ceiling``: neither, at least 2 paired tasks, and the -baseline at the top on all but at most one, so at most one task can improve and a gain resting on one -task has a sign-flip p of 1. Otherwise ``None``. The flag is a note; p and Holm are unchanged. +``floor``: both arms at 0 on every paired task, none of the zeros a grader failure (those say the +measurement failed, not that the tasks had no room); ``near_ceiling``: neither, at least 2 paired tasks, +the baseline at the top on all but at most one and the treatment nowhere below the baseline, so at most +one task can improve and a gain resting on one task has a sign-flip p of 1. A pair whose baseline and +treatment tops differ is on two scales: ``None``. Otherwise ``None``. The flag is a note; p and Holm are unchanged. Each selector's batches are reported with the stamp ``repo run`` pins a batch to (``tree``, ``eval_dir`` digest, ``image``) and the harness versions of its @@ -299,25 +301,32 @@ def _compare_model(treatment: str, model: str, base_vals: dict[tuple[str, str], pairs.append(pair) if pairs: tops = [(base_vals[(model, p["task"])]["top"], treat_vals[(model, p["task"])]["top"]) for p in pairs] + failed = [base_vals[(model, p["task"])]["grader_failed"] or treat_vals[(model, p["task"])]["grader_failed"] + for p in pairs] rows.append({"treatment": treatment, "model": model, "measure": measure, **paired_summary(pairs, measure, seed=seed), - "headroom": headroom(pairs, tops) if measure in PRIMARY else None}) + "headroom": headroom(pairs, tops, failed) if measure in PRIMARY else None}) -def headroom(pairs: list[dict[str, Any]], tops: list[tuple[float | None, float | None]]) -> str | None: +def headroom(pairs: list[dict[str, Any]], tops: list[tuple[float | None, float | None]], + failed: list[bool] | None = None) -> str | None: """``ceiling``, ``floor``, ``near_ceiling`` or ``None`` for one gate or score comparison (module docstring). - ``tops`` holds each pair's (baseline, treatment) top value, ``None`` where it is not known. + ``tops`` holds each pair's (baseline, treatment) top value, ``None`` where it is not known; ``failed`` + marks the pairs with a grader failure in either arm (its score of 0 is not a measurement). """ + if any(b != t for b, t in tops): + return None # two scales: neither "at the top" nor the delta compares def at_top(value: float, top: float | None) -> bool: return top is not None and value >= top - EPS base_top = [at_top(p["baseline"], b) for p, (b, _) in zip(pairs, tops, strict=True)] if all(base_top) and all(at_top(p["treatment"], t) for p, (_, t) in zip(pairs, tops, strict=True)): return "ceiling" - if all(abs(p["baseline"]) <= EPS and abs(p["treatment"]) <= EPS for p in pairs): + if (not any(failed or ()) and + all(abs(p["baseline"]) <= EPS and abs(p["treatment"]) <= EPS for p in pairs)): return "floor" - if len(pairs) >= 2 and base_top.count(False) <= 1: + if len(pairs) >= 2 and base_top.count(False) <= 1 and all(p["treatment"] >= p["baseline"] - EPS for p in pairs): return "near_ceiling" return None @@ -496,7 +505,8 @@ def _task_values(cells: list[dict[str, Any]], metrics: tuple[str, ...]) -> dict[ # If any cell has unknown max, or maxes differ, top is None; otherwise take the single value has_unknown_max = any(v is None for v in score_max_vals) top = (maxes.pop() if len(maxes) == 1 and not has_unknown_max else None) if score_task else 1.0 - out[key] = {"kind": "score" if score_task else "gate", + grader_failed = any(r["grade"]["status"] == "grader_failure" for r in recs) + out[key] = {"kind": "score" if score_task else "gate", "grader_failed": grader_failed, "top": top, "n": {measure: len(v) for measure, v in nums.items()}, "values": {measure: (sum(v) / len(v) if v else None) for measure, v in nums.items()}} diff --git a/dokimi/tests/test_repo_compare.py b/dokimi/tests/test_repo_compare.py index 332b8dbb9..06140a648 100644 --- a/dokimi/tests/test_repo_compare.py +++ b/dokimi/tests/test_repo_compare.py @@ -177,7 +177,8 @@ def test_score_tasks_are_their_own_measure_and_a_grader_failure_scores_zero(tmp_ ((True, True, True), (True, True, True), "ceiling"), ((False, False, False), (False, False, False), "floor"), ((True, True, False), (True, True, True), "near_ceiling"), # one task can move up - ((True, True, True), (True, False, True), "near_ceiling"), # none can move up; a loss still shows + ((True, True, True), (True, False, True), None), # a loss is an effect, not "too few can improve" + ((True, True, True), (False, False, False), None), # the treatment collapsed ((True, False, False), (True, True, True), None), ((False, True, False), (False, True, False), None), # all tied, but not all at one end ]) @@ -219,7 +220,27 @@ def test_score_headroom_is_measured_against_score_max(tmp_path): for i in range(2): _cell(c, f"review_{i}", "guidance:none", 1, score=0.0) _cell(c, f"review_{i}", "guidance:raw", 1, score=0.0, failure=True) # a grader failure scores 0 - assert _row(compare([c], "guidance:none"), "score")["headroom"] == "floor" + assert _row(compare([c], "guidance:none"), "score")["headroom"] is None # the zeros include a grader failure + d = tmp_path / "b3" + for i in range(2): + _cell(d, f"review_{i}", "guidance:none", 1, score=0.0) + _cell(d, f"review_{i}", "guidance:raw", 1, score=0.0) + assert _row(compare([d], "guidance:none"), "score")["headroom"] == "floor" + e = tmp_path / "b4" + for i in range(2): + _cell(e, f"review_{i}", "guidance:none", 1, score=0.0, failure=True) + _cell(e, f"review_{i}", "guidance:raw", 1, score=0.0, failure=True) + assert _row(compare([e], "guidance:none"), "score")["headroom"] is None + + +def test_score_headroom_is_unknown_when_the_arms_have_different_score_max(tmp_path): + b = tmp_path / "b1" + for i in range(2): + _cell(b, f"r{i}", "guidance:none", 1, score=4.0, score_max=4.0) + _cell(b, f"r{i}", "guidance:raw", 1, score=1.0, score_max=1.0) + s = compare([b], "guidance:none") + row = _row(s, "score") + assert row["delta"]["mean"] == -3.0 and row["headroom"] is None def test_score_headroom_unknown_when_pooling_legacy_and_current_cells(tmp_path): diff --git a/dokimi/tests/test_repo_run.py b/dokimi/tests/test_repo_run.py index 08dd8f6ae..d58284066 100644 --- a/dokimi/tests/test_repo_run.py +++ b/dokimi/tests/test_repo_run.py @@ -100,6 +100,18 @@ def test_a_run_answered_by_another_model_is_recorded_and_reported(setup, monkeyp assert summary["model_mismatch"] == {"m-1": ["m-0"]} +def test_result_json_records_the_tasks_own_score_max_and_none_for_a_gate_task(setup): + task_json = setup["root"] / ".agent-eval/tasks/review_add/task.json" + task = json.loads(task_json.read_text()) + task["score"]["max"] = 4.0 # not the 1.0 default of `repo compare` + task_json.write_text(json.dumps(task)) + _run(setup, "--arm", "guidance:raw", source=str(setup["root"])) + batch = setup["out"] / "b1" + review = json.loads((batch / "review_add__guidance-raw__m-1__r1/result.json").read_text()) + gate = json.loads((batch / "fix_add__guidance-raw__m-1__r1/result.json").read_text()) + assert review["score_max"] == 4.0 and gate["score_max"] is None + + @pytest.mark.outcome("cjm-repo-run-resume") def test_already_graded_cells_are_skipped_unless_forced(setup, capsys): _run(setup, "--arm", "guidance:raw", "--task", "fix_add") From 0532d9e05d93fb46e495011d51e5769cc8ea2b2a Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 19 Oct 2026 13:18:35 -0700 Subject: [PATCH 04/20] fix(repos/compare): grader failures never make the one movable task a near-ceiling signal Headroom now returns None when the one non-top baseline pair is a grader failure, and tops and failures are computed only for gate and score rows. Adds tests for failures on either arm, on one rep, and for mixed or unknown tops; fixes the changelog and docstring to match the rules. Signed-off-by: Nikolay Petrov --- ...0000_napetrov_feat-repo-compare-ceiling.md | 2 +- dokimi/dokimi/repos/compare.py | 30 +++++++---- dokimi/tests/test_repo_compare.py | 53 +++++++++++++++++++ 3 files changed, 73 insertions(+), 12 deletions(-) diff --git a/dokimi/changelog.d/20261018_120000_napetrov_feat-repo-compare-ceiling.md b/dokimi/changelog.d/20261018_120000_napetrov_feat-repo-compare-ceiling.md index 38d852c5b..1777bfdb0 100644 --- a/dokimi/changelog.d/20261018_120000_napetrov_feat-repo-compare-ceiling.md +++ b/dokimi/changelog.d/20261018_120000_napetrov_feat-repo-compare-ceiling.md @@ -1,3 +1,3 @@ ### Added -- **`repo compare` flags gate and score comparisons with no headroom** — each comparison gains `headroom`: `"ceiling"` when both arms are at the top (gate 1.0, the task's `score.max`) on every paired task, `"floor"` when both are at 0 on every one, `"near_ceiling"` when the baseline is at the top on all but at most one of 2+ paired tasks, else `null`; the text report prints a note under the comparison. A zero delta on such a task set is no headroom, not no effect. p values and Holm's correction are unchanged. `result.json` now records the task's `score_max`; score rows of batches run before this have no known top and are never flagged `ceiling`. +- **`repo compare` flags gate and score comparisons with no headroom** — each comparison gains `headroom`: `"ceiling"` when both arms are at the top (gate 1.0, the task's `score.max`) on every paired task, `"floor"` when both are at 0 on every one, none of those zeros a grader failure, `"near_ceiling"` when the baseline is at the top on all but at most one of 2+ paired tasks (that one not a grader failure) and the treatment is nowhere below the baseline, else `null`; arms whose tops differ (or are unknown) get `null`. The text report prints a note under the comparison. A zero delta on such a task set is no headroom, not no effect. p values and Holm's correction are unchanged. `result.json` now records the task's `score_max`; score rows of batches run before this have no known top and are never flagged `ceiling`. diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index e7df3f1d6..adfc6f05b 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -34,9 +34,10 @@ top, so its score rows are never at ceiling). ``ceiling``: both arms at the top on every paired task; ``floor``: both arms at 0 on every paired task, none of the zeros a grader failure (those say the measurement failed, not that the tasks had no room); ``near_ceiling``: neither, at least 2 paired tasks, -the baseline at the top on all but at most one and the treatment nowhere below the baseline, so at most -one task can improve and a gain resting on one task has a sign-flip p of 1. A pair whose baseline and -treatment tops differ is on two scales: ``None``. Otherwise ``None``. The flag is a note; p and Holm are unchanged. +the baseline at the top on all but at most one, that one not a grader failure, and the treatment nowhere below +the baseline, so at most one task can improve and a gain resting on one task has a sign-flip p of 1. A pair +whose baseline and treatment tops differ is on two scales, and any other case has no flag: ``None``. The flag +is a note; p and Holm are unchanged. Each selector's batches are reported with the stamp ``repo run`` pins a batch to (``tree``, ``eval_dir`` digest, ``image``) and the harness versions of its @@ -300,12 +301,14 @@ def _compare_model(treatment: str, model: str, base_vals: dict[tuple[str, str], else: pairs.append(pair) if pairs: - tops = [(base_vals[(model, p["task"])]["top"], treat_vals[(model, p["task"])]["top"]) for p in pairs] - failed = [base_vals[(model, p["task"])]["grader_failed"] or treat_vals[(model, p["task"])]["grader_failed"] - for p in pairs] + flag = None + if measure in PRIMARY: + tops = [(base_vals[(model, p["task"])]["top"], treat_vals[(model, p["task"])]["top"]) for p in pairs] + failed = [base_vals[(model, p["task"])]["grader_failed"] + or treat_vals[(model, p["task"])]["grader_failed"] for p in pairs] + flag = headroom(pairs, tops, failed) rows.append({"treatment": treatment, "model": model, "measure": measure, - **paired_summary(pairs, measure, seed=seed), - "headroom": headroom(pairs, tops, failed) if measure in PRIMARY else None}) + **paired_summary(pairs, measure, seed=seed), "headroom": flag}) def headroom(pairs: list[dict[str, Any]], tops: list[tuple[float | None, float | None]], @@ -317,16 +320,21 @@ def headroom(pairs: list[dict[str, Any]], tops: list[tuple[float | None, float | """ if any(b != t for b, t in tops): return None # two scales: neither "at the top" nor the delta compares + if failed is None: + failed = [False] * len(pairs) + def at_top(value: float, top: float | None) -> bool: return top is not None and value >= top - EPS base_top = [at_top(p["baseline"], b) for p, (b, _) in zip(pairs, tops, strict=True)] if all(base_top) and all(at_top(p["treatment"], t) for p, (_, t) in zip(pairs, tops, strict=True)): return "ceiling" - if (not any(failed or ()) and - all(abs(p["baseline"]) <= EPS and abs(p["treatment"]) <= EPS for p in pairs)): + if not any(failed) and all(abs(p["baseline"]) <= EPS and abs(p["treatment"]) <= EPS for p in pairs): return "floor" - if len(pairs) >= 2 and base_top.count(False) <= 1 and all(p["treatment"] >= p["baseline"] - EPS for p in pairs): + # the one task that may still move must have a measured baseline: a grader failure says nothing about its room + not_top_failed = any(f and not t for f, t in zip(failed, base_top, strict=True)) + if (len(pairs) >= 2 and base_top.count(False) <= 1 and not not_top_failed + and all(p["treatment"] >= p["baseline"] - EPS for p in pairs)): return "near_ceiling" return None diff --git a/dokimi/tests/test_repo_compare.py b/dokimi/tests/test_repo_compare.py index 06140a648..df3545185 100644 --- a/dokimi/tests/test_repo_compare.py +++ b/dokimi/tests/test_repo_compare.py @@ -258,6 +258,59 @@ def test_score_headroom_unknown_when_pooling_legacy_and_current_cells(tmp_path): assert _row(compare([b], "guidance:none"), "score")["headroom"] is None +def test_a_grader_failure_in_either_arm_is_not_floor(tmp_path): + b = tmp_path / "b1" + for i in range(2): + _cell(b, f"review_{i}", "guidance:raw", 1, score=0.0) + _cell(b, "review_0", "guidance:none", 1, score=0.0) + _cell(b, "review_1", "guidance:none", 1, score=0.0, failure=True) # baseline only: the treatment is clean + assert _row(compare([b], "guidance:none"), "score")["headroom"] is None + + +def test_a_gate_task_with_a_grader_failure_on_one_rep_is_not_floor(tmp_path): + b = tmp_path / "b1" + for arm in ("guidance:none", "guidance:raw"): + _cell(b, "fix_a", arm, 1, success=False) + _cell(b, "fix_a", arm, 2, success=False, failure=True) # the second rep only + _cell(b, "fix_b", arm, 1, success=False) + _cell(b, "fix_b", arm, 2, success=False) + assert _row(compare([b], "guidance:none"), "gate")["headroom"] is None + + +def test_a_grader_failure_on_the_one_task_that_can_move_is_not_near_ceiling(tmp_path): + b = tmp_path / "b1" + _cell(b, "r0", "guidance:none", 1, score=1.0, score_max=1.0) + _cell(b, "r0", "guidance:raw", 1, score=1.0, score_max=1.0) + _cell(b, "r1", "guidance:none", 1, score=0.0, score_max=1.0, failure=True) # not measured, not "room" + _cell(b, "r1", "guidance:raw", 1, score=1.0, score_max=1.0) + assert _row(compare([b], "guidance:none"), "score")["headroom"] is None + c = tmp_path / "b2" + _cell(c, "r0", "guidance:none", 1, score=1.0, score_max=1.0) + _cell(c, "r0", "guidance:raw", 1, score=1.0, score_max=1.0) + _cell(c, "r1", "guidance:none", 1, score=0.0, score_max=1.0) # the same, measured as a zero + _cell(c, "r1", "guidance:raw", 1, score=1.0, score_max=1.0) + assert _row(compare([c], "guidance:none"), "score")["headroom"] == "near_ceiling" + + +@pytest.mark.parametrize("pairs,tops,failed,expected", [ + # two all-zero pairs; one grader failure is not a floor (any, not all) + ([{"baseline": 0.0, "treatment": 0.0}] * 2, [(1.0, 1.0)] * 2, [True, False], None), + # a legacy baseline (unknown top) against a known one: two scales, whatever the values + ([{"baseline": 0.0, "treatment": 0.0}] * 2, [(None, 4.0), (4.0, 4.0)], None, None), + # tops that differ on one pair + ([{"baseline": 4.0, "treatment": 4.0}, {"baseline": 1.0, "treatment": 1.0}], [(4.0, 4.0), (4.0, 1.0)], None, None), + # equal tops on every pair and all zeros: a floor + ([{"baseline": 0.0, "treatment": 0.0}] * 2, [(4.0, 4.0), (4.0, 4.0)], None, "floor"), + # the one non-top pair is a grader failure: no near_ceiling + ([{"baseline": 1.0, "treatment": 1.0}, {"baseline": 0.5, "treatment": 1.0}], [(1.0, 1.0)] * 2, + [False, True], None), + ([{"baseline": 1.0, "treatment": 1.0}, {"baseline": 0.5, "treatment": 1.0}], [(1.0, 1.0)] * 2, + [False, False], "near_ceiling"), +]) +def test_headroom_on_mixed_tops_and_grader_failures(pairs, tops, failed, expected): + assert compare_mod.headroom(pairs, tops, failed) == expected + + def test_unpaired_tasks_are_listed_not_compared(tmp_path): b = tmp_path / "b1" _cell(b, "fix_a", "guidance:none", 1) From d2d6b57a6dcb3529430f9495560012311b7eafe3 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Mon, 19 Oct 2026 23:52:50 -0700 Subject: [PATCH 05/20] fix(repo): no headroom flag when either arm has a grader failure A grader failure scores 0, which reads as the top when score.max <= 0: every cell failing on a max-0 score task was reported as ceiling. Floor and near_ceiling already skipped such rows; ceiling now does too, and the three per-branch checks collapse into one. Signed-off-by: Alexander Rybkin --- ...0000_napetrov_feat-repo-compare-ceiling.md | 2 +- dokimi/docs/repo-evaluation-contract.md | 10 ++++----- dokimi/dokimi/repos/compare.py | 21 ++++++++----------- dokimi/tests/test_repo_compare.py | 10 +++++++++ 4 files changed, 25 insertions(+), 18 deletions(-) diff --git a/dokimi/changelog.d/20261018_120000_napetrov_feat-repo-compare-ceiling.md b/dokimi/changelog.d/20261018_120000_napetrov_feat-repo-compare-ceiling.md index 1777bfdb0..b833de5e6 100644 --- a/dokimi/changelog.d/20261018_120000_napetrov_feat-repo-compare-ceiling.md +++ b/dokimi/changelog.d/20261018_120000_napetrov_feat-repo-compare-ceiling.md @@ -1,3 +1,3 @@ ### Added -- **`repo compare` flags gate and score comparisons with no headroom** — each comparison gains `headroom`: `"ceiling"` when both arms are at the top (gate 1.0, the task's `score.max`) on every paired task, `"floor"` when both are at 0 on every one, none of those zeros a grader failure, `"near_ceiling"` when the baseline is at the top on all but at most one of 2+ paired tasks (that one not a grader failure) and the treatment is nowhere below the baseline, else `null`; arms whose tops differ (or are unknown) get `null`. The text report prints a note under the comparison. A zero delta on such a task set is no headroom, not no effect. p values and Holm's correction are unchanged. `result.json` now records the task's `score_max`; score rows of batches run before this have no known top and are never flagged `ceiling`. +- **`repo compare` flags gate and score comparisons with no headroom** — each comparison gains `headroom`: `"ceiling"` when both arms are at the top (gate 1.0, the task's `score.max`) on every paired task, `"floor"` when both are at 0 on every one, `"near_ceiling"` when the baseline is at the top on all but at most one of 2+ paired tasks and the treatment is nowhere below the baseline, else `null`; arms whose tops differ (or are unknown), and a grader failure in either arm, get `null`. The text report prints a note under the comparison. A zero delta on such a task set is no headroom, not no effect. p values and Holm's correction are unchanged. `result.json` now records the task's `score_max`; score rows of batches run before this have no known top and are never flagged `ceiling`. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 2c8af1d82..ab240f8f0 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1011,11 +1011,11 @@ python cli.py repo compare repo-runs/pilot --baseline guidance:none --metric roo Holm's correction runs over the gate and score comparisons of one call; the rest are descriptive. A gate or score comparison carries `headroom`: `ceiling` (both arms at 1.0 or the task's `score.max` on every paired - task), `floor` (both at 0 on every one, none of the zeros a grader - failure), `near_ceiling` (the baseline at the top on all but at most one of - 2+ tasks and the treatment nowhere below it), else `null`, with a note in - the text; a zero delta there means no room to move, not no effect. The - top of a score task is the `score.max` that `repo run` records in + task), `floor` (both at 0 on every one), `near_ceiling` (the baseline at + the top on all but at most one of 2+ tasks and the treatment nowhere below + it), else `null`, and `null` whenever either arm has a grader failure; a + note in the text; a zero delta there means no room to move, not no effect. + The top of a score task is the `score.max` that `repo run` records in `result.json` as `score_max`: a batch run before that has no known top, so its score rows are never `ceiling`, and arms whose tops differ are on two scales and get `null`. `BATCH/ARM` takes an arm from one batch only, so two diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index adfc6f05b..546bdbf2c 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -32,12 +32,11 @@ on a task set nobody can fail, or nobody can pass, is not evidence of no effect. A task's top is 1.0 for gate and the task's ``score.max`` for score (recorded in ``result.json``; a batch without it has no known top, so its score rows are never at ceiling). ``ceiling``: both arms at the top on every paired task; -``floor``: both arms at 0 on every paired task, none of the zeros a grader failure (those say the -measurement failed, not that the tasks had no room); ``near_ceiling``: neither, at least 2 paired tasks, -the baseline at the top on all but at most one, that one not a grader failure, and the treatment nowhere below -the baseline, so at most one task can improve and a gain resting on one task has a sign-flip p of 1. A pair -whose baseline and treatment tops differ is on two scales, and any other case has no flag: ``None``. The flag -is a note; p and Holm are unchanged. +``floor``: both arms at 0 on every paired task; ``near_ceiling``: neither, at least 2 paired tasks, the +baseline at the top on all but at most one, and the treatment nowhere below the baseline, so at most one task +can improve and a gain resting on one task has a sign-flip p of 1. A pair whose baseline and treatment tops +differ is on two scales, a grader failure in either arm scores a 0 that says the measurement failed, not that +the task had no room, and any other case has no flag: ``None``. The flag is a note; p and Holm are unchanged. Each selector's batches are reported with the stamp ``repo run`` pins a batch to (``tree``, ``eval_dir`` digest, ``image``) and the harness versions of its @@ -320,8 +319,8 @@ def headroom(pairs: list[dict[str, Any]], tops: list[tuple[float | None, float | """ if any(b != t for b, t in tops): return None # two scales: neither "at the top" nor the delta compares - if failed is None: - failed = [False] * len(pairs) + if failed and any(failed): + return None # its 0 is not a measurement, at either end (a score.max of 0 would read it as the top) def at_top(value: float, top: float | None) -> bool: return top is not None and value >= top - EPS @@ -329,11 +328,9 @@ def at_top(value: float, top: float | None) -> bool: base_top = [at_top(p["baseline"], b) for p, (b, _) in zip(pairs, tops, strict=True)] if all(base_top) and all(at_top(p["treatment"], t) for p, (_, t) in zip(pairs, tops, strict=True)): return "ceiling" - if not any(failed) and all(abs(p["baseline"]) <= EPS and abs(p["treatment"]) <= EPS for p in pairs): + if all(abs(p["baseline"]) <= EPS and abs(p["treatment"]) <= EPS for p in pairs): return "floor" - # the one task that may still move must have a measured baseline: a grader failure says nothing about its room - not_top_failed = any(f and not t for f, t in zip(failed, base_top, strict=True)) - if (len(pairs) >= 2 and base_top.count(False) <= 1 and not not_top_failed + if (len(pairs) >= 2 and base_top.count(False) <= 1 and all(p["treatment"] >= p["baseline"] - EPS for p in pairs)): return "near_ceiling" return None diff --git a/dokimi/tests/test_repo_compare.py b/dokimi/tests/test_repo_compare.py index df3545185..b04da9b53 100644 --- a/dokimi/tests/test_repo_compare.py +++ b/dokimi/tests/test_repo_compare.py @@ -311,6 +311,16 @@ def test_headroom_on_mixed_tops_and_grader_failures(pairs, tops, failed, expecte assert compare_mod.headroom(pairs, tops, failed) == expected +@pytest.mark.parametrize("score_max", [0.0, -1.0]) +def test_a_grader_failure_is_not_ceiling_when_score_max_is_not_above_zero(tmp_path, score_max): + b = tmp_path / "b1" + for i in range(3): # a lower-is-better metric negated, say; the grader failed everywhere and scored 0 + _cell(b, f"r{i}", "guidance:none", 1, score=0.0, score_max=score_max, failure=True) + _cell(b, f"r{i}", "guidance:raw", 1, score=0.0, score_max=score_max, failure=True) + s = compare([b], "guidance:none") + assert _row(s, "score")["headroom"] is None and "ceiling" not in format_compare(s) + + def test_unpaired_tasks_are_listed_not_compared(tmp_path): b = tmp_path / "b1" _cell(b, "fix_a", "guidance:none", 1) From 27fb2c1f35ce6f3a7e9b0dd4b7abce6483a8f16b Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Mon, 19 Oct 2026 23:53:22 -0700 Subject: [PATCH 06/20] fix(repo): a score that moved past score.max is not at the top score.max is what the oracle must reach, not a bound. A treatment scoring 90 against a baseline of 80 on max 80 (p 0.008, 8 up) was flagged ceiling, and one baseline at 70 made it near_ceiling. A pair counts as at the top only when both arms are there and tied. Signed-off-by: Alexander Rybkin --- dokimi/dokimi/repos/compare.py | 4 +++- dokimi/tests/test_repo_compare.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index 546bdbf2c..543245cd2 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -325,7 +325,9 @@ def headroom(pairs: list[dict[str, Any]], tops: list[tuple[float | None, float | def at_top(value: float, top: float | None) -> bool: return top is not None and value >= top - EPS - base_top = [at_top(p["baseline"], b) for p, (b, _) in zip(pairs, tops, strict=True)] + # score.max is what the oracle must reach, not a bound: a pair that moved past it is an effect, not a top + base_top = [at_top(p["baseline"], b) and abs(p["treatment"] - p["baseline"]) <= EPS + for p, (b, _) in zip(pairs, tops, strict=True)] if all(base_top) and all(at_top(p["treatment"], t) for p, (_, t) in zip(pairs, tops, strict=True)): return "ceiling" if all(abs(p["baseline"]) <= EPS and abs(p["treatment"]) <= EPS for p in pairs): diff --git a/dokimi/tests/test_repo_compare.py b/dokimi/tests/test_repo_compare.py index b04da9b53..029a09b67 100644 --- a/dokimi/tests/test_repo_compare.py +++ b/dokimi/tests/test_repo_compare.py @@ -311,6 +311,21 @@ def test_headroom_on_mixed_tops_and_grader_failures(pairs, tops, failed, expecte assert compare_mod.headroom(pairs, tops, failed) == expected +@pytest.mark.parametrize("baseline,expected", [ + ((80.0,) * 8, None), # a gain past score.max on every task: an effect, not a ceiling + ((70.0,) + (80.0,) * 7, None), # nor near_ceiling + ((90.0,) * 8, "ceiling"), # both past score.max and tied: no movement +]) +def test_a_score_past_score_max_is_at_the_top_only_when_the_arms_tie(tmp_path, baseline, expected): + b = tmp_path / "b1" + for i, was in enumerate(baseline): + _cell(b, f"cov_{i}", "guidance:none", 1, score=was, score_max=80.0) + _cell(b, f"cov_{i}", "guidance:raw", 1, score=90.0, score_max=80.0) + s = compare([b], "guidance:none") + assert _row(s, "score")["headroom"] == expected + assert ("ceiling" in format_compare(s)) == (expected is not None) + + @pytest.mark.parametrize("score_max", [0.0, -1.0]) def test_a_grader_failure_is_not_ceiling_when_score_max_is_not_above_zero(tmp_path, score_max): b = tmp_path / "b1" From 851e879d201c3f469f6dcfc4a9754817efbdbfc5 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Mon, 19 Oct 2026 23:53:44 -0700 Subject: [PATCH 07/20] test(repo): pin the headroom rules the suite left open EPS on float means, the treatment side of floor, the two-task minimum of near_ceiling, a small loss, one known top per task, and no flag on cost or turns rows: each of these could be dropped with the suite still green. Signed-off-by: Alexander Rybkin --- dokimi/tests/test_repo_compare.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/dokimi/tests/test_repo_compare.py b/dokimi/tests/test_repo_compare.py index 029a09b67..fdec3093a 100644 --- a/dokimi/tests/test_repo_compare.py +++ b/dokimi/tests/test_repo_compare.py @@ -306,11 +306,39 @@ def test_a_grader_failure_on_the_one_task_that_can_move_is_not_near_ceiling(tmp_ [False, True], None), ([{"baseline": 1.0, "treatment": 1.0}, {"baseline": 0.5, "treatment": 1.0}], [(1.0, 1.0)] * 2, [False, False], "near_ceiling"), + # one arm at 0 is not a floor; one task is not near_ceiling; a small loss is still a loss + ([{"baseline": 0.0, "treatment": 1.0}] * 3, [(1.0, 1.0)] * 3, None, None), + ([{"baseline": 0.0, "treatment": 1.0}], [(1.0, 1.0)], None, None), + ([{"baseline": 1.0, "treatment": 1.0}, {"baseline": 1.0, "treatment": 0.7}], [(1.0, 1.0)] * 2, None, None), ]) def test_headroom_on_mixed_tops_and_grader_failures(pairs, tops, failed, expected): assert compare_mod.headroom(pairs, tops, failed) == expected +def test_score_headroom_allows_for_float_means_and_needs_one_known_top(tmp_path): + b = tmp_path / "b1" + for i in range(2): + for arm in ("guidance:none", "guidance:raw"): + for rep in (1, 2, 3): # a mean of 0.6999999999999998 + _cell(b, f"r{i}", arm, rep, score=0.7, score_max=0.7) + assert _row(compare([b], "guidance:none"), "score")["headroom"] == "ceiling" + c = tmp_path / "b2" + for i in range(2): + for arm in ("guidance:none", "guidance:raw"): # two eval trees in one task: no single top + _cell(c, f"r{i}", arm, 1, score=4.0, score_max=1.0) + _cell(c, f"r{i}", arm, 2, score=4.0, score_max=4.0) + assert _row(compare([c], "guidance:none"), "score")["headroom"] is None + + +def test_cost_and_turns_rows_never_carry_a_headroom_flag(tmp_path): + b = tmp_path / "b1" + for i in range(2): + for arm in ("guidance:none", "guidance:raw"): # values that equal the gate's top + _cell(b, f"fix_{i}", arm, 1, cost=1.0, turns=1) + s = compare([b], "guidance:none") + assert _row(s, "cost_usd")["headroom"] is None and _row(s, "turns")["headroom"] is None + + @pytest.mark.parametrize("baseline,expected", [ ((80.0,) * 8, None), # a gain past score.max on every task: an effect, not a ceiling ((70.0,) + (80.0,) * 7, None), # nor near_ceiling From 50e55904247e884adf2a1c871cfc5c2477138159 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 20 Oct 2026 04:02:02 -0700 Subject: [PATCH 08/20] test(repo): pin the treatment's top in the floor headroom; fix the contract headroom text The mixed-tops test only covered the ceiling and near_ceiling paths, so reading the baseline's top for both arms passed the suite. Add a floor case where both arms score 0 and only the treatment's top differs. Also fix the garbled headroom sentence in the repo evaluation contract and say that non-gate rows carry null. Signed-off-by: Nikolay Petrov --- dokimi/docs/repo-evaluation-contract.md | 5 +++-- dokimi/tests/test_repo_compare.py | 9 +++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index ab240f8f0..20c0f1c23 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1013,8 +1013,9 @@ python cli.py repo compare repo-runs/pilot --baseline guidance:none --metric roo `ceiling` (both arms at 1.0 or the task's `score.max` on every paired task), `floor` (both at 0 on every one), `near_ceiling` (the baseline at the top on all but at most one of 2+ tasks and the treatment nowhere below - it), else `null`, and `null` whenever either arm has a grader failure; a - note in the text; a zero delta there means no room to move, not no effect. + it), else `null`; a grader failure in either arm also gives `null`. The text + prints a note under a flagged comparison. A zero delta there means no room to + move, not no effect. Cost, turns and metric rows have `headroom` `null`. The top of a score task is the `score.max` that `repo run` records in `result.json` as `score_max`: a batch run before that has no known top, so its score rows are never `ceiling`, and arms whose tops differ are on two diff --git a/dokimi/tests/test_repo_compare.py b/dokimi/tests/test_repo_compare.py index fdec3093a..5ef817f83 100644 --- a/dokimi/tests/test_repo_compare.py +++ b/dokimi/tests/test_repo_compare.py @@ -243,6 +243,15 @@ def test_score_headroom_is_unknown_when_the_arms_have_different_score_max(tmp_pa assert row["delta"]["mean"] == -3.0 and row["headroom"] is None +def test_score_floor_is_unknown_when_the_arms_have_different_score_max(tmp_path): + """Both arms score 0 on every task, so only the tops decide: the treatment's top (1.0) must count, not the baseline's.""" + b = tmp_path / "b1" + for i in range(2): + _cell(b, f"r{i}", "guidance:none", 1, score=0.0, score_max=4.0) + _cell(b, f"r{i}", "guidance:raw", 1, score=0.0, score_max=1.0) + assert _row(compare([b], "guidance:none"), "score")["headroom"] is None + + def test_score_headroom_unknown_when_pooling_legacy_and_current_cells(tmp_path): """Regression: when legacy cells (score_max=None) and current cells (score_max=1.0) appear for the same task, do not report ceiling if the legacy cells' maxima are unknown.""" From 8c03e9cc00b4b82e2e21a1d87ec3a9034d378c24 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 20 Oct 2026 14:35:32 -0700 Subject: [PATCH 09/20] test(repo): pin headroom tolerances at their boundaries; fix changelog top wording Tolerance cases for headroom() and a gate top of 1.0 that a 0.5 mutant would miss; the changelog no longer says unknown tops give null, since only tops that differ do (a batch without a recorded top is never ceiling). Signed-off-by: Nikolay Petrov --- ...0000_napetrov_feat-repo-compare-ceiling.md | 2 +- dokimi/tests/test_repo_compare.py | 23 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/dokimi/changelog.d/20261018_120000_napetrov_feat-repo-compare-ceiling.md b/dokimi/changelog.d/20261018_120000_napetrov_feat-repo-compare-ceiling.md index b833de5e6..8872da96a 100644 --- a/dokimi/changelog.d/20261018_120000_napetrov_feat-repo-compare-ceiling.md +++ b/dokimi/changelog.d/20261018_120000_napetrov_feat-repo-compare-ceiling.md @@ -1,3 +1,3 @@ ### Added -- **`repo compare` flags gate and score comparisons with no headroom** — each comparison gains `headroom`: `"ceiling"` when both arms are at the top (gate 1.0, the task's `score.max`) on every paired task, `"floor"` when both are at 0 on every one, `"near_ceiling"` when the baseline is at the top on all but at most one of 2+ paired tasks and the treatment is nowhere below the baseline, else `null`; arms whose tops differ (or are unknown), and a grader failure in either arm, get `null`. The text report prints a note under the comparison. A zero delta on such a task set is no headroom, not no effect. p values and Holm's correction are unchanged. `result.json` now records the task's `score_max`; score rows of batches run before this have no known top and are never flagged `ceiling`. +- **`repo compare` flags gate and score comparisons with no headroom** — each comparison gains `headroom`: `"ceiling"` when both arms are at the top (gate 1.0, the task's `score.max`) on every paired task, `"floor"` when both are at 0 on every one, `"near_ceiling"` when the baseline is at the top on all but at most one of 2+ paired tasks and the treatment is nowhere below the baseline, else `null`; arms whose tops differ, and a grader failure in either arm, get `null`; a batch without a recorded top is never `ceiling`. The text report prints a note under the comparison. A zero delta on such a task set is no headroom, not no effect. p values and Holm's correction are unchanged. `result.json` now records the task's `score_max`; score rows of batches run before this have no known top and are never flagged `ceiling`. diff --git a/dokimi/tests/test_repo_compare.py b/dokimi/tests/test_repo_compare.py index 5ef817f83..3c9247771 100644 --- a/dokimi/tests/test_repo_compare.py +++ b/dokimi/tests/test_repo_compare.py @@ -324,6 +324,29 @@ def test_headroom_on_mixed_tops_and_grader_failures(pairs, tops, failed, expecte assert compare_mod.headroom(pairs, tops, failed) == expected +@pytest.mark.parametrize("pairs,expected", [ + # both arms 0.05 short of the top: a task that still has room is no ceiling, nor near_ceiling + ([{"baseline": 0.95, "treatment": 0.95}] * 2, None), + # the baseline at the top on both tasks, the treatment 0.05 below it on one: a small loss, not near_ceiling + ([{"baseline": 1.0, "treatment": 1.0}, {"baseline": 1.0, "treatment": 0.95}], None), + # the treatment 0.3 past score.max on both tasks: an effect, not a top + ([{"baseline": 1.0, "treatment": 1.3}] * 2, None), +]) +def test_headroom_tolerances_are_exact_not_coarse(pairs, expected): + assert compare_mod.headroom(pairs, [(1.0, 1.0)] * len(pairs), None) == expected + + +def test_gate_headroom_top_is_one_not_half(tmp_path): + # both arms pass half of their reps on every task: a rep mean of 0.5 is not at the gate top of 1.0 + b = tmp_path / "b1" + for i in range(2): + _cell(b, f"fix_{i}", "guidance:none", 1, success=True) + _cell(b, f"fix_{i}", "guidance:none", 2, success=False) + _cell(b, f"fix_{i}", "guidance:raw", 1, success=True) + _cell(b, f"fix_{i}", "guidance:raw", 2, success=False) + assert _row(compare([b], "guidance:none"), "gate")["headroom"] is None + + def test_score_headroom_allows_for_float_means_and_needs_one_known_top(tmp_path): b = tmp_path / "b1" for i in range(2): From 5cee5267b6deaa7cd26e425a64005d2cd6c9e3b7 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Wed, 21 Oct 2026 01:02:22 -0700 Subject: [PATCH 10/20] docs(repo): say that a pair is at the top only when both arms tie 2025887 made a pair at the top only when both arms reach it and tie, but the docstring, the contract and the changelog still said "at the top": 1.0 vs 1.3 on max 1 read as ceiling or near_ceiling there, and gives null. Signed-off-by: Alexander Rybkin --- .../20261018_120000_napetrov_feat-repo-compare-ceiling.md | 2 +- dokimi/docs/repo-evaluation-contract.md | 5 +++-- dokimi/dokimi/repos/compare.py | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/dokimi/changelog.d/20261018_120000_napetrov_feat-repo-compare-ceiling.md b/dokimi/changelog.d/20261018_120000_napetrov_feat-repo-compare-ceiling.md index 8872da96a..eb36d42f9 100644 --- a/dokimi/changelog.d/20261018_120000_napetrov_feat-repo-compare-ceiling.md +++ b/dokimi/changelog.d/20261018_120000_napetrov_feat-repo-compare-ceiling.md @@ -1,3 +1,3 @@ ### Added -- **`repo compare` flags gate and score comparisons with no headroom** — each comparison gains `headroom`: `"ceiling"` when both arms are at the top (gate 1.0, the task's `score.max`) on every paired task, `"floor"` when both are at 0 on every one, `"near_ceiling"` when the baseline is at the top on all but at most one of 2+ paired tasks and the treatment is nowhere below the baseline, else `null`; arms whose tops differ, and a grader failure in either arm, get `null`; a batch without a recorded top is never `ceiling`. The text report prints a note under the comparison. A zero delta on such a task set is no headroom, not no effect. p values and Holm's correction are unchanged. `result.json` now records the task's `score_max`; score rows of batches run before this have no known top and are never flagged `ceiling`. +- **`repo compare` flags gate and score comparisons with no headroom** — each comparison gains `headroom`: `"ceiling"` when both arms are at the top (gate 1.0, the task's `score.max`) on every paired task, `"floor"` when both are at 0 on every one, `"near_ceiling"` when the baseline is at the top on all but at most one of 2+ paired tasks and the treatment is nowhere below the baseline, else `null`; arms whose tops differ, and a grader failure in either arm, get `null`; a batch without a recorded top is never `ceiling`. A pair is at the top only when both arms reach it and tie: a score past `score.max` is an effect, not a top. The text report prints a note under the comparison. A zero delta on such a task set is no headroom, not no effect. p values and Holm's correction are unchanged. `result.json` now records the task's `score_max`; score rows of batches run before this have no known top and are never flagged `ceiling`. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 20c0f1c23..cd73a4500 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1013,8 +1013,9 @@ python cli.py repo compare repo-runs/pilot --baseline guidance:none --metric roo `ceiling` (both arms at 1.0 or the task's `score.max` on every paired task), `floor` (both at 0 on every one), `near_ceiling` (the baseline at the top on all but at most one of 2+ tasks and the treatment nowhere below - it), else `null`; a grader failure in either arm also gives `null`. The text - prints a note under a flagged comparison. A zero delta there means no room to + it), else `null`; a grader failure in either arm also gives `null`. A pair + is at the top only when both arms reach it and tie: a score past `score.max` + is an effect, not a top. The text prints a note under a flagged comparison. A zero delta there means no room to move, not no effect. Cost, turns and metric rows have `headroom` `null`. The top of a score task is the `score.max` that `repo run` records in `result.json` as `score_max`: a batch run before that has no known top, so diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index 543245cd2..09c092d1d 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -31,7 +31,8 @@ A gate or score comparison also says whether its tasks had room to move (``headroom``), since a zero delta on a task set nobody can fail, or nobody can pass, is not evidence of no effect. A task's top is 1.0 for gate and the task's ``score.max`` for score (recorded in ``result.json``; a batch without it has no known -top, so its score rows are never at ceiling). ``ceiling``: both arms at the top on every paired task; +top, so its score rows are never at ceiling). A pair is at the top only when both arms reach it and tie: a +score past ``score.max`` is an effect, not a top. ``ceiling``: both arms at the top on every paired task; ``floor``: both arms at 0 on every paired task; ``near_ceiling``: neither, at least 2 paired tasks, the baseline at the top on all but at most one, and the treatment nowhere below the baseline, so at most one task can improve and a gain resting on one task has a sign-flip p of 1. A pair whose baseline and treatment tops From b0747340570550d1ca266981e39dd3017fee71c6 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 21 Oct 2026 11:55:55 -0700 Subject: [PATCH 11/20] docs(repo): wrap a line of the repo compare headroom paragraph Signed-off-by: Nikolay Petrov --- dokimi/docs/repo-evaluation-contract.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index cd73a4500..5afa0a7b1 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1015,8 +1015,8 @@ python cli.py repo compare repo-runs/pilot --baseline guidance:none --metric roo the top on all but at most one of 2+ tasks and the treatment nowhere below it), else `null`; a grader failure in either arm also gives `null`. A pair is at the top only when both arms reach it and tie: a score past `score.max` - is an effect, not a top. The text prints a note under a flagged comparison. A zero delta there means no room to - move, not no effect. Cost, turns and metric rows have `headroom` `null`. + is an effect, not a top. The text prints a note under a flagged comparison. A + zero delta there means no room to move, not no effect. Cost, turns and metric rows have `headroom` `null`. The top of a score task is the `score.max` that `repo run` records in `result.json` as `score_max`: a batch run before that has no known top, so its score rows are never `ceiling`, and arms whose tops differ are on two From 8ee3c4bfc05e0375245d83f85ebe18302e29cea7 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 21 Oct 2026 11:56:01 -0700 Subject: [PATCH 12/20] docs(repo): wrap the rest of the headroom paragraph Signed-off-by: Nikolay Petrov --- dokimi/docs/repo-evaluation-contract.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 5afa0a7b1..fbd43dc6e 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1016,7 +1016,8 @@ python cli.py repo compare repo-runs/pilot --baseline guidance:none --metric roo it), else `null`; a grader failure in either arm also gives `null`. A pair is at the top only when both arms reach it and tie: a score past `score.max` is an effect, not a top. The text prints a note under a flagged comparison. A - zero delta there means no room to move, not no effect. Cost, turns and metric rows have `headroom` `null`. + zero delta there means no room to move, not no effect. Cost, turns and + metric rows have `headroom` `null`. The top of a score task is the `score.max` that `repo run` records in `result.json` as `score_max`: a batch run before that has no known top, so its score rows are never `ceiling`, and arms whose tops differ are on two From dfbf261bee00caaa9bf29785354395a039148372 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 22 Oct 2026 22:01:41 -0700 Subject: [PATCH 13/20] fix(repos/compare): headroom is null when a cell is an agent outage Signed-off-by: Nikolay Petrov --- ...0000_napetrov_feat-repo-compare-ceiling.md | 2 +- dokimi/docs/repo-evaluation-contract.md | 3 ++- dokimi/dokimi/repos/compare.py | 13 +++++++++++-- dokimi/tests/test_repo_compare.py | 19 ++++++++++++++----- 4 files changed, 28 insertions(+), 9 deletions(-) diff --git a/dokimi/changelog.d/20261018_120000_napetrov_feat-repo-compare-ceiling.md b/dokimi/changelog.d/20261018_120000_napetrov_feat-repo-compare-ceiling.md index eb36d42f9..34b0cff5f 100644 --- a/dokimi/changelog.d/20261018_120000_napetrov_feat-repo-compare-ceiling.md +++ b/dokimi/changelog.d/20261018_120000_napetrov_feat-repo-compare-ceiling.md @@ -1,3 +1,3 @@ ### Added -- **`repo compare` flags gate and score comparisons with no headroom** — each comparison gains `headroom`: `"ceiling"` when both arms are at the top (gate 1.0, the task's `score.max`) on every paired task, `"floor"` when both are at 0 on every one, `"near_ceiling"` when the baseline is at the top on all but at most one of 2+ paired tasks and the treatment is nowhere below the baseline, else `null`; arms whose tops differ, and a grader failure in either arm, get `null`; a batch without a recorded top is never `ceiling`. A pair is at the top only when both arms reach it and tie: a score past `score.max` is an effect, not a top. The text report prints a note under the comparison. A zero delta on such a task set is no headroom, not no effect. p values and Holm's correction are unchanged. `result.json` now records the task's `score_max`; score rows of batches run before this have no known top and are never flagged `ceiling`. +- **`repo compare` flags gate and score comparisons with no headroom** — each comparison gains `headroom`: `"ceiling"` when both arms are at the top (gate 1.0, the task's `score.max`) on every paired task, `"floor"` when both are at 0 on every one, `"near_ceiling"` when the baseline is at the top on all but at most one of 2+ paired tasks and the treatment is nowhere below the baseline, else `null`; arms whose tops differ, and a grader failure or an agent outage (error result, no turn count or at most one turn) in either arm, get `null`; a batch without a recorded top is never `ceiling`. A pair is at the top only when both arms reach it and tie: a score past `score.max` is an effect, not a top. The text report prints a note under the comparison. A zero delta on such a task set is no headroom, not no effect. p values and Holm's correction are unchanged. `result.json` now records the task's `score_max`; score rows of batches run before this have no known top and are never flagged `ceiling`. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index fbd43dc6e..53bbbdc56 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1013,7 +1013,8 @@ python cli.py repo compare repo-runs/pilot --baseline guidance:none --metric roo `ceiling` (both arms at 1.0 or the task's `score.max` on every paired task), `floor` (both at 0 on every one), `near_ceiling` (the baseline at the top on all but at most one of 2+ tasks and the treatment nowhere below - it), else `null`; a grader failure in either arm also gives `null`. A pair + it), else `null`; a grader failure in either arm also gives `null`, as does an agent + outage (an error result with no turn count or at most one turn): neither is a measurement. A pair is at the top only when both arms reach it and tie: a score past `score.max` is an effect, not a top. The text prints a note under a flagged comparison. A zero delta there means no room to move, not no effect. Cost, turns and diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index 09c092d1d..e4dc72e2e 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -316,7 +316,7 @@ def headroom(pairs: list[dict[str, Any]], tops: list[tuple[float | None, float | """``ceiling``, ``floor``, ``near_ceiling`` or ``None`` for one gate or score comparison (module docstring). ``tops`` holds each pair's (baseline, treatment) top value, ``None`` where it is not known; ``failed`` - marks the pairs with a grader failure in either arm (its score of 0 is not a measurement). + marks the pairs with a grader failure or an agent outage in either arm (its 0 is not a measurement). """ if any(b != t for b, t in tops): return None # two scales: neither "at the top" nor the delta compares @@ -513,7 +513,9 @@ def _task_values(cells: list[dict[str, Any]], metrics: tuple[str, ...]) -> dict[ # If any cell has unknown max, or maxes differ, top is None; otherwise take the single value has_unknown_max = any(v is None for v in score_max_vals) top = (maxes.pop() if len(maxes) == 1 and not has_unknown_max else None) if score_task else 1.0 - grader_failed = any(r["grade"]["status"] == "grader_failure" for r in recs) + # unmeasured: a grader failure, or an agent outage (an error result with no or one turn); either way + # its 0 is not a measurement + grader_failed = any(r["grade"]["status"] == "grader_failure" or _agent_outage(r) for r in recs) out[key] = {"kind": "score" if score_task else "gate", "grader_failed": grader_failed, "top": top, "n": {measure: len(v) for measure, v in nums.items()}, @@ -521,6 +523,13 @@ def _task_values(cells: list[dict[str, Any]], metrics: tuple[str, ...]) -> dict[ return out +def _agent_outage(rec: dict[str, Any]) -> bool: + """A cell whose agent never ran: an error result with no turn count or at most one turn.""" + agent = rec.get("agent") or {} + turns = agent.get("num_turns") + return bool(agent.get("is_error")) and (turns is None or turns <= 1) + + def _grade_metrics(path: Path, names: tuple[str, ...]) -> dict[str, float]: """The requested numeric or boolean ``metrics`` of a cell's ``grade.json``; absent ones are left out.""" try: diff --git a/dokimi/tests/test_repo_compare.py b/dokimi/tests/test_repo_compare.py index 3c9247771..4991ad850 100644 --- a/dokimi/tests/test_repo_compare.py +++ b/dokimi/tests/test_repo_compare.py @@ -21,8 +21,8 @@ def _cell(batch: Path, task: str, arm: str, rep: int, *, success: bool | None = True, score: float | None = None, failure: bool = False, cost: float | None = 0.1, turns: int = 10, metrics: dict | None = None, - model: str = "m-1", harness_version: str = "1.0", model_resolved: str | list[str] | None = None, - score_max: float | None = 1.0) -> None: + model: str = "m-1", harness_version: str = "1.0", score_max: float | None = 1.0, + model_resolved: str | list[str] | None = None, is_error: bool = False) -> None: rd = batch / f"{task}__{arm.replace(':', '-')}__{model}__r{rep}" rd.mkdir(parents=True) score_task = score is not None @@ -30,9 +30,9 @@ def _cell(batch: Path, task: str, arm: str, rep: int, *, success: bool | None = "success": False if failure else (None if score_task else success), "score": None if failure else score, "score_only": score_task and not failure, "reasons": []} rec = {"task": task, "arm": arm, "model": model, "rep": rep, "score_task": score_task, "grade": grade, - "harness_version": harness_version, "model_resolved": model_resolved, - "score_max": score_max if score_task else None, - "agent": {"cost_usd": cost, "num_turns": turns, "timed_out": False, "is_error": False, + "harness_version": harness_version, "score_max": score_max if score_task else None, + "model_resolved": model_resolved, + "agent": {"cost_usd": cost, "num_turns": turns, "timed_out": False, "is_error": is_error, "over_budget": False, "network": "none"}} (rd / "meta.json").write_text("{}") (rd / "result.json").write_text(json.dumps(rec)) @@ -233,6 +233,15 @@ def test_score_headroom_is_measured_against_score_max(tmp_path): assert _row(compare([e], "guidance:none"), "score")["headroom"] is None +@pytest.mark.parametrize("turns,expected", [(None, None), (1, None), (5, "floor")]) +def test_an_agent_outage_is_unmeasured_not_a_floor(tmp_path, turns, expected): + b = tmp_path / "b1" + for i in range(3): + for arm in ("guidance:none", "guidance:raw"): + _cell(b, f"fix_{i}", arm, 1, success=False, turns=turns, is_error=True) + assert _row(compare([b], "guidance:none"), "gate")["headroom"] == expected + + def test_score_headroom_is_unknown_when_the_arms_have_different_score_max(tmp_path): b = tmp_path / "b1" for i in range(2): From b5317aa176ec2131c30893e1e8fc149a7d42626e Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Wed, 21 Oct 2026 21:26:42 -0700 Subject: [PATCH 14/20] fix(repo): a turn count that is not a number no longer crashes compare An error cell whose num_turns is not a number ("1" from a hand-edited or foreign result.json) raised TypeError in the outage check and aborted the whole repo compare. It now counts as no turn count, as the turns row does. Signed-off-by: Alexander Rybkin --- dokimi/dokimi/repos/compare.py | 4 ++-- dokimi/tests/test_repo_compare.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index e4dc72e2e..b5a5bcf97 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -526,8 +526,8 @@ def _task_values(cells: list[dict[str, Any]], metrics: tuple[str, ...]) -> dict[ def _agent_outage(rec: dict[str, Any]) -> bool: """A cell whose agent never ran: an error result with no turn count or at most one turn.""" agent = rec.get("agent") or {} - turns = agent.get("num_turns") - return bool(agent.get("is_error")) and (turns is None or turns <= 1) + turns = _finite([agent.get("num_turns")]) # a count that is not a number is no count, as in the turns row + return bool(agent.get("is_error")) and (not turns or turns[0] <= 1) def _grade_metrics(path: Path, names: tuple[str, ...]) -> dict[str, float]: diff --git a/dokimi/tests/test_repo_compare.py b/dokimi/tests/test_repo_compare.py index 4991ad850..3633e1541 100644 --- a/dokimi/tests/test_repo_compare.py +++ b/dokimi/tests/test_repo_compare.py @@ -233,7 +233,7 @@ def test_score_headroom_is_measured_against_score_max(tmp_path): assert _row(compare([e], "guidance:none"), "score")["headroom"] is None -@pytest.mark.parametrize("turns,expected", [(None, None), (1, None), (5, "floor")]) +@pytest.mark.parametrize("turns,expected", [(None, None), (1, None), (5, "floor"), ("1", None)]) def test_an_agent_outage_is_unmeasured_not_a_floor(tmp_path, turns, expected): b = tmp_path / "b1" for i in range(3): From 5409b627e3d60c4fbc5fdc77f221c2bc593997b9 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Wed, 21 Oct 2026 21:26:51 -0700 Subject: [PATCH 15/20] docs(repo): the compare docstring names the agent-outage exclusion The module docstring, which the PR names as the rule's reference, listed only a grader failure as unmeasured; an all-outage task set read as floor there and gives null in the code, the contract and the changelog. Signed-off-by: Alexander Rybkin --- dokimi/dokimi/repos/compare.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index b5a5bcf97..c4a1dcd08 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -36,8 +36,9 @@ ``floor``: both arms at 0 on every paired task; ``near_ceiling``: neither, at least 2 paired tasks, the baseline at the top on all but at most one, and the treatment nowhere below the baseline, so at most one task can improve and a gain resting on one task has a sign-flip p of 1. A pair whose baseline and treatment tops -differ is on two scales, a grader failure in either arm scores a 0 that says the measurement failed, not that -the task had no room, and any other case has no flag: ``None``. The flag is a note; p and Holm are unchanged. +differ is on two scales, a grader failure or an agent outage (an error result with no turn count or at most +one turn) in either arm scores a 0 that says the measurement failed, not that the task had no room, and any +other case has no flag: ``None``. The flag is a note; p and Holm are unchanged. Each selector's batches are reported with the stamp ``repo run`` pins a batch to (``tree``, ``eval_dir`` digest, ``image``) and the harness versions of its From b45a0d2b1f902a824f0f8c6da36af742fa9a078d Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 22 Oct 2026 08:14:40 -0700 Subject: [PATCH 16/20] fix(repo): a harness that exits non-zero with no result is an agent outage in compare headroom Signed-off-by: Nikolay Petrov --- ...0000_napetrov_feat-repo-compare-ceiling.md | 2 +- dokimi/docs/repo-evaluation-contract.md | 3 ++- dokimi/dokimi/repos/compare.py | 15 ++++++++---- dokimi/tests/test_repo_compare.py | 24 +++++++++++++++++-- 4 files changed, 36 insertions(+), 8 deletions(-) diff --git a/dokimi/changelog.d/20261018_120000_napetrov_feat-repo-compare-ceiling.md b/dokimi/changelog.d/20261018_120000_napetrov_feat-repo-compare-ceiling.md index 34b0cff5f..88481d503 100644 --- a/dokimi/changelog.d/20261018_120000_napetrov_feat-repo-compare-ceiling.md +++ b/dokimi/changelog.d/20261018_120000_napetrov_feat-repo-compare-ceiling.md @@ -1,3 +1,3 @@ ### Added -- **`repo compare` flags gate and score comparisons with no headroom** — each comparison gains `headroom`: `"ceiling"` when both arms are at the top (gate 1.0, the task's `score.max`) on every paired task, `"floor"` when both are at 0 on every one, `"near_ceiling"` when the baseline is at the top on all but at most one of 2+ paired tasks and the treatment is nowhere below the baseline, else `null`; arms whose tops differ, and a grader failure or an agent outage (error result, no turn count or at most one turn) in either arm, get `null`; a batch without a recorded top is never `ceiling`. A pair is at the top only when both arms reach it and tie: a score past `score.max` is an effect, not a top. The text report prints a note under the comparison. A zero delta on such a task set is no headroom, not no effect. p values and Holm's correction are unchanged. `result.json` now records the task's `score_max`; score rows of batches run before this have no known top and are never flagged `ceiling`. +- **`repo compare` flags gate and score comparisons with no headroom** — each comparison gains `headroom`: `"ceiling"` when both arms are at the top (gate 1.0, the task's `score.max`) on every paired task, `"floor"` when both are at 0 on every one, `"near_ceiling"` when the baseline is at the top on all but at most one of 2+ paired tasks and the treatment is nowhere below the baseline, else `null`; arms whose tops differ, and a grader failure or an agent outage (error result with no turn count or at most one turn, or a non-zero exit with no turn count that is not a timeout) in either arm, get `null`; a batch without a recorded top is never `ceiling`. A pair is at the top only when both arms reach it and tie: a score past `score.max` is an effect, not a top. The text report prints a note under the comparison. A zero delta on such a task set is no headroom, not no effect. p values and Holm's correction are unchanged. `result.json` now records the task's `score_max`; score rows of batches run before this have no known top and are never flagged `ceiling`. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 53bbbdc56..945ab89c9 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1014,7 +1014,8 @@ python cli.py repo compare repo-runs/pilot --baseline guidance:none --metric roo task), `floor` (both at 0 on every one), `near_ceiling` (the baseline at the top on all but at most one of 2+ tasks and the treatment nowhere below it), else `null`; a grader failure in either arm also gives `null`, as does an agent - outage (an error result with no turn count or at most one turn): neither is a measurement. A pair + outage (an error result with no turn count or at most one turn, or a harness that exited + non-zero without reporting a turn and was not killed at its timeout): neither is a measurement. A pair is at the top only when both arms reach it and tie: a score past `score.max` is an effect, not a top. The text prints a note under a flagged comparison. A zero delta there means no room to move, not no effect. Cost, turns and diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index c4a1dcd08..0ba42d33b 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -37,7 +37,7 @@ baseline at the top on all but at most one, and the treatment nowhere below the baseline, so at most one task can improve and a gain resting on one task has a sign-flip p of 1. A pair whose baseline and treatment tops differ is on two scales, a grader failure or an agent outage (an error result with no turn count or at most -one turn) in either arm scores a 0 that says the measurement failed, not that the task had no room, and any +one turn, or a non-zero exit with no turn count that is not a timeout) in either arm scores a 0 that says the measurement failed, not that the task had no room, and any other case has no flag: ``None``. The flag is a note; p and Holm are unchanged. Each selector's batches are reported with the stamp ``repo run`` pins a batch @@ -514,7 +514,8 @@ def _task_values(cells: list[dict[str, Any]], metrics: tuple[str, ...]) -> dict[ # If any cell has unknown max, or maxes differ, top is None; otherwise take the single value has_unknown_max = any(v is None for v in score_max_vals) top = (maxes.pop() if len(maxes) == 1 and not has_unknown_max else None) if score_task else 1.0 - # unmeasured: a grader failure, or an agent outage (an error result with no or one turn); either way + # unmeasured: a grader failure, or an agent outage (an error result with no or one turn, or a non-zero exit + # with no turn count); either way # its 0 is not a measurement grader_failed = any(r["grade"]["status"] == "grader_failure" or _agent_outage(r) for r in recs) out[key] = {"kind": "score" if score_task else "gate", "grader_failed": grader_failed, @@ -525,10 +526,16 @@ def _task_values(cells: list[dict[str, Any]], metrics: tuple[str, ...]) -> dict[ def _agent_outage(rec: dict[str, Any]) -> bool: - """A cell whose agent never ran: an error result with no turn count or at most one turn.""" + """A cell whose agent never ran: an error result with no turn count or at most one turn, or a harness that + exited non-zero before it reported any turn (no result record, so no ``is_error``) and was not killed for + its timeout: an agent killed at its timeout did attempt the task.""" agent = rec.get("agent") or {} turns = _finite([agent.get("num_turns")]) # a count that is not a number is no count, as in the turns row - return bool(agent.get("is_error")) and (not turns or turns[0] <= 1) + if agent.get("is_error") and (not turns or turns[0] <= 1): + return True + rc = agent.get("rc") + crashed = isinstance(rc, int) and not isinstance(rc, bool) and rc != 0 + return crashed and not turns and not agent.get("timed_out") def _grade_metrics(path: Path, names: tuple[str, ...]) -> dict[str, float]: diff --git a/dokimi/tests/test_repo_compare.py b/dokimi/tests/test_repo_compare.py index 3633e1541..7e19a729c 100644 --- a/dokimi/tests/test_repo_compare.py +++ b/dokimi/tests/test_repo_compare.py @@ -22,7 +22,8 @@ def _cell(batch: Path, task: str, arm: str, rep: int, *, success: bool | None = True, score: float | None = None, failure: bool = False, cost: float | None = 0.1, turns: int = 10, metrics: dict | None = None, model: str = "m-1", harness_version: str = "1.0", score_max: float | None = 1.0, - model_resolved: str | list[str] | None = None, is_error: bool = False) -> None: + model_resolved: str | list[str] | None = None, is_error: bool = False, rc: int | None = None, + timed_out: bool = False) -> None: rd = batch / f"{task}__{arm.replace(':', '-')}__{model}__r{rep}" rd.mkdir(parents=True) score_task = score is not None @@ -32,7 +33,7 @@ def _cell(batch: Path, task: str, arm: str, rep: int, *, success: bool | None = rec = {"task": task, "arm": arm, "model": model, "rep": rep, "score_task": score_task, "grade": grade, "harness_version": harness_version, "score_max": score_max if score_task else None, "model_resolved": model_resolved, - "agent": {"cost_usd": cost, "num_turns": turns, "timed_out": False, "is_error": is_error, + "agent": {"cost_usd": cost, "num_turns": turns, "timed_out": timed_out, "is_error": is_error, "rc": rc, "over_budget": False, "network": "none"}} (rd / "meta.json").write_text("{}") (rd / "result.json").write_text(json.dumps(rec)) @@ -242,6 +243,25 @@ def test_an_agent_outage_is_unmeasured_not_a_floor(tmp_path, turns, expected): assert _row(compare([b], "guidance:none"), "gate")["headroom"] == expected +@pytest.mark.parametrize("rc,turns,timed_out,expected", [ + (1, None, False, None), (137, None, False, None), # exited non-zero, no result record: the agent never ran + (0, None, False, "floor"), (None, None, False, "floor"), # exit 0 or no rc recorded: not an outage + (1, 5, False, "floor"), # it reported turns: an attempt + (137, None, True, "floor"), # killed at its timeout: an attempt +]) +def test_a_harness_that_exits_non_zero_with_no_result_is_an_agent_outage(tmp_path, rc, turns, timed_out, expected): + b = tmp_path / "b1" + for i in range(3): + for arm in ("guidance:none", "guidance:raw"): + _cell(b, f"fix_{i}", arm, 1, success=False, turns=turns, rc=rc, timed_out=timed_out) + assert _row(compare([b], "guidance:none"), "gate")["headroom"] == expected + b2 = tmp_path / "b2" + for i in range(3): + for arm in ("guidance:none", "guidance:raw"): + _cell(b2, f"r_{i}", arm, 1, score=0.0, turns=turns, rc=rc, timed_out=timed_out) + assert _row(compare([b2], "guidance:none"), "score")["headroom"] == expected + + def test_score_headroom_is_unknown_when_the_arms_have_different_score_max(tmp_path): b = tmp_path / "b1" for i in range(2): From 14185920272c042b076f8e73316203f47aaa1191 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Thu, 22 Oct 2026 17:10:10 +0000 Subject: [PATCH 17/20] Re-wrap the headroom outage sentence in the compare docstring Signed-off-by: Anton Volkov --- dokimi/dokimi/repos/compare.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index 0ba42d33b..886c179ac 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -36,9 +36,10 @@ ``floor``: both arms at 0 on every paired task; ``near_ceiling``: neither, at least 2 paired tasks, the baseline at the top on all but at most one, and the treatment nowhere below the baseline, so at most one task can improve and a gain resting on one task has a sign-flip p of 1. A pair whose baseline and treatment tops -differ is on two scales, a grader failure or an agent outage (an error result with no turn count or at most -one turn, or a non-zero exit with no turn count that is not a timeout) in either arm scores a 0 that says the measurement failed, not that the task had no room, and any -other case has no flag: ``None``. The flag is a note; p and Holm are unchanged. +differ is on two scales. A grader failure or an agent outage (an error result with no turn count or at most +one turn, or a non-zero exit with no turn count that is not a timeout) in either arm scores a 0 that says the +measurement failed, not that the task had no room. Any other case has no flag: ``None``. The flag is a note; +p and Holm are unchanged. Each selector's batches are reported with the stamp ``repo run`` pins a batch to (``tree``, ``eval_dir`` digest, ``image``) and the harness versions of its From 0dbcdb5158c3a0dae67d5c467fcc2e9c52cafa58 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Thu, 22 Oct 2026 17:10:28 +0000 Subject: [PATCH 18/20] Re-wrap the agent-outage clause in the contract headroom bullet Signed-off-by: Anton Volkov --- dokimi/docs/repo-evaluation-contract.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 945ab89c9..ebd1da428 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1013,9 +1013,10 @@ python cli.py repo compare repo-runs/pilot --baseline guidance:none --metric roo `ceiling` (both arms at 1.0 or the task's `score.max` on every paired task), `floor` (both at 0 on every one), `near_ceiling` (the baseline at the top on all but at most one of 2+ tasks and the treatment nowhere below - it), else `null`; a grader failure in either arm also gives `null`, as does an agent - outage (an error result with no turn count or at most one turn, or a harness that exited - non-zero without reporting a turn and was not killed at its timeout): neither is a measurement. A pair + it), else `null`; a grader failure in either arm also gives `null`, as does + an agent outage (an error result with no turn count or at most one turn, or + a harness that exited non-zero without reporting a turn and was not killed + at its timeout): neither is a measurement. A pair is at the top only when both arms reach it and tie: a score past `score.max` is an effect, not a top. The text prints a note under a flagged comparison. A zero delta there means no room to move, not no effect. Cost, turns and From e077e13018374fbd705fa256ca949f2dbe383753 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 22 Oct 2026 11:40:25 -0700 Subject: [PATCH 19/20] test(repo): pin the sign of rc, the bool guard and the turn limit of the agent outage rule Signed-off-by: Nikolay Petrov --- dokimi/tests/test_repo_compare.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dokimi/tests/test_repo_compare.py b/dokimi/tests/test_repo_compare.py index 7e19a729c..4282ca7c1 100644 --- a/dokimi/tests/test_repo_compare.py +++ b/dokimi/tests/test_repo_compare.py @@ -234,7 +234,7 @@ def test_score_headroom_is_measured_against_score_max(tmp_path): assert _row(compare([e], "guidance:none"), "score")["headroom"] is None -@pytest.mark.parametrize("turns,expected", [(None, None), (1, None), (5, "floor"), ("1", None)]) +@pytest.mark.parametrize("turns,expected", [(None, None), (1, None), (2, "floor"), (5, "floor"), ("1", None)]) def test_an_agent_outage_is_unmeasured_not_a_floor(tmp_path, turns, expected): b = tmp_path / "b1" for i in range(3): @@ -245,6 +245,8 @@ def test_an_agent_outage_is_unmeasured_not_a_floor(tmp_path, turns, expected): @pytest.mark.parametrize("rc,turns,timed_out,expected", [ (1, None, False, None), (137, None, False, None), # exited non-zero, no result record: the agent never ran + (-9, None, False, None), # killed by a signal: Completed.returncode is negative + (True, None, False, "floor"), # a stray bool is not an exit code (0, None, False, "floor"), (None, None, False, "floor"), # exit 0 or no rc recorded: not an outage (1, 5, False, "floor"), # it reported turns: an attempt (137, None, True, "floor"), # killed at its timeout: an attempt From 050c64208fadd7fefb9400ebffeb3340d7ea5a7a Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Fri, 23 Oct 2026 12:35:06 +0000 Subject: [PATCH 20/20] Rewrite the unmeasured-cell comment in _task_values Signed-off-by: Anton Volkov --- dokimi/dokimi/repos/compare.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index 886c179ac..aacc38c7a 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -515,9 +515,7 @@ def _task_values(cells: list[dict[str, Any]], metrics: tuple[str, ...]) -> dict[ # If any cell has unknown max, or maxes differ, top is None; otherwise take the single value has_unknown_max = any(v is None for v in score_max_vals) top = (maxes.pop() if len(maxes) == 1 and not has_unknown_max else None) if score_task else 1.0 - # unmeasured: a grader failure, or an agent outage (an error result with no or one turn, or a non-zero exit - # with no turn count); either way - # its 0 is not a measurement + # unmeasured: a grader failure, or an agent outage (see _agent_outage); either way its 0 is not a measurement grader_failed = any(r["grade"]["status"] == "grader_failure" or _agent_outage(r) for r in recs) out[key] = {"kind": "score" if score_task else "gate", "grader_failed": grader_failed, "top": top,