diff --git a/dokimi/changelog.d/20261018_120000_napetrov_feat-repo-compare-ceiling.md b/dokimi/changelog.d/20261018_120000_napetrov_feat-repo-compare-ceiling.md new file mode 100644 index 000000000..88481d503 --- /dev/null +++ b/dokimi/changelog.d/20261018_120000_napetrov_feat-repo-compare-ceiling.md @@ -0,0 +1,3 @@ +### Added + +- **`repo compare` flags gate and score comparisons with no headroom** — each comparison gains `headroom`: `"ceiling"` when both arms are at the top (gate 1.0, the task's `score.max`) on every paired task, `"floor"` when both are at 0 on every one, `"near_ceiling"` when the baseline is at the top on all but at most one of 2+ paired tasks and the treatment is nowhere below the baseline, else `null`; arms whose tops differ, and a grader failure or an agent outage (error result with no turn count or at most one turn, or a non-zero exit with no turn count that is not a timeout) in either arm, get `null`; a batch without a recorded top is never `ceiling`. A pair is at the top only when both arms reach it and tie: a score past `score.max` is an effect, not a top. The text report prints a note under the comparison. A zero delta on such a task set is no headroom, not no effect. p values and Holm's correction are unchanged. `result.json` now records the task's `score_max`; score rows of batches run before this have no known top and are never flagged `ceiling`. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index f3fbc3989..ebd1da428 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1009,7 +1009,22 @@ python cli.py repo compare repo-runs/pilot --baseline guidance:none --metric roo task), the tasks up, down and tied, and a two-sided sign-flip permutation p (exact up to 16 changed tasks; at one cell per task and arm it equals McNemar's exact test). Holm's correction runs over the gate and score comparisons of one call; the - rest are descriptive. `BATCH/ARM` takes an arm from one batch only, so two + rest are descriptive. A gate or score comparison carries `headroom`: + `ceiling` (both arms at 1.0 or the task's `score.max` on every paired + task), `floor` (both at 0 on every one), `near_ceiling` (the baseline at + the top on all but at most one of 2+ tasks and the treatment nowhere below + it), else `null`; a grader failure in either arm also gives `null`, as does + an agent outage (an error result with no turn count or at most one turn, or + a harness that exited non-zero without reporting a turn and was not killed + at its timeout): neither is a measurement. A pair + is at the top only when both arms reach it and tie: a score past `score.max` + is an effect, not a top. The text prints a note under a flagged comparison. A + zero delta there means no room to move, not no effect. Cost, turns and + metric rows have `headroom` `null`. + The top of a score task is the `score.max` that `repo run` records in + `result.json` as `score_max`: a batch run before that has no known top, so + its score rows are never `ceiling`, and arms whose tops differ are on two + scales and get `null`. `BATCH/ARM` takes an arm from one batch only, so two batches of the same arm (for example two revisions of the guidance) compare too; two selectors of the same cells are refused. A `--metric` no finished cell reports as a number or boolean is refused. Tasks left out are listed as diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index 48ed1f61f..aacc38c7a 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -28,6 +28,19 @@ the gate and score comparisons of one invocation; cost, turns and metrics are descriptive and keep their raw p. +A gate or score comparison also says whether its tasks had room to move (``headroom``), since a zero delta +on a task set nobody can fail, or nobody can pass, is not evidence of no effect. A task's top is 1.0 for +gate and the task's ``score.max`` for score (recorded in ``result.json``; a batch without it has no known +top, so its score rows are never at ceiling). A pair is at the top only when both arms reach it and tie: a +score past ``score.max`` is an effect, not a top. ``ceiling``: both arms at the top on every paired task; +``floor``: both arms at 0 on every paired task; ``near_ceiling``: neither, at least 2 paired tasks, the +baseline at the top on all but at most one, and the treatment nowhere below the baseline, so at most one task +can improve and a gain resting on one task has a sign-flip p of 1. A pair whose baseline and treatment tops +differ is on two scales. A grader failure or an agent outage (an error result with no turn count or at most +one turn, or a non-zero exit with no turn count that is not a timeout) in either arm scores a 0 that says the +measurement failed, not that the task had no room. Any other case has no flag: ``None``. The flag is a note; +p and Holm are unchanged. + Each selector's batches are reported with the stamp ``repo run`` pins a batch to (``tree``, ``eval_dir`` digest, ``image``) and the harness versions of its cells. A warning is printed, not an error, when a bare ``ARM`` pools cells of @@ -290,8 +303,42 @@ def _compare_model(treatment: str, model: str, base_vals: dict[tuple[str, str], else: pairs.append(pair) if pairs: + flag = None + if measure in PRIMARY: + tops = [(base_vals[(model, p["task"])]["top"], treat_vals[(model, p["task"])]["top"]) for p in pairs] + failed = [base_vals[(model, p["task"])]["grader_failed"] + or treat_vals[(model, p["task"])]["grader_failed"] for p in pairs] + flag = headroom(pairs, tops, failed) rows.append({"treatment": treatment, "model": model, "measure": measure, - **paired_summary(pairs, measure, seed=seed)}) + **paired_summary(pairs, measure, seed=seed), "headroom": flag}) + + +def headroom(pairs: list[dict[str, Any]], tops: list[tuple[float | None, float | None]], + failed: list[bool] | None = None) -> str | None: + """``ceiling``, ``floor``, ``near_ceiling`` or ``None`` for one gate or score comparison (module docstring). + + ``tops`` holds each pair's (baseline, treatment) top value, ``None`` where it is not known; ``failed`` + marks the pairs with a grader failure or an agent outage in either arm (its 0 is not a measurement). + """ + if any(b != t for b, t in tops): + return None # two scales: neither "at the top" nor the delta compares + if failed and any(failed): + return None # its 0 is not a measurement, at either end (a score.max of 0 would read it as the top) + + def at_top(value: float, top: float | None) -> bool: + return top is not None and value >= top - EPS + + # score.max is what the oracle must reach, not a bound: a pair that moved past it is an effect, not a top + base_top = [at_top(p["baseline"], b) and abs(p["treatment"] - p["baseline"]) <= EPS + for p, (b, _) in zip(pairs, tops, strict=True)] + if all(base_top) and all(at_top(p["treatment"], t) for p, (_, t) in zip(pairs, tops, strict=True)): + return "ceiling" + if all(abs(p["baseline"]) <= EPS and abs(p["treatment"]) <= EPS for p in pairs): + return "floor" + if (len(pairs) >= 2 and base_top.count(False) <= 1 + and all(p["treatment"] >= p["baseline"] - EPS for p in pairs)): + return "near_ceiling" + return None def _unpaired(treatment: str, model: str, task: str, measure: str | None, reason: str) -> dict[str, Any]: @@ -409,6 +456,8 @@ def format_compare(summary: dict[str, Any]) -> str: if moved: lines.append(f" {r['measure']} changed: " + ", ".join( f"{t['task']} {t['delta']:+.2f}" for t in sorted(moved, key=lambda t: t["delta"]))) + if r["headroom"]: + lines.append(f" {r['measure']} {_headroom_note(r)}") if summary["unpaired"]: shown = summary["unpaired"][:10] lines.append("unpaired: " + ", ".join( @@ -422,6 +471,17 @@ def format_compare(summary: dict[str, Any]) -> str: return "\n".join(lines) +def _headroom_note(row: dict[str, Any]) -> str: + n, gate = row["n_tasks"], row["measure"] == "gate" + if row["headroom"] == "ceiling": + return (f"ceiling: both arms {'pass' if gate else 'reach score.max on'} all {n} paired tasks; " + "no headroom to show an effect") + if row["headroom"] == "floor": + return f"floor: both arms {'fail' if gate else 'score 0 on'} all {n} paired tasks; no headroom to show an effect" + return (f"near ceiling: the baseline {'passes' if gate else 'reaches score.max on'} all but at most one of " + f"{n} paired tasks; too few can improve to show a gain") + + def _format_header(summary: dict[str, Any]) -> list[str]: lines = [f"baseline {summary['baseline']}: {summary['cells']} finished cells in " f"{len(summary['batches'])} batch(es), {len(summary['missing'])} missing"] @@ -450,12 +510,33 @@ def _task_values(cells: list[dict[str, Any]], metrics: tuple[str, ...]) -> dict[ else: found["gate"] = [float(bool(r["grade"]["success"])) for r in recs] nums = {measure: _finite(vals) for measure, vals in found.items()} - out[key] = {"kind": "score" if score_task else "gate", + score_max_vals = [r.get("score_max") for r in recs] + maxes = set(_finite(score_max_vals)) # cells of two eval trees may differ + # If any cell has unknown max, or maxes differ, top is None; otherwise take the single value + has_unknown_max = any(v is None for v in score_max_vals) + top = (maxes.pop() if len(maxes) == 1 and not has_unknown_max else None) if score_task else 1.0 + # unmeasured: a grader failure, or an agent outage (see _agent_outage); either way its 0 is not a measurement + grader_failed = any(r["grade"]["status"] == "grader_failure" or _agent_outage(r) for r in recs) + out[key] = {"kind": "score" if score_task else "gate", "grader_failed": grader_failed, + "top": top, "n": {measure: len(v) for measure, v in nums.items()}, "values": {measure: (sum(v) / len(v) if v else None) for measure, v in nums.items()}} return out +def _agent_outage(rec: dict[str, Any]) -> bool: + """A cell whose agent never ran: an error result with no turn count or at most one turn, or a harness that + exited non-zero before it reported any turn (no result record, so no ``is_error``) and was not killed for + its timeout: an agent killed at its timeout did attempt the task.""" + agent = rec.get("agent") or {} + turns = _finite([agent.get("num_turns")]) # a count that is not a number is no count, as in the turns row + if agent.get("is_error") and (not turns or turns[0] <= 1): + return True + rc = agent.get("rc") + crashed = isinstance(rc, int) and not isinstance(rc, bool) and rc != 0 + return crashed and not turns and not agent.get("timed_out") + + def _grade_metrics(path: Path, names: tuple[str, ...]) -> dict[str, float]: """The requested numeric or boolean ``metrics`` of a cell's ``grade.json``; absent ones are left out.""" try: diff --git a/dokimi/dokimi/repos/runner.py b/dokimi/dokimi/repos/runner.py index ad7750eae..1fb146853 100644 --- a/dokimi/dokimi/repos/runner.py +++ b/dokimi/dokimi/repos/runner.py @@ -226,7 +226,8 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, record: dict[str, Any] = {"task": task.id, "family": task.family, "arm": arm, "model": opts.model, "model_resolved": meta.get("model_resolved"), "rep": rep, "harness": opts.profile.name, "harness_version": meta["harness_version"], - "base_date": meta["base_date"], "score_task": bool(task.score), "agent": agent, "grade": grade} + "base_date": meta["base_date"], "score_task": bool(task.score), + "score_max": task.score["max"] if task.score else None, "agent": agent, "grade": grade} if any(uptake_of): # an arm that added something to use: did the agent use it (#41 D4) record["uptake"] = arm_uptake(events, *uptake_of, WORKSPACE) try: diff --git a/dokimi/tests/test_repo_compare.py b/dokimi/tests/test_repo_compare.py index a63b9c740..4282ca7c1 100644 --- a/dokimi/tests/test_repo_compare.py +++ b/dokimi/tests/test_repo_compare.py @@ -21,7 +21,9 @@ def _cell(batch: Path, task: str, arm: str, rep: int, *, success: bool | None = True, score: float | None = None, failure: bool = False, cost: float | None = 0.1, turns: int = 10, metrics: dict | None = None, - model: str = "m-1", harness_version: str = "1.0", model_resolved: str | list[str] | None = None) -> None: + model: str = "m-1", harness_version: str = "1.0", score_max: float | None = 1.0, + model_resolved: str | list[str] | None = None, is_error: bool = False, rc: int | None = None, + timed_out: bool = False) -> None: rd = batch / f"{task}__{arm.replace(':', '-')}__{model}__r{rep}" rd.mkdir(parents=True) score_task = score is not None @@ -29,8 +31,9 @@ def _cell(batch: Path, task: str, arm: str, rep: int, *, success: bool | None = "success": False if failure else (None if score_task else success), "score": None if failure else score, "score_only": score_task and not failure, "reasons": []} rec = {"task": task, "arm": arm, "model": model, "rep": rep, "score_task": score_task, "grade": grade, - "harness_version": harness_version, "model_resolved": model_resolved, - "agent": {"cost_usd": cost, "num_turns": turns, "timed_out": False, "is_error": False, + "harness_version": harness_version, "score_max": score_max if score_task else None, + "model_resolved": model_resolved, + "agent": {"cost_usd": cost, "num_turns": turns, "timed_out": timed_out, "is_error": is_error, "rc": rc, "over_budget": False, "network": "none"}} (rd / "meta.json").write_text("{}") (rd / "result.json").write_text(json.dumps(rec)) @@ -171,6 +174,259 @@ def test_score_tasks_are_their_own_measure_and_a_grader_failure_scores_zero(tmp_ assert _row(s, "gate")["n_tasks"] == 1 # the gated task, not the score task +@pytest.mark.parametrize("base,treat,expected", [ + ((True, True, True), (True, True, True), "ceiling"), + ((False, False, False), (False, False, False), "floor"), + ((True, True, False), (True, True, True), "near_ceiling"), # one task can move up + ((True, True, True), (True, False, True), None), # a loss is an effect, not "too few can improve" + ((True, True, True), (False, False, False), None), # the treatment collapsed + ((True, False, False), (True, True, True), None), + ((False, True, False), (False, True, False), None), # all tied, but not all at one end +]) +def test_gate_headroom_flags_a_task_set_that_cannot_move(tmp_path, base, treat, expected): + b = tmp_path / "b1" + for i, (was, now) in enumerate(zip(base, treat, strict=True)): + _cell(b, f"fix_{i}", "guidance:none", 1, success=was) + _cell(b, f"fix_{i}", "guidance:raw", 1, success=now) + s = compare([b], "guidance:none") + gate = _row(s, "gate") + assert gate["headroom"] == expected and _row(s, "cost_usd")["headroom"] is None + text = format_compare(s) + if expected == "ceiling": + assert gate["p"] is None and gate["p_holm"] is None # the flag is a note: p and Holm are unchanged + assert "gate ceiling: both arms pass all 3 paired tasks; no headroom to show an effect" in text + elif expected == "floor": + assert "gate floor: both arms fail all 3 paired tasks" in text + elif expected == "near_ceiling": + assert "gate near ceiling: the baseline passes all but at most one of 3 paired tasks" in text + else: + assert "ceiling" not in text and "floor" not in text + + +def test_score_headroom_is_measured_against_score_max(tmp_path): + b = tmp_path / "b1" + for i in range(2): + _cell(b, f"review_{i}", "guidance:none", 1, score=4.0, score_max=4.0) + _cell(b, f"review_{i}", "guidance:raw", 1, score=4.0, score_max=4.0) + _cell(b, f"old_{i}", "guidance:none", 1, score=1.0, score_max=None) # a batch from before score_max + _cell(b, f"old_{i}", "guidance:raw", 1, score=1.0, score_max=None) + assert _row(compare([b], "guidance:none"), "score")["headroom"] is None # old_* has no known top + for i in range(2): + for arm in ("guidance:none", "guidance:raw"): + (b / f"old_{i}__{arm.replace(':', '-')}__m-1__r1/result.json").unlink() + s = compare([b], "guidance:none") + assert _row(s, "score")["headroom"] == "ceiling" + assert "score ceiling: both arms reach score.max on all 2 paired tasks" in format_compare(s) + c = tmp_path / "b2" + for i in range(2): + _cell(c, f"review_{i}", "guidance:none", 1, score=0.0) + _cell(c, f"review_{i}", "guidance:raw", 1, score=0.0, failure=True) # a grader failure scores 0 + assert _row(compare([c], "guidance:none"), "score")["headroom"] is None # the zeros include a grader failure + d = tmp_path / "b3" + for i in range(2): + _cell(d, f"review_{i}", "guidance:none", 1, score=0.0) + _cell(d, f"review_{i}", "guidance:raw", 1, score=0.0) + assert _row(compare([d], "guidance:none"), "score")["headroom"] == "floor" + e = tmp_path / "b4" + for i in range(2): + _cell(e, f"review_{i}", "guidance:none", 1, score=0.0, failure=True) + _cell(e, f"review_{i}", "guidance:raw", 1, score=0.0, failure=True) + assert _row(compare([e], "guidance:none"), "score")["headroom"] is None + + +@pytest.mark.parametrize("turns,expected", [(None, None), (1, None), (2, "floor"), (5, "floor"), ("1", None)]) +def test_an_agent_outage_is_unmeasured_not_a_floor(tmp_path, turns, expected): + b = tmp_path / "b1" + for i in range(3): + for arm in ("guidance:none", "guidance:raw"): + _cell(b, f"fix_{i}", arm, 1, success=False, turns=turns, is_error=True) + assert _row(compare([b], "guidance:none"), "gate")["headroom"] == expected + + +@pytest.mark.parametrize("rc,turns,timed_out,expected", [ + (1, None, False, None), (137, None, False, None), # exited non-zero, no result record: the agent never ran + (-9, None, False, None), # killed by a signal: Completed.returncode is negative + (True, None, False, "floor"), # a stray bool is not an exit code + (0, None, False, "floor"), (None, None, False, "floor"), # exit 0 or no rc recorded: not an outage + (1, 5, False, "floor"), # it reported turns: an attempt + (137, None, True, "floor"), # killed at its timeout: an attempt +]) +def test_a_harness_that_exits_non_zero_with_no_result_is_an_agent_outage(tmp_path, rc, turns, timed_out, expected): + b = tmp_path / "b1" + for i in range(3): + for arm in ("guidance:none", "guidance:raw"): + _cell(b, f"fix_{i}", arm, 1, success=False, turns=turns, rc=rc, timed_out=timed_out) + assert _row(compare([b], "guidance:none"), "gate")["headroom"] == expected + b2 = tmp_path / "b2" + for i in range(3): + for arm in ("guidance:none", "guidance:raw"): + _cell(b2, f"r_{i}", arm, 1, score=0.0, turns=turns, rc=rc, timed_out=timed_out) + assert _row(compare([b2], "guidance:none"), "score")["headroom"] == expected + + +def test_score_headroom_is_unknown_when_the_arms_have_different_score_max(tmp_path): + b = tmp_path / "b1" + for i in range(2): + _cell(b, f"r{i}", "guidance:none", 1, score=4.0, score_max=4.0) + _cell(b, f"r{i}", "guidance:raw", 1, score=1.0, score_max=1.0) + s = compare([b], "guidance:none") + row = _row(s, "score") + assert row["delta"]["mean"] == -3.0 and row["headroom"] is None + + +def test_score_floor_is_unknown_when_the_arms_have_different_score_max(tmp_path): + """Both arms score 0 on every task, so only the tops decide: the treatment's top (1.0) must count, not the baseline's.""" + b = tmp_path / "b1" + for i in range(2): + _cell(b, f"r{i}", "guidance:none", 1, score=0.0, score_max=4.0) + _cell(b, f"r{i}", "guidance:raw", 1, score=0.0, score_max=1.0) + assert _row(compare([b], "guidance:none"), "score")["headroom"] is None + + +def test_score_headroom_unknown_when_pooling_legacy_and_current_cells(tmp_path): + """Regression: when legacy cells (score_max=None) and current cells (score_max=1.0) appear for the same task, + do not report ceiling if the legacy cells' maxima are unknown.""" + b = tmp_path / "b1" + for i in range(2): + # Current cells with known score_max + _cell(b, f"task_{i}", "guidance:none", 1, score=1.0, score_max=1.0) + _cell(b, f"task_{i}", "guidance:raw", 1, score=1.0, score_max=1.0) + # Legacy cells from before score_max tracking, same task IDs + _cell(b, f"task_{i}", "guidance:none", 2, score=1.0, score_max=None) + _cell(b, f"task_{i}", "guidance:raw", 2, score=1.0, score_max=None) + # Both arms reach 1.0, but legacy cells have unknown maxima: headroom should be None, not "ceiling" + assert _row(compare([b], "guidance:none"), "score")["headroom"] is None + + +def test_a_grader_failure_in_either_arm_is_not_floor(tmp_path): + b = tmp_path / "b1" + for i in range(2): + _cell(b, f"review_{i}", "guidance:raw", 1, score=0.0) + _cell(b, "review_0", "guidance:none", 1, score=0.0) + _cell(b, "review_1", "guidance:none", 1, score=0.0, failure=True) # baseline only: the treatment is clean + assert _row(compare([b], "guidance:none"), "score")["headroom"] is None + + +def test_a_gate_task_with_a_grader_failure_on_one_rep_is_not_floor(tmp_path): + b = tmp_path / "b1" + for arm in ("guidance:none", "guidance:raw"): + _cell(b, "fix_a", arm, 1, success=False) + _cell(b, "fix_a", arm, 2, success=False, failure=True) # the second rep only + _cell(b, "fix_b", arm, 1, success=False) + _cell(b, "fix_b", arm, 2, success=False) + assert _row(compare([b], "guidance:none"), "gate")["headroom"] is None + + +def test_a_grader_failure_on_the_one_task_that_can_move_is_not_near_ceiling(tmp_path): + b = tmp_path / "b1" + _cell(b, "r0", "guidance:none", 1, score=1.0, score_max=1.0) + _cell(b, "r0", "guidance:raw", 1, score=1.0, score_max=1.0) + _cell(b, "r1", "guidance:none", 1, score=0.0, score_max=1.0, failure=True) # not measured, not "room" + _cell(b, "r1", "guidance:raw", 1, score=1.0, score_max=1.0) + assert _row(compare([b], "guidance:none"), "score")["headroom"] is None + c = tmp_path / "b2" + _cell(c, "r0", "guidance:none", 1, score=1.0, score_max=1.0) + _cell(c, "r0", "guidance:raw", 1, score=1.0, score_max=1.0) + _cell(c, "r1", "guidance:none", 1, score=0.0, score_max=1.0) # the same, measured as a zero + _cell(c, "r1", "guidance:raw", 1, score=1.0, score_max=1.0) + assert _row(compare([c], "guidance:none"), "score")["headroom"] == "near_ceiling" + + +@pytest.mark.parametrize("pairs,tops,failed,expected", [ + # two all-zero pairs; one grader failure is not a floor (any, not all) + ([{"baseline": 0.0, "treatment": 0.0}] * 2, [(1.0, 1.0)] * 2, [True, False], None), + # a legacy baseline (unknown top) against a known one: two scales, whatever the values + ([{"baseline": 0.0, "treatment": 0.0}] * 2, [(None, 4.0), (4.0, 4.0)], None, None), + # tops that differ on one pair + ([{"baseline": 4.0, "treatment": 4.0}, {"baseline": 1.0, "treatment": 1.0}], [(4.0, 4.0), (4.0, 1.0)], None, None), + # equal tops on every pair and all zeros: a floor + ([{"baseline": 0.0, "treatment": 0.0}] * 2, [(4.0, 4.0), (4.0, 4.0)], None, "floor"), + # the one non-top pair is a grader failure: no near_ceiling + ([{"baseline": 1.0, "treatment": 1.0}, {"baseline": 0.5, "treatment": 1.0}], [(1.0, 1.0)] * 2, + [False, True], None), + ([{"baseline": 1.0, "treatment": 1.0}, {"baseline": 0.5, "treatment": 1.0}], [(1.0, 1.0)] * 2, + [False, False], "near_ceiling"), + # one arm at 0 is not a floor; one task is not near_ceiling; a small loss is still a loss + ([{"baseline": 0.0, "treatment": 1.0}] * 3, [(1.0, 1.0)] * 3, None, None), + ([{"baseline": 0.0, "treatment": 1.0}], [(1.0, 1.0)], None, None), + ([{"baseline": 1.0, "treatment": 1.0}, {"baseline": 1.0, "treatment": 0.7}], [(1.0, 1.0)] * 2, None, None), +]) +def test_headroom_on_mixed_tops_and_grader_failures(pairs, tops, failed, expected): + assert compare_mod.headroom(pairs, tops, failed) == expected + + +@pytest.mark.parametrize("pairs,expected", [ + # both arms 0.05 short of the top: a task that still has room is no ceiling, nor near_ceiling + ([{"baseline": 0.95, "treatment": 0.95}] * 2, None), + # the baseline at the top on both tasks, the treatment 0.05 below it on one: a small loss, not near_ceiling + ([{"baseline": 1.0, "treatment": 1.0}, {"baseline": 1.0, "treatment": 0.95}], None), + # the treatment 0.3 past score.max on both tasks: an effect, not a top + ([{"baseline": 1.0, "treatment": 1.3}] * 2, None), +]) +def test_headroom_tolerances_are_exact_not_coarse(pairs, expected): + assert compare_mod.headroom(pairs, [(1.0, 1.0)] * len(pairs), None) == expected + + +def test_gate_headroom_top_is_one_not_half(tmp_path): + # both arms pass half of their reps on every task: a rep mean of 0.5 is not at the gate top of 1.0 + b = tmp_path / "b1" + for i in range(2): + _cell(b, f"fix_{i}", "guidance:none", 1, success=True) + _cell(b, f"fix_{i}", "guidance:none", 2, success=False) + _cell(b, f"fix_{i}", "guidance:raw", 1, success=True) + _cell(b, f"fix_{i}", "guidance:raw", 2, success=False) + assert _row(compare([b], "guidance:none"), "gate")["headroom"] is None + + +def test_score_headroom_allows_for_float_means_and_needs_one_known_top(tmp_path): + b = tmp_path / "b1" + for i in range(2): + for arm in ("guidance:none", "guidance:raw"): + for rep in (1, 2, 3): # a mean of 0.6999999999999998 + _cell(b, f"r{i}", arm, rep, score=0.7, score_max=0.7) + assert _row(compare([b], "guidance:none"), "score")["headroom"] == "ceiling" + c = tmp_path / "b2" + for i in range(2): + for arm in ("guidance:none", "guidance:raw"): # two eval trees in one task: no single top + _cell(c, f"r{i}", arm, 1, score=4.0, score_max=1.0) + _cell(c, f"r{i}", arm, 2, score=4.0, score_max=4.0) + assert _row(compare([c], "guidance:none"), "score")["headroom"] is None + + +def test_cost_and_turns_rows_never_carry_a_headroom_flag(tmp_path): + b = tmp_path / "b1" + for i in range(2): + for arm in ("guidance:none", "guidance:raw"): # values that equal the gate's top + _cell(b, f"fix_{i}", arm, 1, cost=1.0, turns=1) + s = compare([b], "guidance:none") + assert _row(s, "cost_usd")["headroom"] is None and _row(s, "turns")["headroom"] is None + + +@pytest.mark.parametrize("baseline,expected", [ + ((80.0,) * 8, None), # a gain past score.max on every task: an effect, not a ceiling + ((70.0,) + (80.0,) * 7, None), # nor near_ceiling + ((90.0,) * 8, "ceiling"), # both past score.max and tied: no movement +]) +def test_a_score_past_score_max_is_at_the_top_only_when_the_arms_tie(tmp_path, baseline, expected): + b = tmp_path / "b1" + for i, was in enumerate(baseline): + _cell(b, f"cov_{i}", "guidance:none", 1, score=was, score_max=80.0) + _cell(b, f"cov_{i}", "guidance:raw", 1, score=90.0, score_max=80.0) + s = compare([b], "guidance:none") + assert _row(s, "score")["headroom"] == expected + assert ("ceiling" in format_compare(s)) == (expected is not None) + + +@pytest.mark.parametrize("score_max", [0.0, -1.0]) +def test_a_grader_failure_is_not_ceiling_when_score_max_is_not_above_zero(tmp_path, score_max): + b = tmp_path / "b1" + for i in range(3): # a lower-is-better metric negated, say; the grader failed everywhere and scored 0 + _cell(b, f"r{i}", "guidance:none", 1, score=0.0, score_max=score_max, failure=True) + _cell(b, f"r{i}", "guidance:raw", 1, score=0.0, score_max=score_max, failure=True) + s = compare([b], "guidance:none") + assert _row(s, "score")["headroom"] is None and "ceiling" not in format_compare(s) + + def test_unpaired_tasks_are_listed_not_compared(tmp_path): b = tmp_path / "b1" _cell(b, "fix_a", "guidance:none", 1) diff --git a/dokimi/tests/test_repo_run.py b/dokimi/tests/test_repo_run.py index 43161e66e..d58284066 100644 --- a/dokimi/tests/test_repo_run.py +++ b/dokimi/tests/test_repo_run.py @@ -100,6 +100,18 @@ def test_a_run_answered_by_another_model_is_recorded_and_reported(setup, monkeyp assert summary["model_mismatch"] == {"m-1": ["m-0"]} +def test_result_json_records_the_tasks_own_score_max_and_none_for_a_gate_task(setup): + task_json = setup["root"] / ".agent-eval/tasks/review_add/task.json" + task = json.loads(task_json.read_text()) + task["score"]["max"] = 4.0 # not the 1.0 default of `repo compare` + task_json.write_text(json.dumps(task)) + _run(setup, "--arm", "guidance:raw", source=str(setup["root"])) + batch = setup["out"] / "b1" + review = json.loads((batch / "review_add__guidance-raw__m-1__r1/result.json").read_text()) + gate = json.loads((batch / "fix_add__guidance-raw__m-1__r1/result.json").read_text()) + assert review["score_max"] == 4.0 and gate["score_max"] is None + + @pytest.mark.outcome("cjm-repo-run-resume") def test_already_graded_cells_are_skipped_unless_forced(setup, capsys): _run(setup, "--arm", "guidance:raw", "--task", "fix_add") @@ -127,6 +139,7 @@ def test_score_only_task_and_missing_cells(setup, monkeypatch, capsys): batch = setup["out"] / "b1" review = json.loads((batch / "review_add__guidance-raw__m-1__r1/result.json").read_text()) assert review["grade"]["score_only"] and review["grade"]["score"] == 1.0 and review["grade"]["success"] is None + assert review["score_max"] == 1.0 # the task's score.max, for `repo compare`'s headroom (batch / "fix_add__guidance-raw__m-1__r3/result.json").unlink() # crashed before it finished shutil.rmtree(batch / "fix_add__guidance-raw__m-1__r2") # never started (only in the plan) summary = _report(setup)