From 22262523a3fde46b303f7140aebbb7719ec3912d Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 28 Oct 2026 17:38:34 -0700 Subject: [PATCH 01/47] feat(eval): flag arms items at the ceiling or floor with no headroom The headroom command reads arms.v1 runs of one question set and flags an item as a ceiling when its no-skill arm's mean is at least 0.9, or as a floor when every arm's mean is at most 0. An item with fewer than three runs behind it is provisional. The table goes to stdout; -o writes a headroom.v1 artifact. Flags are descriptive: no score or gate changes. Signed-off-by: Nikolay Petrov --- ...1028_120000_napetrov_feat-eval-headroom.md | 3 + dokimi/docs/evaluating-treatments.md | 22 ++ dokimi/dokimi/artifacts.py | 1 + dokimi/dokimi/commands/evaluate.py | 4 + dokimi/dokimi/commands/headroom.py | 59 ++++++ dokimi/dokimi/eval/headroom.py | 199 ++++++++++++++++++ dokimi/dokimi/schemas/headroom.v1.json | 54 +++++ .../tests/fixtures/artifacts/headroom.v1.json | 33 +++ dokimi/tests/test_headroom.py | 148 +++++++++++++ 9 files changed, 523 insertions(+) create mode 100644 dokimi/changelog.d/20261028_120000_napetrov_feat-eval-headroom.md create mode 100644 dokimi/dokimi/commands/headroom.py create mode 100644 dokimi/dokimi/eval/headroom.py create mode 100644 dokimi/dokimi/schemas/headroom.v1.json create mode 100644 dokimi/tests/fixtures/artifacts/headroom.v1.json create mode 100644 dokimi/tests/test_headroom.py diff --git a/dokimi/changelog.d/20261028_120000_napetrov_feat-eval-headroom.md b/dokimi/changelog.d/20261028_120000_napetrov_feat-eval-headroom.md new file mode 100644 index 000000000..6cec77380 --- /dev/null +++ b/dokimi/changelog.d/20261028_120000_napetrov_feat-eval-headroom.md @@ -0,0 +1,3 @@ +### Added + +- **`eval headroom` flags arms items with no headroom** — reads one or more `arms.v1` files of the same question set and flags an item as a `ceiling` when the no-skill arm's mean is at least `--ceiling` (default 0.9, on 0-1), or as a `floor` when every arm's mean is at most `--floor` (default 0). An item with fewer than `--min-reps` runs (default 3) behind it is `provisional`. The human table goes to stdout; `-o` writes a `headroom.v1` artifact with every item. The flags change no score. diff --git a/dokimi/docs/evaluating-treatments.md b/dokimi/docs/evaluating-treatments.md index 691ab695a..1f9a97d33 100644 --- a/dokimi/docs/evaluating-treatments.md +++ b/dokimi/docs/evaluating-treatments.md @@ -723,6 +723,28 @@ Two gates are deliberately statistical rather than absolute: questions both arms scored. It pools questions; a mean of per-cell means differs slightly when the sets have different sizes — say which one a report uses. +## Headroom: items a comparison cannot learn from + +A skill cannot raise an item the no-skill arm already answers, and an item no arm +answers points at the question or its grader, not at the skill. `eval headroom` flags +both, from one or more arms files of the same question set (one file per repetition): + +```bash +python cli.py eval headroom run1/arms.json run2/arms.json run3/arms.json \ + --no-skill-arm baseline --ceiling 0.9 --floor 0 --min-reps 3 -o headroom.json +``` + +- **ceiling**: the no-skill arm's mean over the runs is at least `--ceiling` (0-1; the + 0-100 aggregate divided by 100). Default 0.9. +- **floor**: every arm's mean is at most `--floor`. Default 0. +- **provisional**: fewer than `--min-reps` runs scored the item (the smallest count over + its arms). Default 3. One file gives every item one rep, so every item is provisional. + +The table on stdout lists flagged and provisional items; `-o` writes a `headroom.v1` +artifact with every item. `--no-skill-arm` defaults to the baseline the runs declare, and +the runs must agree on it. Only arms runs are read. The flags are descriptive: they do +not change any score or any gate. + ## Scope and caveats - **Three levels of fidelity for skills/MCP.** (1) `skill:` / `mcp:` inject diff --git a/dokimi/dokimi/artifacts.py b/dokimi/dokimi/artifacts.py index 5731c0f46..510bf8d4f 100644 --- a/dokimi/dokimi/artifacts.py +++ b/dokimi/dokimi/artifacts.py @@ -57,6 +57,7 @@ "result": "result.v1", "repo_budget": "repo_budget.v1", "task_inspect": "task_inspect.v1", + "headroom": "headroom.v1", } # kind -> superseded versions still accepted on *load*, newest first. An artifact written under an diff --git a/dokimi/dokimi/commands/evaluate.py b/dokimi/dokimi/commands/evaluate.py index 9d3602f31..05f169cab 100644 --- a/dokimi/dokimi/commands/evaluate.py +++ b/dokimi/dokimi/commands/evaluate.py @@ -565,6 +565,10 @@ def register(sub, positive_int) -> None: answer_formats.register_eval(eval_sub, positive_int) # eval reference, eval native-import + from dokimi.commands import headroom + + headroom.register_eval(eval_sub) # eval headroom + # eval panel-score panel_p = eval_sub.add_parser("panel-score", help="Score answers using a multi-judge panel") panel_p.add_argument("--answers", required=True, help="Path to answers JSON file") diff --git a/dokimi/dokimi/commands/headroom.py b/dokimi/dokimi/commands/headroom.py new file mode 100644 index 000000000..620f52b33 --- /dev/null +++ b/dokimi/dokimi/commands/headroom.py @@ -0,0 +1,59 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""``eval headroom``: flag the ceiling, floor and provisional items of arms runs. + +Reads one or more ``arms.v1`` files of the same question set (one per repetition) and writes a +``headroom.v1`` artifact with ``-o``, always printing the human table. See +:mod:`dokimi.eval.headroom` for the rules. +""" + +from __future__ import annotations + +import argparse +import sys + +from dokimi import records + + +def cmd_eval_headroom(args: argparse.Namespace) -> None: + from dokimi.artifacts import save_artifact + from dokimi.eval.headroom import format_table, headroom + + runs = [] + for path in args.arms: + try: + runs.append(records.load(path)) + except records.RecordError as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(2) + try: + result = headroom(runs, no_skill_arm=args.no_skill_arm, ceiling=args.ceiling, + floor=args.floor, min_reps=args.min_reps) + except ValueError as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(2) + result["inputs"] = [str(p) for p in args.arms] + print(format_table(result)) + if args.output: + save_artifact("headroom", result, args.output) + print(f"Wrote headroom artifact to {args.output}") + + +def register_eval(eval_sub) -> None: + from dokimi.eval.headroom import DEFAULT_CEILING, DEFAULT_FLOOR, DEFAULT_MIN_REPS + + h = eval_sub.add_parser( + "headroom", + help="Flag arms items with no headroom: no-skill ceiling, all-zero floor, provisional reps") + h.add_argument("arms", nargs="+", help="arms.v1 JSON, one per repetition of the question set") + h.add_argument("--no-skill-arm", default=None, + help="Arm read as the no-skill control (default: the baseline the runs declare)") + h.add_argument("--ceiling", type=float, default=DEFAULT_CEILING, + help=f"Flag an item when the no-skill arm scores at least this, 0-1 (default {DEFAULT_CEILING})") + h.add_argument("--floor", type=float, default=DEFAULT_FLOOR, + help=f"Flag an item when every arm scores at most this, 0-1 (default {DEFAULT_FLOOR})") + h.add_argument("--min-reps", type=int, default=DEFAULT_MIN_REPS, dest="min_reps", + help=f"Items with fewer runs than this are provisional (default {DEFAULT_MIN_REPS})") + h.add_argument("-o", "--output", default=None, help="headroom.v1 JSON to write") + h.set_defaults(func=cmd_eval_headroom) diff --git a/dokimi/dokimi/eval/headroom.py b/dokimi/dokimi/eval/headroom.py new file mode 100644 index 000000000..9666116f7 --- /dev/null +++ b/dokimi/dokimi/eval/headroom.py @@ -0,0 +1,199 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Headroom: the items of an arms comparison that cannot tell a skill apart. + +An item is a **ceiling** when its no-skill arm already scores at least ``ceiling`` (0-1: the +judge's or the grader's 0-100 aggregate divided by 100). A skill cannot raise it, so the item +measures nothing about the skill. An item is a **floor** when every arm scores at most +``floor`` (default 0): no arm answers it, which points at the question, its checks or the +grader rather than at any skill. + +An item with fewer than ``min_reps`` runs behind it is ``provisional``: one run's flag may be +noise. Its reps are the smallest number of runs in which any of its arms was scored, so a single +artifact gives every item one rep. Each arm's score is the mean over the runs that scored it. + +The input is one or more ``arms.v1`` files (or any artifact :mod:`dokimi.records` reads as an +arms run), one per repetition of the same question set. Nothing here draws a confidence +interval; the paired statistics of a comparison stay in :mod:`dokimi.eval.stats`. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Sequence, Set, Tuple + +from dokimi.records import Run, arm_names + +DEFAULT_CEILING = 0.9 +DEFAULT_FLOOR = 0.0 +DEFAULT_MIN_REPS = 3 + +CEILING = "ceiling" +FLOOR = "floor" + + +@dataclass(frozen=True) +class ItemHeadroom: + question_id: str + reps: int + provisional: bool + no_skill: Optional[float] + arms: Dict[str, Optional[float]] + flag: Optional[str] + + def as_dict(self) -> Dict[str, Any]: + return { + "question_id": self.question_id, + "reps": self.reps, + "provisional": self.provisional, + "no_skill": self.no_skill, + "arms": self.arms, + "flag": self.flag, + } + + +def check_thresholds(ceiling: float, floor: float, min_reps: int) -> None: + """Refuse thresholds that cannot flag anything sensibly (ValueError).""" + if not 0.0 < ceiling <= 1.0: + raise ValueError(f"ceiling must be in (0, 1], not {ceiling!r}") + if not 0.0 <= floor < ceiling: + raise ValueError(f"floor must be in [0, ceiling), not {floor!r} (ceiling {ceiling!r})") + if min_reps < 1: + raise ValueError(f"min_reps must be at least 1, not {min_reps!r}") + + +def _mean(values: List[float]) -> Optional[float]: + return round(sum(values) / len(values), 4) if values else None + + +def _flag(no_skill: Optional[float], arms: Dict[str, Optional[float]], ceiling: float, floor: float) -> Optional[str]: + if no_skill is not None and no_skill >= ceiling: + return CEILING + if arms and all(v is not None and v <= floor for v in arms.values()): + return FLOOR + return None + + +def headroom( + runs: Sequence[Run], + *, + no_skill_arm: Optional[str] = None, + ceiling: float = DEFAULT_CEILING, + floor: float = DEFAULT_FLOOR, + min_reps: int = DEFAULT_MIN_REPS, +) -> Dict[str, Any]: + """Flag every item of ``runs`` (arms runs of one question set) by its headroom. + + ``no_skill_arm`` defaults to the baseline the runs declare. Raises ValueError for + thresholds that cannot be used, a run that is not an arms run, or runs that disagree on + the no-skill arm. + """ + check_thresholds(ceiling, floor, min_reps) + control = _no_skill_arm(runs, no_skill_arm) + scores, arms_of = _collect(runs) + + items: List[ItemHeadroom] = [] + for qid in sorted(scores, key=_sort_key): + per_arm = scores[qid] + arms = {arm: _mean(per_arm.get(arm, [])) for arm in sorted(arms_of[qid])} + reps = min((len(per_arm.get(arm, [])) for arm in arms_of[qid]), default=0) + no_skill = arms.get(control) + items.append(ItemHeadroom( + question_id=qid, + reps=reps, + provisional=reps < min_reps, + no_skill=no_skill, + arms=arms, + flag=_flag(no_skill, arms, ceiling, floor), + )) + + summary = { + "items": len(items), + "ceiling": sum(1 for i in items if i.flag == CEILING), + "floor": sum(1 for i in items if i.flag == FLOOR), + "provisional": sum(1 for i in items if i.provisional), + "flagged_provisional": sum(1 for i in items if i.flag and i.provisional), + "no_flag": sum(1 for i in items if i.flag is None), + } + return { + "no_skill_arm": control, + "thresholds": {"ceiling": ceiling, "floor": floor, "min_reps": min_reps}, + "runs": len(runs), + "items": [i.as_dict() for i in items], + "summary": summary, + } + + +def _no_skill_arm(runs: Sequence[Run], given: Optional[str]) -> str: + """The control arm: ``given``, or the one baseline all runs declare. Refuses anything else.""" + if not runs: + raise ValueError("no runs to read") + for run in runs: + if run.shape != "arms": + raise ValueError("headroom reads arms runs only; a binary answers or eval file was given") + if given is not None: + return given + declared = {run.baseline for run in runs} + only = next(iter(declared)) if len(declared) == 1 else None + if only is None: + raise ValueError("the runs do not declare one baseline arm; pass --no-skill-arm") + return only + + +def _collect(runs: Sequence[Run]) -> Tuple[Dict[str, Dict[str, List[float]]], Dict[str, Set[str]]]: + """Question id -> arm -> the per-run scores (0-1), and question id -> the arms it listed.""" + scores: Dict[str, Dict[str, List[float]]] = {} + arms_of: Dict[str, Set[str]] = {} + for run in runs: + for rec in run: + names = set(arm_names(rec)) | set(rec.scores) + arms_of.setdefault(rec.key, set()).update(names) + per_arm = scores.setdefault(rec.key, {}) + for arm in names: + value = rec.score(arm) + if value is not None: + per_arm.setdefault(arm, []).append(float(value) / 100.0) + return scores, arms_of + + +def _sort_key(qid: str): + return (0, int(qid), "") if qid.isdigit() else (1, 0, qid) + + +def format_table(result: Dict[str, Any]) -> str: + """The human-readable table: one row per flagged or provisional item, then the counts.""" + arm_names_ = sorted({a for item in result["items"] for a in item["arms"]}) + head = ["question", "reps", "no-skill", *arm_names_, "flag", "provisional"] + rows = [] + for item in result["items"]: + if item["flag"] is None and not item["provisional"]: + continue + cells = [ + str(item["question_id"]), str(item["reps"]), _fmt(item["no_skill"]), + *[_fmt(item["arms"].get(a)) for a in arm_names_], + item["flag"] or "-", "yes" if item["provisional"] else "no", + ] + rows.append(cells) + widths = [max(len(r[c]) for r in [head, *rows]) for c in range(len(head))] if rows else [len(h) for h in head] + lines = [_line(head, widths)] + [_line(r, widths) for r in rows] + if not rows: + lines.append("(no item is flagged or provisional)") + s = result["summary"] + th = result["thresholds"] + lines.append("") + lines.append( + f"{s['items']} items; no-skill arm {result['no_skill_arm']!r}; ceiling >= {th['ceiling']:g}, " + f"floor <= {th['floor']:g}, provisional below {th['min_reps']} reps") + lines.append( + f"ceiling {s['ceiling']}, floor {s['floor']}, provisional {s['provisional']} " + f"({s['flagged_provisional']} of them flagged), unflagged {s['no_flag']}") + return "\n".join(lines) + + +def _fmt(value: Optional[float]) -> str: + return "-" if value is None else f"{value:.2f}" + + +def _line(cells: List[str], widths: List[int]) -> str: + return " ".join(c.ljust(w) for c, w in zip(cells, widths, strict=True)).rstrip() diff --git a/dokimi/dokimi/schemas/headroom.v1.json b/dokimi/dokimi/schemas/headroom.v1.json new file mode 100644 index 000000000..69a00a55b --- /dev/null +++ b/dokimi/dokimi/schemas/headroom.v1.json @@ -0,0 +1,54 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/intel/dokimi/schemas/headroom.v1.json", + "title": "Arms headroom v1", + "description": "Which items of an arms comparison have no headroom: the no-skill arm already at the ceiling (or every arm at the floor), and items with too few repetitions to trust a flag. Written by `eval headroom`; reads arms.v1 files only.", + "type": "object", + "required": ["schema_version", "no_skill_arm", "thresholds", "runs", "items", "summary", "inputs"], + "additionalProperties": false, + "properties": { + "schema_version": {"const": "headroom.v1"}, + "no_skill_arm": {"type": "string", "description": "The control arm read as no-skill."}, + "thresholds": { + "type": "object", + "required": ["ceiling", "floor", "min_reps"], + "additionalProperties": false, + "properties": { + "ceiling": {"type": "number", "exclusiveMinimum": 0, "maximum": 1}, + "floor": {"type": "number", "minimum": 0}, + "min_reps": {"type": "integer", "minimum": 1} + } + }, + "runs": {"type": "integer", "minimum": 1, "description": "Number of arms.v1 inputs."}, + "inputs": {"type": "array", "items": {"type": "string"}}, + "items": { + "type": "array", + "items": { + "type": "object", + "required": ["question_id", "reps", "provisional", "no_skill", "arms", "flag"], + "additionalProperties": false, + "properties": { + "question_id": {"type": "string"}, + "reps": {"type": "integer", "minimum": 0, "description": "Smallest number of runs in which any of the item's arms was scored."}, + "provisional": {"type": "boolean", "description": "True when reps is below thresholds.min_reps."}, + "no_skill": {"type": ["number", "null"], "description": "Mean over runs of the no-skill arm, 0-1."}, + "arms": {"type": "object", "additionalProperties": {"type": ["number", "null"]}, "description": "Mean over runs of each arm, 0-1; null when no run scored it."}, + "flag": {"enum": ["ceiling", "floor", null]} + } + } + }, + "summary": { + "type": "object", + "required": ["items", "ceiling", "floor", "provisional", "flagged_provisional", "no_flag"], + "additionalProperties": false, + "properties": { + "items": {"type": "integer", "minimum": 0}, + "ceiling": {"type": "integer", "minimum": 0}, + "floor": {"type": "integer", "minimum": 0}, + "provisional": {"type": "integer", "minimum": 0}, + "flagged_provisional": {"type": "integer", "minimum": 0}, + "no_flag": {"type": "integer", "minimum": 0} + } + } + } +} diff --git a/dokimi/tests/fixtures/artifacts/headroom.v1.json b/dokimi/tests/fixtures/artifacts/headroom.v1.json new file mode 100644 index 000000000..874aa7f23 --- /dev/null +++ b/dokimi/tests/fixtures/artifacts/headroom.v1.json @@ -0,0 +1,33 @@ +{ + "schema_version": "headroom.v1", + "no_skill_arm": "baseline", + "thresholds": {"ceiling": 0.9, "floor": 0.0, "min_reps": 3}, + "runs": 1, + "inputs": ["arms.json"], + "items": [ + { + "question_id": "1", + "reps": 1, + "provisional": true, + "no_skill": 0.95, + "arms": {"baseline": 0.95, "skill": 0.94}, + "flag": "ceiling" + }, + { + "question_id": "2", + "reps": 1, + "provisional": true, + "no_skill": 0.4, + "arms": {"baseline": 0.4, "skill": 0.7}, + "flag": null + } + ], + "summary": { + "items": 2, + "ceiling": 1, + "floor": 0, + "provisional": 2, + "flagged_provisional": 1, + "no_flag": 1 + } +} diff --git a/dokimi/tests/test_headroom.py b/dokimi/tests/test_headroom.py new file mode 100644 index 000000000..1f695338c --- /dev/null +++ b/dokimi/tests/test_headroom.py @@ -0,0 +1,148 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for ``dokimi.eval.headroom`` and ``eval headroom``: ceiling, floor and provisional items. + +Runs are built as arms.v1 payloads in memory; no model is called. +""" + +from __future__ import annotations + +import argparse +import json + +import pytest + +from dokimi import records +from dokimi.commands.headroom import cmd_eval_headroom +from dokimi.eval.headroom import CEILING, FLOOR, format_table, headroom + + +def _run(items, baseline="baseline"): + """An arms.v1 payload: ``items`` is ``[(question_id, {arm: aggregate-or-None})]``.""" + answers, evaluations = [], [] + for qid, scores in items: + answers.append({"question_id": qid, "question_text": f"question {qid}", + "arms": {arm: {"answer": "x"} for arm in scores}}) + evaluations.append({"question_id": qid, + "scores": {arm: ({"aggregate": v} if v is not None else None) + for arm, v in scores.items()}}) + return records.read({"baseline_arm": baseline, "answers": answers, "evaluations": evaluations}, + source="test") + + +def _flags(result): + return {i["question_id"]: i["flag"] for i in result["items"]} + + +def test_no_skill_at_the_ceiling_is_flagged_and_just_below_is_not(): + run = _run([("1", {"baseline": 90, "skill": 95}), ("2", {"baseline": 89.9, "skill": 99})]) + assert _flags(headroom([run])) == {"1": CEILING, "2": None} + + +def test_floor_needs_every_arm_at_zero_not_just_the_baseline(): + run = _run([("1", {"baseline": 0, "skill": 0}), ("2", {"baseline": 0, "skill": 40})]) + assert _flags(headroom([run])) == {"1": FLOOR, "2": None} + + +def test_thresholds_are_configurable(): + run = _run([("1", {"baseline": 60, "skill": 10}), ("2", {"baseline": 5, "skill": 15})]) + result = headroom([run], ceiling=0.5, floor=0.2) + assert _flags(result) == {"1": CEILING, "2": FLOOR} + assert result["thresholds"] == {"ceiling": 0.5, "floor": 0.2, "min_reps": 3} + + +def test_a_single_run_is_provisional_and_three_runs_are_not(): + one = _run([("1", {"baseline": 95, "skill": 95})]) + item = headroom([one])["items"][0] + assert (item["reps"], item["provisional"]) == (1, True) + three = [_run([("1", {"baseline": 95, "skill": 95})]) for _ in range(3)] + item = headroom(three)["items"][0] + assert (item["reps"], item["provisional"], item["flag"]) == (3, False, CEILING) + + +def test_provisional_flag_is_kept_next_to_the_ceiling_flag(): + result = headroom([_run([("1", {"baseline": 95, "skill": 95})])]) + assert result["summary"] == {"items": 1, "ceiling": 1, "floor": 0, "provisional": 1, + "flagged_provisional": 1, "no_flag": 0} + + +def test_the_ceiling_reads_the_mean_over_runs(): + # One run at the top does not make a ceiling: the mean (0.85) is below 0.9. + runs = [_run([("1", {"baseline": 100, "skill": 100})]), _run([("1", {"baseline": 70, "skill": 70})])] + item = headroom(runs, min_reps=1)["items"][0] + assert item["no_skill"] == 0.85 and item["flag"] is None + runs[1] = _run([("1", {"baseline": 90, "skill": 90})]) + item = headroom(runs, min_reps=1)["items"][0] + assert item["no_skill"] == 0.95 and item["flag"] == CEILING + + +def test_reps_count_the_arm_scored_least_often(): + runs = [_run([("1", {"baseline": 50, "skill": 50})]) for _ in range(3)] + runs[2] = _run([("1", {"baseline": 50, "skill": None})]) + item = headroom(runs)["items"][0] + assert item["reps"] == 2 and item["provisional"] is True + + +def test_missing_no_skill_arm_is_never_a_ceiling(): + run = _run([("1", {"baseline": 99, "skill": 99})]) + result = headroom([run], no_skill_arm="nope") + assert result["items"][0]["flag"] is None and result["no_skill_arm"] == "nope" + + +@pytest.mark.parametrize("kwargs", [{"ceiling": 0.0}, {"ceiling": 1.5}, {"floor": 0.9}, + {"min_reps": 0}]) +def test_unusable_thresholds_are_refused(kwargs): + run = _run([("1", {"baseline": 50, "skill": 50})]) + with pytest.raises(ValueError): + headroom([run], **kwargs) + + +def test_binary_answers_are_refused(): + binary = records.read({"answers": [{"question_id": "1", "question_text": "q", + "with_docs": {"answer": "a"}, "without_docs": {"answer": "b"}}]}) + with pytest.raises(ValueError, match="arms runs only"): + headroom([binary], no_skill_arm="without_docs") + + +def test_runs_that_declare_different_baselines_need_an_explicit_arm(): + a = _run([("1", {"baseline": 50, "skill": 50})], baseline="baseline") + b = _run([("1", {"control": 50, "skill": 50})], baseline="control") + with pytest.raises(ValueError, match="--no-skill-arm"): + headroom([a, b]) + assert headroom([a, b], no_skill_arm="baseline")["no_skill_arm"] == "baseline" + + +def test_table_names_each_flagged_item(): + result = headroom([_run([("7", {"baseline": 97, "skill": 98}), ("8", {"baseline": 40, "skill": 60})])], + min_reps=1) + table = format_table(result) + assert "ceiling" in table and "\n7 " in table and "\n8 " not in table + assert "ceiling 1, floor 0" in table + + +def _args(paths, **overrides): + base = {"arms": [str(p) for p in paths], "no_skill_arm": None, "ceiling": 0.9, "floor": 0.0, + "min_reps": 3, "output": None} + return argparse.Namespace(**{**base, **overrides}) + + +def test_command_writes_a_headroom_artifact_and_prints_the_table(tmp_path, capsys): + arms_path = tmp_path / "arms.json" + arms_path.write_text(json.dumps({"baseline_arm": "baseline", "answers": [ + {"question_id": "1", "question_text": "q", "arms": {"baseline": {"answer": "a"}, "skill": {"answer": "b"}}}], + "evaluations": [{"question_id": "1", "scores": {"baseline": {"aggregate": 93}, "skill": {"aggregate": 94}}}]})) + out = tmp_path / "headroom.json" + cmd_eval_headroom(_args([arms_path], output=out)) + saved = json.loads(out.read_text(encoding="utf-8")) + assert saved["schema_version"] == "headroom.v1" + assert saved["items"][0]["flag"] == CEILING and saved["inputs"] == [str(arms_path)] + printed = capsys.readouterr().out + assert "ceiling" in printed and f"Wrote headroom artifact to {out}" in printed + + +def test_command_exits_2_on_an_unreadable_input(tmp_path, capsys): + with pytest.raises(SystemExit) as exc: + cmd_eval_headroom(_args([tmp_path / "missing.json"])) + assert exc.value.code == 2 + assert "file not found" in capsys.readouterr().err From 87335ba97fb552583d6be61a514b4454804e0aa6 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 28 Oct 2026 17:39:53 -0700 Subject: [PATCH 02/47] fix(eval): compare the unrounded headroom mean to the ceiling The per-arm mean was rounded to four decimals before the threshold check, so an item at 89.996 (below 0.9) was flagged as a ceiling. The flags now compare the exact mean; the artifact still shows four decimals. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/eval/headroom.py | 11 ++++++++--- dokimi/tests/test_headroom.py | 6 ++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/dokimi/dokimi/eval/headroom.py b/dokimi/dokimi/eval/headroom.py index 9666116f7..fadcb384a 100644 --- a/dokimi/dokimi/eval/headroom.py +++ b/dokimi/dokimi/eval/headroom.py @@ -47,8 +47,8 @@ def as_dict(self) -> Dict[str, Any]: "question_id": self.question_id, "reps": self.reps, "provisional": self.provisional, - "no_skill": self.no_skill, - "arms": self.arms, + "no_skill": _rounded(self.no_skill), + "arms": {arm: _rounded(v) for arm, v in self.arms.items()}, "flag": self.flag, } @@ -64,7 +64,12 @@ def check_thresholds(ceiling: float, floor: float, min_reps: int) -> None: def _mean(values: List[float]) -> Optional[float]: - return round(sum(values) / len(values), 4) if values else None + # Unrounded: the thresholds compare the exact mean (a 89.996 item is not a ceiling). + return sum(values) / len(values) if values else None + + +def _rounded(value: Optional[float]) -> Optional[float]: + return None if value is None else round(value, 4) def _flag(no_skill: Optional[float], arms: Dict[str, Optional[float]], ceiling: float, floor: float) -> Optional[str]: diff --git a/dokimi/tests/test_headroom.py b/dokimi/tests/test_headroom.py index 1f695338c..6eb8141a1 100644 --- a/dokimi/tests/test_headroom.py +++ b/dokimi/tests/test_headroom.py @@ -40,6 +40,12 @@ def test_no_skill_at_the_ceiling_is_flagged_and_just_below_is_not(): assert _flags(headroom([run])) == {"1": CEILING, "2": None} +def test_the_threshold_compares_the_unrounded_mean(): + # 89.996 is below 0.9; rounding the mean to 4 decimals would have flagged it. + run = _run([("1", {"baseline": 89.996, "skill": 99})]) + assert _flags(headroom([run])) == {"1": None} + + def test_floor_needs_every_arm_at_zero_not_just_the_baseline(): run = _run([("1", {"baseline": 0, "skill": 0}), ("2", {"baseline": 0, "skill": 40})]) assert _flags(headroom([run])) == {"1": FLOOR, "2": None} From 096522107f6ce80a9760aaa6681e96863e73de49 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 28 Oct 2026 20:23:27 -0700 Subject: [PATCH 03/47] fix(headroom): flag means exactly at a threshold despite float rounding Signed-off-by: Nikolay Petrov --- dokimi/dokimi/eval/headroom.py | 9 +++++++-- dokimi/tests/test_headroom.py | 12 ++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/eval/headroom.py b/dokimi/dokimi/eval/headroom.py index fadcb384a..779072779 100644 --- a/dokimi/dokimi/eval/headroom.py +++ b/dokimi/dokimi/eval/headroom.py @@ -29,6 +29,8 @@ DEFAULT_FLOOR = 0.0 DEFAULT_MIN_REPS = 3 +_EPS = 1e-9 # float error bound for a threshold comparison on a mean of 0-1 scores + CEILING = "ceiling" FLOOR = "floor" @@ -73,9 +75,12 @@ def _rounded(value: Optional[float]) -> Optional[float]: def _flag(no_skill: Optional[float], arms: Dict[str, Optional[float]], ceiling: float, floor: float) -> Optional[str]: - if no_skill is not None and no_skill >= ceiling: + # The mean of 0-1 scores is float division: a mean that is exactly the threshold can land a + # few ulps off it. The tolerance is far below one score step (0.01 at the finest), so it only + # absorbs that error. + if no_skill is not None and no_skill >= ceiling - _EPS: return CEILING - if arms and all(v is not None and v <= floor for v in arms.values()): + if arms and all(v is not None and v <= floor + _EPS for v in arms.values()): return FLOOR return None diff --git a/dokimi/tests/test_headroom.py b/dokimi/tests/test_headroom.py index 6eb8141a1..a202ec4c0 100644 --- a/dokimi/tests/test_headroom.py +++ b/dokimi/tests/test_headroom.py @@ -46,6 +46,18 @@ def test_the_threshold_compares_the_unrounded_mean(): assert _flags(headroom([run])) == {"1": None} +def test_a_mean_exactly_at_the_ceiling_is_flagged_though_float_division_undershoots(): + # Two runs at 85 and 95 average to exactly 90; 0.85 + 0.95 over 2 is 0.8999999999999999 in floats. + runs = [_run([("1", {"baseline": 85, "skill": 85})]), _run([("1", {"baseline": 95, "skill": 95})])] + assert _flags(headroom(runs, min_reps=1)) == {"1": CEILING} + + +def test_a_mean_exactly_at_the_floor_is_flagged_though_float_division_overshoots(): + # Two runs at 20 and 40 average to exactly 30; every arm at 30 is at the floor 0.3. + runs = [_run([("1", {"baseline": 20, "skill": 20})]), _run([("1", {"baseline": 40, "skill": 40})])] + assert _flags(headroom(runs, floor=0.3, min_reps=1)) == {"1": FLOOR} + + def test_floor_needs_every_arm_at_zero_not_just_the_baseline(): run = _run([("1", {"baseline": 0, "skill": 0}), ("2", {"baseline": 0, "skill": 40})]) assert _flags(headroom([run])) == {"1": FLOOR, "2": None} From df78ff5e388089c3fdcfe0b9041ae322d791f2fa Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 28 Oct 2026 20:33:03 -0700 Subject: [PATCH 04/47] fix(headroom): refuse unknown no-skill arms, count distinct runs A mistyped --no-skill-arm read as "no ceiling anywhere" and exited 0. It is now refused (exit 2), naming the arms the runs list. reps counted score entries, so one file passed three times, or a question id repeated within one file, faked three runs and lifted the provisional guard. The CLI now drops repeated input files by content digest with a note, and a question id repeated within one run is an error. The library counts each Run as one run. Also: the JSON keeps the unrounded mean (no 4-decimal rounding that turned 0.89996 into 0.9), the table rounds down with the flag's tolerance, the table lists the control arm once, a non-ASCII digit question id is named in the error instead of crashing, and the docs say the explicit --no-skill-arm skips the baseline agreement check. Signed-off-by: Nikolay Petrov --- dokimi/docs/evaluating-treatments.md | 15 +++-- dokimi/dokimi/commands/headroom.py | 31 +++++++-- dokimi/dokimi/eval/headroom.py | 67 ++++++++++++++----- dokimi/dokimi/schemas/headroom.v1.json | 6 +- dokimi/tests/test_headroom.py | 92 +++++++++++++++++++++++++- 5 files changed, 178 insertions(+), 33 deletions(-) diff --git a/dokimi/docs/evaluating-treatments.md b/dokimi/docs/evaluating-treatments.md index 1f9a97d33..285157b10 100644 --- a/dokimi/docs/evaluating-treatments.md +++ b/dokimi/docs/evaluating-treatments.md @@ -740,10 +740,17 @@ python cli.py eval headroom run1/arms.json run2/arms.json run3/arms.json \ - **provisional**: fewer than `--min-reps` runs scored the item (the smallest count over its arms). Default 3. One file gives every item one rep, so every item is provisional. -The table on stdout lists flagged and provisional items; `-o` writes a `headroom.v1` -artifact with every item. `--no-skill-arm` defaults to the baseline the runs declare, and -the runs must agree on it. Only arms runs are read. The flags are descriptive: they do -not change any score or any gate. +A run is one file. A file repeated on the command line, or byte-identical to an earlier +one, is skipped with a note, so it does not add a rep. A question id repeated within one +file is an error (exit 2): it would count as two runs. + +The table on stdout lists flagged and provisional items, with the no-skill arm in its own +column; `-o` writes a `headroom.v1` artifact with every item. Scores are compared unrounded +and shown rounded down, so 0.89996 prints as 0.89. Without `--no-skill-arm`, the runs must +declare the same baseline; the default is that baseline. With `--no-skill-arm`, the arm must +be one the runs list (exit 2 otherwise, naming the arms that exist), and the declared +baselines are not checked. Only arms runs are read. The flags are descriptive: they do not +change any score or any gate. ## Scope and caveats diff --git a/dokimi/dokimi/commands/headroom.py b/dokimi/dokimi/commands/headroom.py index 620f52b33..fe7be137a 100644 --- a/dokimi/dokimi/commands/headroom.py +++ b/dokimi/dokimi/commands/headroom.py @@ -4,36 +4,53 @@ """``eval headroom``: flag the ceiling, floor and provisional items of arms runs. Reads one or more ``arms.v1`` files of the same question set (one per repetition) and writes a -``headroom.v1`` artifact with ``-o``, always printing the human table. See -:mod:`dokimi.eval.headroom` for the rules. +``headroom.v1`` artifact with ``-o``, always printing the human table. A file repeated on the +command line, or byte-identical to an earlier one, is one run: the repeats are skipped with a +note on stderr. See :mod:`dokimi.eval.headroom` for the rules. """ from __future__ import annotations import argparse +import hashlib import sys +from pathlib import Path from dokimi import records def cmd_eval_headroom(args: argparse.Namespace) -> None: from dokimi.artifacts import save_artifact - from dokimi.eval.headroom import format_table, headroom + from dokimi.eval.headroom import format_table, headroom, repeated_ids - runs = [] + runs: list[records.Run] = [] + inputs: list[str] = [] + first_seen: dict[str, str] = {} for path in args.arms: try: - runs.append(records.load(path)) - except records.RecordError as e: + run = records.load(path) + digest = hashlib.sha256(Path(path).read_bytes()).hexdigest() + except (records.RecordError, OSError) as e: print(f"Error: {e}", file=sys.stderr) sys.exit(2) + if digest in first_seen: + print(f"note: skipped {path}: same content as {first_seen[digest]}, counted once", file=sys.stderr) + continue + dupes = repeated_ids(run) + if dupes: + print(f"Error: {path}: question_id {dupes[0]!r} appears more than once; " + "a file holds each question once", file=sys.stderr) + sys.exit(2) + first_seen[digest] = str(path) + runs.append(run) + inputs.append(str(path)) try: result = headroom(runs, no_skill_arm=args.no_skill_arm, ceiling=args.ceiling, floor=args.floor, min_reps=args.min_reps) except ValueError as e: print(f"Error: {e}", file=sys.stderr) sys.exit(2) - result["inputs"] = [str(p) for p in args.arms] + result["inputs"] = inputs print(format_table(result)) if args.output: save_artifact("headroom", result, args.output) diff --git a/dokimi/dokimi/eval/headroom.py b/dokimi/dokimi/eval/headroom.py index 779072779..8ca637ad3 100644 --- a/dokimi/dokimi/eval/headroom.py +++ b/dokimi/dokimi/eval/headroom.py @@ -13,13 +13,16 @@ noise. Its reps are the smallest number of runs in which any of its arms was scored, so a single artifact gives every item one rep. Each arm's score is the mean over the runs that scored it. -The input is one or more ``arms.v1`` files (or any artifact :mod:`dokimi.records` reads as an -arms run), one per repetition of the same question set. Nothing here draws a confidence -interval; the paired statistics of a comparison stay in :mod:`dokimi.eval.stats`. +A run is one ``arms.v1`` file (or any artifact :mod:`dokimi.records` reads as an arms run), one +per repetition of the same question set. Each ``Run`` passed here counts as one run, so callers +drop repeated inputs first (the CLI does, by content digest). A question id repeated within one +run is refused: it would count as two runs. Nothing here draws a confidence interval; the paired +statistics of a comparison stay in :mod:`dokimi.eval.stats`. """ from __future__ import annotations +import math from dataclasses import dataclass from typing import Any, Dict, List, Optional, Sequence, Set, Tuple @@ -49,8 +52,8 @@ def as_dict(self) -> Dict[str, Any]: "question_id": self.question_id, "reps": self.reps, "provisional": self.provisional, - "no_skill": _rounded(self.no_skill), - "arms": {arm: _rounded(v) for arm, v in self.arms.items()}, + "no_skill": self.no_skill, + "arms": dict(self.arms), "flag": self.flag, } @@ -70,10 +73,6 @@ def _mean(values: List[float]) -> Optional[float]: return sum(values) / len(values) if values else None -def _rounded(value: Optional[float]) -> Optional[float]: - return None if value is None else round(value, 4) - - def _flag(no_skill: Optional[float], arms: Dict[str, Optional[float]], ceiling: float, floor: float) -> Optional[str]: # The mean of 0-1 scores is float division: a mean that is exactly the threshold can land a # few ulps off it. The tolerance is far below one score step (0.01 at the finest), so it only @@ -95,9 +94,10 @@ def headroom( ) -> Dict[str, Any]: """Flag every item of ``runs`` (arms runs of one question set) by its headroom. - ``no_skill_arm`` defaults to the baseline the runs declare. Raises ValueError for - thresholds that cannot be used, a run that is not an arms run, or runs that disagree on - the no-skill arm. + ``no_skill_arm`` defaults to the baseline the runs declare, and must otherwise be an arm + the runs list. Raises ValueError for thresholds that cannot be used, a run that is not an + arms run, a question id repeated within a run, an unknown no-skill arm, or runs that + disagree on the baseline when none is given. """ check_thresholds(ceiling, floor, min_reps) control = _no_skill_arm(runs, no_skill_arm) @@ -143,6 +143,10 @@ def _no_skill_arm(runs: Sequence[Run], given: Optional[str]) -> str: if run.shape != "arms": raise ValueError("headroom reads arms runs only; a binary answers or eval file was given") if given is not None: + known = _known_arms(runs) + if given not in known: + listed = ", ".join(sorted(known)) or "none" + raise ValueError(f"no-skill arm {given!r} is not an arm of these runs; the arms are: {listed}") return given declared = {run.baseline for run in runs} only = next(iter(declared)) if len(declared) == 1 else None @@ -151,11 +155,37 @@ def _no_skill_arm(runs: Sequence[Run], given: Optional[str]) -> str: return only +def _known_arms(runs: Sequence[Run]) -> Set[str]: + """Every arm any run declares or any record lists, ``null`` ones included.""" + known: Set[str] = set() + for run in runs: + known.update(run.conditions) + for rec in run: + known.update(arm_names(rec)) + return known + + +def repeated_ids(run: Run) -> List[str]: + """The question ids that more than one record of ``run`` carries, in first-seen order.""" + seen: Set[str] = set() + repeated: List[str] = [] + for rec in run: + if rec.key in seen and rec.key not in repeated: + repeated.append(rec.key) + seen.add(rec.key) + return repeated + + def _collect(runs: Sequence[Run]) -> Tuple[Dict[str, Dict[str, List[float]]], Dict[str, Set[str]]]: """Question id -> arm -> the per-run scores (0-1), and question id -> the arms it listed.""" scores: Dict[str, Dict[str, List[float]]] = {} arms_of: Dict[str, Set[str]] = {} - for run in runs: + for index, run in enumerate(runs, start=1): + dupes = repeated_ids(run) + if dupes: + raise ValueError( + f"run {index} lists question_id {dupes[0]!r} more than once; " + "a run holds each question once (a repeat would count as a second run)") for rec in run: names = set(arm_names(rec)) | set(rec.scores) arms_of.setdefault(rec.key, set()).update(names) @@ -168,12 +198,16 @@ def _collect(runs: Sequence[Run]) -> Tuple[Dict[str, Dict[str, List[float]]], Di def _sort_key(qid: str): - return (0, int(qid), "") if qid.isdigit() else (1, 0, qid) + if qid.isdigit() and not qid.isascii(): + # isdigit() accepts superscripts and other digits int() refuses: name the id, do not crash. + raise ValueError(f"question id {qid!r} uses non-ASCII digits; use ASCII digits or a non-numeric id") + return (0, int(qid), qid) if qid.isdigit() else (1, 0, qid) def format_table(result: Dict[str, Any]) -> str: """The human-readable table: one row per flagged or provisional item, then the counts.""" - arm_names_ = sorted({a for item in result["items"] for a in item["arms"]}) + control = result["no_skill_arm"] + arm_names_ = sorted({a for item in result["items"] for a in item["arms"]} - {control}) head = ["question", "reps", "no-skill", *arm_names_, "flag", "provisional"] rows = [] for item in result["items"]: @@ -202,7 +236,8 @@ def format_table(result: Dict[str, Any]) -> str: def _fmt(value: Optional[float]) -> str: - return "-" if value is None else f"{value:.2f}" + # Rounded down, with the flag's tolerance: 0.89996 shows 0.89, an exact 0.9 shows 0.90. + return "-" if value is None else f"{math.floor((value + _EPS) * 100) / 100:.2f}" def _line(cells: List[str], widths: List[int]) -> str: diff --git a/dokimi/dokimi/schemas/headroom.v1.json b/dokimi/dokimi/schemas/headroom.v1.json index 69a00a55b..475da32b0 100644 --- a/dokimi/dokimi/schemas/headroom.v1.json +++ b/dokimi/dokimi/schemas/headroom.v1.json @@ -19,8 +19,8 @@ "min_reps": {"type": "integer", "minimum": 1} } }, - "runs": {"type": "integer", "minimum": 1, "description": "Number of arms.v1 inputs."}, - "inputs": {"type": "array", "items": {"type": "string"}}, + "runs": {"type": "integer", "minimum": 1, "description": "Number of distinct arms.v1 runs; a repeated input file counts once."}, + "inputs": {"type": "array", "items": {"type": "string"}, "description": "The arms.v1 files read, repeats dropped."}, "items": { "type": "array", "items": { @@ -31,7 +31,7 @@ "question_id": {"type": "string"}, "reps": {"type": "integer", "minimum": 0, "description": "Smallest number of runs in which any of the item's arms was scored."}, "provisional": {"type": "boolean", "description": "True when reps is below thresholds.min_reps."}, - "no_skill": {"type": ["number", "null"], "description": "Mean over runs of the no-skill arm, 0-1."}, + "no_skill": {"type": ["number", "null"], "description": "Unrounded mean over runs of the no-skill arm, 0-1."}, "arms": {"type": "object", "additionalProperties": {"type": ["number", "null"]}, "description": "Mean over runs of each arm, 0-1; null when no run scored it."}, "flag": {"enum": ["ceiling", "floor", null]} } diff --git a/dokimi/tests/test_headroom.py b/dokimi/tests/test_headroom.py index a202ec4c0..feb77e106 100644 --- a/dokimi/tests/test_headroom.py +++ b/dokimi/tests/test_headroom.py @@ -102,10 +102,40 @@ def test_reps_count_the_arm_scored_least_often(): assert item["reps"] == 2 and item["provisional"] is True -def test_missing_no_skill_arm_is_never_a_ceiling(): +def test_unknown_no_skill_arm_is_refused_and_lists_the_arms(): + # A mistyped arm used to read as "no ceiling anywhere" with exit 0. run = _run([("1", {"baseline": 99, "skill": 99})]) - result = headroom([run], no_skill_arm="nope") - assert result["items"][0]["flag"] is None and result["no_skill_arm"] == "nope" + with pytest.raises(ValueError, match="'nope'.*the arms are: baseline, skill"): + headroom([run], no_skill_arm="nope") + + +def test_repeated_question_id_in_one_run_is_refused(): + run = records.read({"baseline_arm": "baseline", "answers": [ + {"question_id": "1", "question_text": "q", "arms": {"baseline": {"answer": "a"}}}, + {"question_id": "1", "question_text": "q", "arms": {"baseline": {"answer": "b"}}}]}) + with pytest.raises(ValueError, match="question_id '1' more than once"): + headroom([run], no_skill_arm="baseline") + + +def test_a_non_ascii_digit_question_id_is_refused_by_name(): + # '²'.isdigit() is true and int('²') raises: the id must be named, not crash. + run = _run([("²", {"baseline": 50, "skill": 50})]) + with pytest.raises(ValueError, match="question id '²'"): + headroom([run]) + + +def test_the_table_and_json_do_not_round_0_89996_up_to_the_ceiling(): + run = _run([("1", {"baseline": 89.996, "skill": 99})]) + result = headroom([run], min_reps=2) + assert result["items"][0]["no_skill"] == pytest.approx(0.89996) and result["items"][0]["flag"] is None + table = format_table(result) + assert "0.89" in table and "0.90" not in table + + +def test_the_table_lists_the_control_arm_once(): + result = headroom([_run([("1", {"baseline": 40, "skill": 60})])], min_reps=2) + header = format_table(result).splitlines()[0].split() + assert header == ["question", "reps", "no-skill", "skill", "flag", "provisional"] @pytest.mark.parametrize("kwargs", [{"ceiling": 0.0}, {"ceiling": 1.5}, {"floor": 0.9}, @@ -164,3 +194,59 @@ def test_command_exits_2_on_an_unreadable_input(tmp_path, capsys): cmd_eval_headroom(_args([tmp_path / "missing.json"])) assert exc.value.code == 2 assert "file not found" in capsys.readouterr().err + + +def _arms_file(path, scores, baseline="baseline"): + """An arms.v1 file on disk: ``scores`` is ``{question_id: {arm: aggregate}}``.""" + payload = {"baseline_arm": baseline, + "answers": [{"question_id": qid, "question_text": f"question {qid}", + "arms": {arm: {"answer": "x"} for arm in arms}} + for qid, arms in scores.items()], + "evaluations": [{"question_id": qid, + "scores": {arm: {"aggregate": v} for arm, v in arms.items()}} + for qid, arms in scores.items()]} + path.write_text(json.dumps(payload), encoding="utf-8") + return path + + +def test_command_refuses_an_unknown_no_skill_arm_and_lists_the_arms(tmp_path, capsys): + path = _arms_file(tmp_path / "arms.json", {"1": {"baseline": 99, "skill": 99}}) + with pytest.raises(SystemExit) as exc: + cmd_eval_headroom(_args([path], no_skill_arm="basline")) + assert exc.value.code == 2 + err = capsys.readouterr().err + assert "'basline'" in err and "the arms are: baseline, skill" in err + + +def test_repeated_input_files_count_as_one_run(tmp_path, capsys): + one = _arms_file(tmp_path / "run1.json", {"1": {"baseline": 95, "skill": 95}}) + copy = tmp_path / "copy.json" + copy.write_bytes(one.read_bytes()) + out = tmp_path / "headroom.json" + cmd_eval_headroom(_args([one, one, one, copy], output=out)) + saved = json.loads(out.read_text(encoding="utf-8")) + assert saved["runs"] == 1 and saved["inputs"] == [str(one)] + assert saved["items"][0]["reps"] == 1 and saved["items"][0]["provisional"] is True + err = capsys.readouterr().err + assert err.count("counted once") == 3 and f"skipped {copy}: same content as {one}" in err + + +def test_distinct_input_files_each_count_as_a_run(tmp_path): + paths = [_arms_file(tmp_path / f"run{i}.json", {"1": {"baseline": 95 - i, "skill": 95}}) for i in range(3)] + out = tmp_path / "headroom.json" + cmd_eval_headroom(_args(paths, output=out)) + saved = json.loads(out.read_text(encoding="utf-8")) + item = saved["items"][0] + assert (saved["runs"], item["reps"], item["provisional"], item["flag"]) == (3, 3, False, CEILING) + + +def test_command_refuses_a_question_id_repeated_within_one_file(tmp_path, capsys): + path = tmp_path / "dup.json" + path.write_text(json.dumps({"baseline_arm": "baseline", "answers": [ + {"question_id": "1", "question_text": "q", "arms": {"baseline": {"answer": "a"}}}, + {"question_id": "1", "question_text": "q", "arms": {"baseline": {"answer": "b"}}}]})) + with pytest.raises(SystemExit) as exc: + cmd_eval_headroom(_args([path])) + assert exc.value.code == 2 + err = capsys.readouterr().err + assert str(path) in err and "question_id '1' appears more than once" in err From 050de2e06cc84981cedd46800403c921b4932762 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 28 Oct 2026 20:42:02 -0700 Subject: [PATCH 05/47] fix(headroom): reps follow the no-skill arm, unscored arms skip the floor reps was the smallest run count over the arms, so one arm the skill missed in a run lowered the item's reps. It is now the number of runs in which the no-skill arm scored the item. An arm no run scored is listed in the item's unscored_arms (JSON and a table note) and is left out of the floor test. The floor also needs at least one non-baseline arm that scored, so the baseline alone at zero is not a floor. -o that cannot be written prints "Error: ..." and exits 2. The duplicate-input note says "byte-identical", as the docs do. The docs state that JSON goes to -o only and the table always goes to stdout. headroom.v1 is not on intel/main yet, so the schema gains unscored_arms in place without a version bump. Signed-off-by: Nikolay Petrov --- dokimi/docs/evaluating-treatments.md | 15 +++-- dokimi/dokimi/commands/headroom.py | 10 ++- dokimi/dokimi/eval/headroom.py | 25 ++++++-- dokimi/dokimi/schemas/headroom.v1.json | 5 +- .../tests/fixtures/artifacts/headroom.v1.json | 2 + dokimi/tests/test_headroom.py | 61 ++++++++++++++++++- 6 files changed, 99 insertions(+), 19 deletions(-) diff --git a/dokimi/docs/evaluating-treatments.md b/dokimi/docs/evaluating-treatments.md index 285157b10..77473f75f 100644 --- a/dokimi/docs/evaluating-treatments.md +++ b/dokimi/docs/evaluating-treatments.md @@ -736,16 +736,21 @@ python cli.py eval headroom run1/arms.json run2/arms.json run3/arms.json \ - **ceiling**: the no-skill arm's mean over the runs is at least `--ceiling` (0-1; the 0-100 aggregate divided by 100). Default 0.9. -- **floor**: every arm's mean is at most `--floor`. Default 0. -- **provisional**: fewer than `--min-reps` runs scored the item (the smallest count over - its arms). Default 3. One file gives every item one rep, so every item is provisional. +- **floor**: every arm that a run scored has a mean at most `--floor`, and at least one + non-baseline arm scored. Default 0. An arm no run scored is left out of this test and + listed in the item's `unscored_arms`; the baseline alone at zero is not a floor. +- **provisional**: fewer than `--min-reps` runs scored the item with its no-skill arm (the + reps are those runs). Default 3. One file gives every item one rep, so every item is + provisional. A run is one file. A file repeated on the command line, or byte-identical to an earlier one, is skipped with a note, so it does not add a rep. A question id repeated within one file is an error (exit 2): it would count as two runs. -The table on stdout lists flagged and provisional items, with the no-skill arm in its own -column; `-o` writes a `headroom.v1` artifact with every item. Scores are compared unrounded +The table always goes to stdout: the flagged and provisional items, with the no-skill arm +in its own column, a note naming any arm no run scored, and the counts. The JSON goes to +`-o` only, as a `headroom.v1` artifact with every item; if that file cannot be written the +command prints `Error: ...` on stderr and exits 2, after the table. Scores are compared unrounded and shown rounded down, so 0.89996 prints as 0.89. Without `--no-skill-arm`, the runs must declare the same baseline; the default is that baseline. With `--no-skill-arm`, the arm must be one the runs list (exit 2 otherwise, naming the arms that exist), and the declared diff --git a/dokimi/dokimi/commands/headroom.py b/dokimi/dokimi/commands/headroom.py index fe7be137a..7b502eefb 100644 --- a/dokimi/dokimi/commands/headroom.py +++ b/dokimi/dokimi/commands/headroom.py @@ -20,7 +20,7 @@ def cmd_eval_headroom(args: argparse.Namespace) -> None: - from dokimi.artifacts import save_artifact + from dokimi.artifacts import ArtifactValidationError, save_artifact from dokimi.eval.headroom import format_table, headroom, repeated_ids runs: list[records.Run] = [] @@ -34,7 +34,7 @@ def cmd_eval_headroom(args: argparse.Namespace) -> None: print(f"Error: {e}", file=sys.stderr) sys.exit(2) if digest in first_seen: - print(f"note: skipped {path}: same content as {first_seen[digest]}, counted once", file=sys.stderr) + print(f"note: skipped {path}: byte-identical to {first_seen[digest]}, counted once", file=sys.stderr) continue dupes = repeated_ids(run) if dupes: @@ -53,7 +53,11 @@ def cmd_eval_headroom(args: argparse.Namespace) -> None: result["inputs"] = inputs print(format_table(result)) if args.output: - save_artifact("headroom", result, args.output) + try: + save_artifact("headroom", result, args.output) + except (OSError, ArtifactValidationError) as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(2) print(f"Wrote headroom artifact to {args.output}") diff --git a/dokimi/dokimi/eval/headroom.py b/dokimi/dokimi/eval/headroom.py index 8ca637ad3..3cf2bf470 100644 --- a/dokimi/dokimi/eval/headroom.py +++ b/dokimi/dokimi/eval/headroom.py @@ -9,9 +9,12 @@ ``floor`` (default 0): no arm answers it, which points at the question, its checks or the grader rather than at any skill. +An item's floor test leaves out an arm no run scored (listed in ``unscored_arms``), and needs at +least one non-baseline arm that did score: the baseline alone at zero shows nothing about a skill. + An item with fewer than ``min_reps`` runs behind it is ``provisional``: one run's flag may be -noise. Its reps are the smallest number of runs in which any of its arms was scored, so a single -artifact gives every item one rep. Each arm's score is the mean over the runs that scored it. +noise. Its reps are the number of runs in which the no-skill arm scored it, so a single artifact +gives every item one rep. Each arm's score is the mean over the runs that scored it. A run is one ``arms.v1`` file (or any artifact :mod:`dokimi.records` reads as an arms run), one per repetition of the same question set. Each ``Run`` passed here counts as one run, so callers @@ -45,6 +48,7 @@ class ItemHeadroom: provisional: bool no_skill: Optional[float] arms: Dict[str, Optional[float]] + unscored_arms: List[str] flag: Optional[str] def as_dict(self) -> Dict[str, Any]: @@ -54,6 +58,7 @@ def as_dict(self) -> Dict[str, Any]: "provisional": self.provisional, "no_skill": self.no_skill, "arms": dict(self.arms), + "unscored_arms": list(self.unscored_arms), "flag": self.flag, } @@ -73,13 +78,16 @@ def _mean(values: List[float]) -> Optional[float]: return sum(values) / len(values) if values else None -def _flag(no_skill: Optional[float], arms: Dict[str, Optional[float]], ceiling: float, floor: float) -> Optional[str]: +def _flag( + no_skill: Optional[float], arms: Dict[str, Optional[float]], control: str, ceiling: float, floor: float, +) -> Optional[str]: # The mean of 0-1 scores is float division: a mean that is exactly the threshold can land a # few ulps off it. The tolerance is far below one score step (0.01 at the finest), so it only # absorbs that error. if no_skill is not None and no_skill >= ceiling - _EPS: return CEILING - if arms and all(v is not None and v <= floor + _EPS for v in arms.values()): + scored = {arm: v for arm, v in arms.items() if v is not None} + if any(arm != control for arm in scored) and all(v <= floor + _EPS for v in scored.values()): return FLOOR return None @@ -107,7 +115,7 @@ def headroom( for qid in sorted(scores, key=_sort_key): per_arm = scores[qid] arms = {arm: _mean(per_arm.get(arm, [])) for arm in sorted(arms_of[qid])} - reps = min((len(per_arm.get(arm, [])) for arm in arms_of[qid]), default=0) + reps = len(per_arm.get(control, [])) no_skill = arms.get(control) items.append(ItemHeadroom( question_id=qid, @@ -115,7 +123,8 @@ def headroom( provisional=reps < min_reps, no_skill=no_skill, arms=arms, - flag=_flag(no_skill, arms, ceiling, floor), + unscored_arms=[arm for arm, value in arms.items() if value is None], + flag=_flag(no_skill, arms, control, ceiling, floor), )) summary = { @@ -223,6 +232,10 @@ def format_table(result: Dict[str, Any]) -> str: lines = [_line(head, widths)] + [_line(r, widths) for r in rows] if not rows: lines.append("(no item is flagged or provisional)") + unscored = [f"{item['question_id']} ({', '.join(item['unscored_arms'])})" + for item in result["items"] if item["unscored_arms"]] + if unscored: + lines.append("note: no run scored these arms, so the floor test leaves them out: " + "; ".join(unscored)) s = result["summary"] th = result["thresholds"] lines.append("") diff --git a/dokimi/dokimi/schemas/headroom.v1.json b/dokimi/dokimi/schemas/headroom.v1.json index 475da32b0..fe2587187 100644 --- a/dokimi/dokimi/schemas/headroom.v1.json +++ b/dokimi/dokimi/schemas/headroom.v1.json @@ -25,14 +25,15 @@ "type": "array", "items": { "type": "object", - "required": ["question_id", "reps", "provisional", "no_skill", "arms", "flag"], + "required": ["question_id", "reps", "provisional", "no_skill", "arms", "unscored_arms", "flag"], "additionalProperties": false, "properties": { "question_id": {"type": "string"}, - "reps": {"type": "integer", "minimum": 0, "description": "Smallest number of runs in which any of the item's arms was scored."}, + "reps": {"type": "integer", "minimum": 0, "description": "Number of runs in which the no-skill arm scored the item."}, "provisional": {"type": "boolean", "description": "True when reps is below thresholds.min_reps."}, "no_skill": {"type": ["number", "null"], "description": "Unrounded mean over runs of the no-skill arm, 0-1."}, "arms": {"type": "object", "additionalProperties": {"type": ["number", "null"]}, "description": "Mean over runs of each arm, 0-1; null when no run scored it."}, + "unscored_arms": {"type": "array", "items": {"type": "string"}, "uniqueItems": true, "description": "The arms no run scored (null in arms), sorted; the floor test leaves them out."}, "flag": {"enum": ["ceiling", "floor", null]} } } diff --git a/dokimi/tests/fixtures/artifacts/headroom.v1.json b/dokimi/tests/fixtures/artifacts/headroom.v1.json index 874aa7f23..385000cec 100644 --- a/dokimi/tests/fixtures/artifacts/headroom.v1.json +++ b/dokimi/tests/fixtures/artifacts/headroom.v1.json @@ -11,6 +11,7 @@ "provisional": true, "no_skill": 0.95, "arms": {"baseline": 0.95, "skill": 0.94}, + "unscored_arms": [], "flag": "ceiling" }, { @@ -19,6 +20,7 @@ "provisional": true, "no_skill": 0.4, "arms": {"baseline": 0.4, "skill": 0.7}, + "unscored_arms": [], "flag": null } ], diff --git a/dokimi/tests/test_headroom.py b/dokimi/tests/test_headroom.py index feb77e106..570811105 100644 --- a/dokimi/tests/test_headroom.py +++ b/dokimi/tests/test_headroom.py @@ -95,11 +95,50 @@ def test_the_ceiling_reads_the_mean_over_runs(): assert item["no_skill"] == 0.95 and item["flag"] == CEILING -def test_reps_count_the_arm_scored_least_often(): +def test_reps_count_the_runs_that_scored_the_baseline(): + # The skill arm missed one run; the baseline scored all three, so the item has three reps. runs = [_run([("1", {"baseline": 50, "skill": 50})]) for _ in range(3)] runs[2] = _run([("1", {"baseline": 50, "skill": None})]) item = headroom(runs)["items"][0] - assert item["reps"] == 2 and item["provisional"] is True + assert item["reps"] == 3 and item["provisional"] is False + + +def test_reps_follow_the_baseline_even_when_a_skill_arm_was_scored_in_more_runs(): + runs = [_run([("1", {"baseline": 50, "skill": 50})]), _run([("1", {"baseline": None, "skill": 50})]), + _run([("1", {"baseline": None, "skill": 50})])] + item = headroom(runs)["items"][0] + assert item["reps"] == 1 and item["provisional"] is True + + +def test_an_item_the_baseline_never_scored_has_no_reps_and_no_flag(): + runs = [_run([("1", {"baseline": None, "skill": 95})]) for _ in range(3)] + item = headroom(runs)["items"][0] + assert (item["reps"], item["no_skill"], item["flag"]) == (0, None, None) + + +def test_an_arm_no_run_scored_is_listed_as_unscored(): + run = _run([("1", {"baseline": 40, "skill": 60, "other": None})]) + item = headroom([run], min_reps=1)["items"][0] + assert item["unscored_arms"] == ["other"] and item["arms"]["other"] is None + + +def test_an_arm_scored_in_some_runs_is_not_unscored(): + runs = [_run([("1", {"baseline": 40, "skill": 60, "other": 50})]), + _run([("1", {"baseline": 40, "skill": 60, "other": None})])] + item = headroom(runs, min_reps=1)["items"][0] + assert item["unscored_arms"] == [] and item["arms"]["other"] == 0.5 + + +def test_floor_leaves_out_an_arm_no_run_scored(): + # "other" never answered; the arms that did are all at zero, so the item is a floor. + run = _run([("1", {"baseline": 0, "skill": 0, "other": None})]) + assert _flags(headroom([run])) == {"1": FLOOR} + + +def test_floor_needs_a_non_baseline_arm_that_scored(): + # Only the baseline answered (at zero): nothing shows the question is unanswerable for a skill. + assert _flags(headroom([_run([("1", {"baseline": 0})])])) == {"1": None} + assert _flags(headroom([_run([("1", {"baseline": 0, "skill": None})])])) == {"1": None} def test_unknown_no_skill_arm_is_refused_and_lists_the_arms(): @@ -161,6 +200,12 @@ def test_runs_that_declare_different_baselines_need_an_explicit_arm(): assert headroom([a, b], no_skill_arm="baseline")["no_skill_arm"] == "baseline" +def test_table_notes_the_arms_no_run_scored(): + run = _run([("1", {"baseline": 40, "skill": 60, "other": None}), ("2", {"baseline": 40, "skill": 60})]) + table = format_table(headroom([run], min_reps=1)) + assert "no run scored" in table and "1 (other)" in table and "2 (" not in table + + def test_table_names_each_flagged_item(): result = headroom([_run([("7", {"baseline": 97, "skill": 98}), ("8", {"baseline": 40, "skill": 60})])], min_reps=1) @@ -228,7 +273,7 @@ def test_repeated_input_files_count_as_one_run(tmp_path, capsys): assert saved["runs"] == 1 and saved["inputs"] == [str(one)] assert saved["items"][0]["reps"] == 1 and saved["items"][0]["provisional"] is True err = capsys.readouterr().err - assert err.count("counted once") == 3 and f"skipped {copy}: same content as {one}" in err + assert err.count("counted once") == 3 and f"skipped {copy}: byte-identical to {one}, counted once" in err def test_distinct_input_files_each_count_as_a_run(tmp_path): @@ -240,6 +285,16 @@ def test_distinct_input_files_each_count_as_a_run(tmp_path): assert (saved["runs"], item["reps"], item["provisional"], item["flag"]) == (3, 3, False, CEILING) +def test_command_exits_2_when_the_output_cannot_be_written(tmp_path, capsys): + path = _arms_file(tmp_path / "arms.json", {"1": {"baseline": 95, "skill": 95}}) + blocker = tmp_path / "a-file" + blocker.write_text("not a directory, so -o cannot be written beneath it", encoding="utf-8") + with pytest.raises(SystemExit) as exc: + cmd_eval_headroom(_args([path], output=blocker / "headroom.json")) + assert exc.value.code == 2 + assert capsys.readouterr().err.startswith("Error: ") + + def test_command_refuses_a_question_id_repeated_within_one_file(tmp_path, capsys): path = tmp_path / "dup.json" path.write_text(json.dumps({"baseline_arm": "baseline", "answers": [ From 38d219c805f6807fadbb45ce8a38efdb270acd33 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Thu, 29 Oct 2026 11:06:27 +0000 Subject: [PATCH 06/47] docs(eval): document the headroom threshold range checks Signed-off-by: Anton Volkov --- dokimi/docs/evaluating-treatments.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dokimi/docs/evaluating-treatments.md b/dokimi/docs/evaluating-treatments.md index 77473f75f..19f66e8df 100644 --- a/dokimi/docs/evaluating-treatments.md +++ b/dokimi/docs/evaluating-treatments.md @@ -754,7 +754,8 @@ command prints `Error: ...` on stderr and exits 2, after the table. Scores are c and shown rounded down, so 0.89996 prints as 0.89. Without `--no-skill-arm`, the runs must declare the same baseline; the default is that baseline. With `--no-skill-arm`, the arm must be one the runs list (exit 2 otherwise, naming the arms that exist), and the declared -baselines are not checked. Only arms runs are read. The flags are descriptive: they do not +baselines are not checked. A threshold out of range (`--ceiling` in (0, 1], `--floor` in +[0, ceiling), `--min-reps` at least 1) exits 2. Only arms runs are read. The flags are descriptive: they do not change any score or any gate. ## Scope and caveats From d5113e0b9c85208fbbaf2994bf8f6c01c0445f46 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:39:23 -0700 Subject: [PATCH 07/47] Address review: pin the baseline term of the floor test An item with baseline 40 and skill 0 must not be a floor. Red when the floor test drops the baseline (all(... if a != control)), green on head. Thread: PRRT_kwDOVBg7g86xhYFw Thread: PRR_kwDOVBg7g88AAAABU4x1TA Thread: PRR_kwDOVBg7g88AAAABU5Bc8g Signed-off-by: Nikolay Petrov --- dokimi/tests/test_headroom.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dokimi/tests/test_headroom.py b/dokimi/tests/test_headroom.py index 570811105..95a83d58e 100644 --- a/dokimi/tests/test_headroom.py +++ b/dokimi/tests/test_headroom.py @@ -63,6 +63,12 @@ def test_floor_needs_every_arm_at_zero_not_just_the_baseline(): assert _flags(headroom([run])) == {"1": FLOOR, "2": None} +def test_floor_needs_the_baseline_at_the_floor_too_not_just_the_skill_arms(): + # The baseline answers question 1 (40) though the skill arm scores 0: no floor. + run = _run([("1", {"baseline": 40, "skill": 0}), ("2", {"baseline": 0, "skill": 0})]) + assert _flags(headroom([run])) == {"1": None, "2": FLOOR} + + def test_thresholds_are_configurable(): run = _run([("1", {"baseline": 60, "skill": 10}), ("2", {"baseline": 5, "skill": 15})]) result = headroom([run], ceiling=0.5, floor=0.2) From 15695acc627defe7f1268ea019628281edf6721b Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:39:32 -0700 Subject: [PATCH 08/47] Address review: drive non-default thresholds through the headroom command Red when the command calls headroom() with the default thresholds; the test asserts the flags and the thresholds of the -o artifact for ceiling 0.5, floor 0.2, min-reps 1. Thread: PRRT_kwDOVBg7g86xhYFz Thread: PRR_kwDOVBg7g88AAAABU4x1TA Thread: PRR_kwDOVBg7g88AAAABU5Bc8g Signed-off-by: Nikolay Petrov --- dokimi/tests/test_headroom.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/dokimi/tests/test_headroom.py b/dokimi/tests/test_headroom.py index 95a83d58e..655e8c2a3 100644 --- a/dokimi/tests/test_headroom.py +++ b/dokimi/tests/test_headroom.py @@ -311,3 +311,15 @@ def test_command_refuses_a_question_id_repeated_within_one_file(tmp_path, capsys assert exc.value.code == 2 err = capsys.readouterr().err assert str(path) in err and "question_id '1' appears more than once" in err + + +def test_command_passes_non_default_thresholds_to_the_analysis(tmp_path): + # With the defaults neither item is flagged: 0.6 < 0.9, and 0.15 > 0.0. + path = _arms_file(tmp_path / "arms.json", {"1": {"baseline": 60, "skill": 10}, + "2": {"baseline": 5, "skill": 15}}) + out = tmp_path / "headroom.json" + cmd_eval_headroom(_args([path], ceiling=0.5, floor=0.2, min_reps=1, output=out)) + saved = json.loads(out.read_text(encoding="utf-8")) + assert {i["question_id"]: i["flag"] for i in saved["items"]} == {"1": CEILING, "2": FLOOR} + assert saved["thresholds"] == {"ceiling": 0.5, "floor": 0.2, "min_reps": 1} + assert [i["provisional"] for i in saved["items"]] == [False, False] From 4bbc04aba242fd5d0d2c674fc70b9e8666f3abbe Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:39:51 -0700 Subject: [PATCH 09/47] Address review: run the headroom command through the real parser Red when an option is renamed, its dest changes or a default drifts from the DEFAULT_* constants; one test also runs the command from parsed arguments. Thread: PRRT_kwDOVBg7g86xlrgs Signed-off-by: Nikolay Petrov --- dokimi/tests/test_headroom.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/dokimi/tests/test_headroom.py b/dokimi/tests/test_headroom.py index 655e8c2a3..1069478f2 100644 --- a/dokimi/tests/test_headroom.py +++ b/dokimi/tests/test_headroom.py @@ -14,8 +14,11 @@ import pytest from dokimi import records +from dokimi.cli import build_parser from dokimi.commands.headroom import cmd_eval_headroom -from dokimi.eval.headroom import CEILING, FLOOR, format_table, headroom +from dokimi.eval.headroom import ( + CEILING, DEFAULT_CEILING, DEFAULT_FLOOR, DEFAULT_MIN_REPS, FLOOR, format_table, headroom, +) def _run(items, baseline="baseline"): @@ -323,3 +326,22 @@ def test_command_passes_non_default_thresholds_to_the_analysis(tmp_path): assert {i["question_id"]: i["flag"] for i in saved["items"]} == {"1": CEILING, "2": FLOOR} assert saved["thresholds"] == {"ceiling": 0.5, "floor": 0.2, "min_reps": 1} assert [i["provisional"] for i in saved["items"]] == [False, False] + + +def test_the_registered_options_default_to_the_documented_thresholds(tmp_path): + args = build_parser().parse_args(["eval", "headroom", str(tmp_path / "a.json")]) + assert (args.ceiling, args.floor, args.min_reps) == (DEFAULT_CEILING, DEFAULT_FLOOR, DEFAULT_MIN_REPS) + assert (args.no_skill_arm, args.output) == (None, None) + assert args.func is cmd_eval_headroom + + +def test_the_registered_options_reach_the_analysis_through_the_real_parser(tmp_path): + path = _arms_file(tmp_path / "arms.json", {"1": {"baseline": 60, "skill": 10}, + "2": {"baseline": 5, "skill": 15}}) + out = tmp_path / "headroom.json" + args = build_parser().parse_args(["eval", "headroom", str(path), "--ceiling", "0.5", "--floor", "0.2", + "--min-reps", "1", "--no-skill-arm", "baseline", "-o", str(out)]) + args.func(args) + saved = json.loads(out.read_text(encoding="utf-8")) + assert saved["thresholds"] == {"ceiling": 0.5, "floor": 0.2, "min_reps": 1} + assert {i["question_id"]: i["flag"] for i in saved["items"]} == {"1": CEILING, "2": FLOOR} From 329baac3e7918b7ea5cf9927c3681966f0b13ffb Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:40:12 -0700 Subject: [PATCH 10/47] Address review: pin each summary counter with a mixed item set Five items cover ceiling+provisional, floor, unflagged+provisional and unflagged+steady. Red when flagged_provisional counts every provisional item, no_flag is hard-coded, or the floor count is read as the ceiling count. Thread: PRRT_kwDOVBg7g86xlrf- Signed-off-by: Nikolay Petrov --- dokimi/tests/test_headroom.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/dokimi/tests/test_headroom.py b/dokimi/tests/test_headroom.py index 1069478f2..b8e1b3871 100644 --- a/dokimi/tests/test_headroom.py +++ b/dokimi/tests/test_headroom.py @@ -94,6 +94,17 @@ def test_provisional_flag_is_kept_next_to_the_ceiling_flag(): "flagged_provisional": 1, "no_flag": 0} +def test_summary_counts_each_kind_of_item_apart(): + # Three runs. "f1", "f2" and "ok" are in all three; "cp" and "up" only in the first, so they are provisional. + steady = [("f1", {"baseline": 0, "skill": 0}), ("f2", {"baseline": 0, "skill": 0}), + ("ok", {"baseline": 50, "skill": 60})] + first = [("cp", {"baseline": 95, "skill": 95}), ("up", {"baseline": 50, "skill": 60}), *steady] + result = headroom([_run(first), _run(steady), _run(steady)]) + assert _flags(result) == {"cp": CEILING, "up": None, "f1": FLOOR, "f2": FLOOR, "ok": None} + assert result["summary"] == {"items": 5, "ceiling": 1, "floor": 2, "provisional": 2, + "flagged_provisional": 1, "no_flag": 2} + + def test_the_ceiling_reads_the_mean_over_runs(): # One run at the top does not make a ceiling: the mean (0.85) is below 0.9. runs = [_run([("1", {"baseline": 100, "skill": 100})]), _run([("1", {"baseline": 70, "skill": 70})])] From 19b25df556f267017d56d6294ce2ee91f8a84a1f Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:40:33 -0700 Subject: [PATCH 11/47] Address review: pin the item order of the headroom result Numeric ids come first in numeric order, then the other ids. Red when the sort is lexicographic, insertion order, or ignores the numeric key. Thread: PRRT_kwDOVBg7g86xlrgN Signed-off-by: Nikolay Petrov --- dokimi/tests/test_headroom.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dokimi/tests/test_headroom.py b/dokimi/tests/test_headroom.py index b8e1b3871..344e5a287 100644 --- a/dokimi/tests/test_headroom.py +++ b/dokimi/tests/test_headroom.py @@ -105,6 +105,12 @@ def test_summary_counts_each_kind_of_item_apart(): "flagged_provisional": 1, "no_flag": 2} +def test_items_list_numeric_ids_in_numeric_order_then_the_other_ids(): + scores = {"baseline": 50, "skill": 60} + run = _run([(qid, scores) for qid in ["q1", "10", "b", "2", "a", "1"]]) + assert [i["question_id"] for i in headroom([run])["items"]] == ["1", "2", "10", "a", "b", "q1"] + + def test_the_ceiling_reads_the_mean_over_runs(): # One run at the top does not make a ceiling: the mean (0.85) is below 0.9. runs = [_run([("1", {"baseline": 100, "skill": 100})]), _run([("1", {"baseline": 70, "skill": 70})])] From 8e5245ee940ea9a9f540b4c882af961f12e994b1 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:41:03 -0700 Subject: [PATCH 12/47] Address review: match the threshold refusals and pin their boundaries Each rejected threshold now names its own message (ceiling 0, negative and above 1; floor equal to the ceiling and negative; min_reps 0 and negative), and ceiling=1.0, floor=0 and min_reps=1 are accepted. Red when a bound moves. Thread: PRRT_kwDOVBg7g86xlrgX Signed-off-by: Nikolay Petrov --- dokimi/tests/test_headroom.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/dokimi/tests/test_headroom.py b/dokimi/tests/test_headroom.py index 344e5a287..cd31f9305 100644 --- a/dokimi/tests/test_headroom.py +++ b/dokimi/tests/test_headroom.py @@ -203,14 +203,30 @@ def test_the_table_lists_the_control_arm_once(): assert header == ["question", "reps", "no-skill", "skill", "flag", "provisional"] -@pytest.mark.parametrize("kwargs", [{"ceiling": 0.0}, {"ceiling": 1.5}, {"floor": 0.9}, - {"min_reps": 0}]) -def test_unusable_thresholds_are_refused(kwargs): +@pytest.mark.parametrize("kwargs, message", [ + ({"ceiling": 0.0}, r"ceiling must be in \(0, 1\], not 0\.0"), + ({"ceiling": -0.1}, r"ceiling must be in \(0, 1\], not -0\.1"), + ({"ceiling": 1.5}, r"ceiling must be in \(0, 1\], not 1\.5"), + ({"floor": 0.9}, r"floor must be in \[0, ceiling\), not 0\.9 \(ceiling 0\.9\)"), + ({"floor": -0.1}, r"floor must be in \[0, ceiling\), not -0\.1"), + ({"min_reps": 0}, r"min_reps must be at least 1, not 0"), + ({"min_reps": -1}, r"min_reps must be at least 1, not -1"), +]) +def test_unusable_thresholds_are_refused(kwargs, message): run = _run([("1", {"baseline": 50, "skill": 50})]) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match=message): headroom([run], **kwargs) +def test_the_threshold_boundaries_are_accepted(): + # ceiling 1.0 is the top of (0, 1]; floor 0 is the bottom of [0, ceiling); one rep is the least. + run = _run([("1", {"baseline": 100, "skill": 100}), ("2", {"baseline": 99, "skill": 99})]) + result = headroom([run], ceiling=1.0, floor=0.0, min_reps=1) + assert _flags(result) == {"1": CEILING, "2": None} + assert result["thresholds"] == {"ceiling": 1.0, "floor": 0.0, "min_reps": 1} + assert headroom([run], ceiling=0.5, floor=0.49)["thresholds"]["floor"] == 0.49 + + def test_binary_answers_are_refused(): binary = records.read({"answers": [{"question_id": "1", "question_text": "q", "with_docs": {"answer": "a"}, "without_docs": {"answer": "b"}}]}) From 544394efde0aa542396205ddd302287207280c80 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:41:30 -0700 Subject: [PATCH 13/47] Address review: cover empty and unjudged headroom input headroom([]) is refused as "no runs to read"; a run with no questions and a run nothing was judged in give no flag, at library and command level. Red when the empty-input refusal is removed or the command stops printing the table. Thread: PRRT_kwDOVBg7g86xlrgc Signed-off-by: Nikolay Petrov --- dokimi/tests/test_headroom.py | 41 +++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/dokimi/tests/test_headroom.py b/dokimi/tests/test_headroom.py index cd31f9305..22f361b96 100644 --- a/dokimi/tests/test_headroom.py +++ b/dokimi/tests/test_headroom.py @@ -227,6 +227,27 @@ def test_the_threshold_boundaries_are_accepted(): assert headroom([run], ceiling=0.5, floor=0.49)["thresholds"]["floor"] == 0.49 +def test_no_runs_are_refused(): + with pytest.raises(ValueError, match="no runs to read"): + headroom([]) + + +def test_a_run_with_no_questions_has_no_items(): + result = headroom([_run([])]) + assert result["items"] == [] and result["runs"] == 1 + assert result["summary"] == {"items": 0, "ceiling": 0, "floor": 0, "provisional": 0, + "flagged_provisional": 0, "no_flag": 0} + + +def test_a_run_with_no_evaluations_flags_nothing_and_has_no_reps(): + # The answers are there but nothing was judged: every score is None. + run = records.read({"baseline_arm": "baseline", "answers": [ + {"question_id": "1", "question_text": "q", "arms": {"baseline": {"answer": "a"}, "skill": {"answer": "b"}}}]}) + item = headroom([run])["items"][0] + assert (item["reps"], item["provisional"], item["flag"], item["no_skill"]) == (0, True, None, None) + assert item["unscored_arms"] == ["baseline", "skill"] + + def test_binary_answers_are_refused(): binary = records.read({"answers": [{"question_id": "1", "question_text": "q", "with_docs": {"answer": "a"}, "without_docs": {"answer": "b"}}]}) @@ -378,3 +399,23 @@ def test_the_registered_options_reach_the_analysis_through_the_real_parser(tmp_p saved = json.loads(out.read_text(encoding="utf-8")) assert saved["thresholds"] == {"ceiling": 0.5, "floor": 0.2, "min_reps": 1} assert {i["question_id"]: i["flag"] for i in saved["items"]} == {"1": CEILING, "2": FLOOR} + + +def test_command_reads_a_file_with_no_questions(tmp_path, capsys): + path = tmp_path / "empty.json" + path.write_text(json.dumps({"baseline_arm": "baseline", "answers": [], "evaluations": []}), encoding="utf-8") + out = tmp_path / "headroom.json" + cmd_eval_headroom(_args([path], output=out)) + assert "(no item is flagged or provisional)" in capsys.readouterr().out + saved = json.loads(out.read_text(encoding="utf-8")) + assert saved["items"] == [] and saved["summary"]["items"] == 0 and saved["runs"] == 1 + + +def test_command_reads_a_file_nothing_was_judged_in(tmp_path, capsys): + path = tmp_path / "unjudged.json" + path.write_text(json.dumps({"baseline_arm": "baseline", "answers": [ + {"question_id": "1", "question_text": "q", "arms": {"baseline": {"answer": "a"}, "skill": {"answer": "b"}}}]}), + encoding="utf-8") + cmd_eval_headroom(_args([path])) + printed = capsys.readouterr().out + assert "yes" in printed and "no run scored these arms" in printed and "1 (baseline, skill)" in printed From fe66ffc8d015f2adbe94d30dd0162f39c77409e6 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:41:50 -0700 Subject: [PATCH 14/47] Address review: pin a given no-skill arm on the success path With no_skill_arm="skill" on runs that declare "baseline", no_skill, reps and both flags follow "skill"; the mixed-baseline case now asserts its item too. Red when a given arm is ignored and the declared baseline is returned. Thread: PRRT_kwDOVBg7g86xlrgh Signed-off-by: Nikolay Petrov --- dokimi/tests/test_headroom.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/dokimi/tests/test_headroom.py b/dokimi/tests/test_headroom.py index 22f361b96..f3a239bd3 100644 --- a/dokimi/tests/test_headroom.py +++ b/dokimi/tests/test_headroom.py @@ -167,6 +167,24 @@ def test_floor_needs_a_non_baseline_arm_that_scored(): assert _flags(headroom([_run([("1", {"baseline": 0, "skill": None})])])) == {"1": None} +def test_a_given_no_skill_arm_replaces_the_declared_baseline(): + # The runs declare "baseline"; the caller reads "skill" as the control. Every verdict follows "skill". + runs = [_run([("1", {"baseline": 10, "skill": 95}), ("2", {"baseline": 30, "skill": 0}), + ("3", {"baseline": 0, "skill": 0}), ("4", {"baseline": 40, "skill": None})]) + for _ in range(2)] + runs.append(_run([("1", {"baseline": 10, "skill": None}), ("2", {"baseline": 30, "skill": 0}), + ("3", {"baseline": 0, "skill": 0}), ("4", {"baseline": 40, "skill": None})])) + declared = headroom(runs) + assert declared["no_skill_arm"] == "baseline" and _flags(declared) == {"1": None, "2": None, "3": FLOOR, "4": None} + given = headroom(runs, no_skill_arm="skill") + assert given["no_skill_arm"] == "skill" + assert _flags(given) == {"1": CEILING, "2": None, "3": FLOOR, "4": None} + by_id = {i["question_id"]: i for i in given["items"]} + assert (by_id["1"]["no_skill"], by_id["1"]["reps"], by_id["1"]["provisional"]) == (0.95, 2, True) + assert (by_id["2"]["reps"], by_id["2"]["provisional"]) == (3, False) + assert (by_id["4"]["no_skill"], by_id["4"]["reps"]) == (None, 0) + + def test_unknown_no_skill_arm_is_refused_and_lists_the_arms(): # A mistyped arm used to read as "no ceiling anywhere" with exit 0. run = _run([("1", {"baseline": 99, "skill": 99})]) @@ -260,7 +278,13 @@ def test_runs_that_declare_different_baselines_need_an_explicit_arm(): b = _run([("1", {"control": 50, "skill": 50})], baseline="control") with pytest.raises(ValueError, match="--no-skill-arm"): headroom([a, b]) - assert headroom([a, b], no_skill_arm="baseline")["no_skill_arm"] == "baseline" + result = headroom([a, b], no_skill_arm="baseline") + assert result["no_skill_arm"] == "baseline" and result["runs"] == 2 + item = result["items"][0] + # "baseline" was scored in run a only, so one rep; "control" is just another arm here. + assert (item["reps"], item["provisional"], item["no_skill"], item["flag"]) == (1, True, 0.5, None) + assert item["arms"] == {"baseline": 0.5, "control": 0.5, "skill": 0.5} + assert headroom([a, b], no_skill_arm="control")["items"][0]["reps"] == 1 def test_table_notes_the_arms_no_run_scored(): From 95f84dd47846fb7e76a9d36877e4d986c5e51ed7 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:44:30 -0700 Subject: [PATCH 15/47] Thread: IC_kwDOVBg7g88AAAABgreQtQ: already fixed in 544394ef The patch-coverage report named lines of dokimi/eval/headroom.py (the empty-input refusal) and the output-less branch of dokimi/commands/headroom.py. The tests of 544394ef and 4bbc04ab cover both: test_headroom.py alone reaches 100% of lines and branches of the two files. Signed-off-by: Nikolay Petrov From e5faa9239379096eeafe4af8b28640a01280a285 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 17:52:51 -0700 Subject: [PATCH 16/47] fix(headroom): exit 2 on an input nested too deeply to read json.loads raises RecursionError on a deeply nested file, which the headroom command did not catch, so it ended in a traceback. Catch it with the other read errors. Thread: PRRT_kwDOVBg7g86xtTdU Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/headroom.py | 2 +- dokimi/tests/test_headroom.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/commands/headroom.py b/dokimi/dokimi/commands/headroom.py index 7b502eefb..ef3e65401 100644 --- a/dokimi/dokimi/commands/headroom.py +++ b/dokimi/dokimi/commands/headroom.py @@ -30,7 +30,7 @@ def cmd_eval_headroom(args: argparse.Namespace) -> None: try: run = records.load(path) digest = hashlib.sha256(Path(path).read_bytes()).hexdigest() - except (records.RecordError, OSError) as e: + except (records.RecordError, OSError, RecursionError) as e: # RecursionError: nesting too deep for json print(f"Error: {e}", file=sys.stderr) sys.exit(2) if digest in first_seen: diff --git a/dokimi/tests/test_headroom.py b/dokimi/tests/test_headroom.py index f3a239bd3..93c2c00a0 100644 --- a/dokimi/tests/test_headroom.py +++ b/dokimi/tests/test_headroom.py @@ -443,3 +443,12 @@ def test_command_reads_a_file_nothing_was_judged_in(tmp_path, capsys): cmd_eval_headroom(_args([path])) printed = capsys.readouterr().out assert "yes" in printed and "no run scored these arms" in printed and "1 (baseline, skill)" in printed + + +def test_command_exits_2_on_an_input_nested_too_deep_to_read(tmp_path, capsys): + path = tmp_path / "deep.json" + path.write_text("[" * 100000, encoding="utf-8") + with pytest.raises(SystemExit) as exc: + cmd_eval_headroom(_args([path])) + assert exc.value.code == 2 + assert "Error:" in capsys.readouterr().err From 37cd2c859fcf2194999fec84140e491cef0d8e0d Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 17:53:07 -0700 Subject: [PATCH 17/47] fix(headroom): --min-reps refuses 0 with the usage message --min-reps used type=int, so 0 reached the analysis and failed as an Error line. Pass the eval flags' positive_int, as the other eval commands do, so argparse names the flag. Thread: PRRT_kwDOVBg7g86xyyV1 Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/evaluate.py | 2 +- dokimi/dokimi/commands/headroom.py | 4 ++-- dokimi/tests/test_headroom.py | 7 +++++++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/dokimi/dokimi/commands/evaluate.py b/dokimi/dokimi/commands/evaluate.py index 05f169cab..4c3e7213b 100644 --- a/dokimi/dokimi/commands/evaluate.py +++ b/dokimi/dokimi/commands/evaluate.py @@ -567,7 +567,7 @@ def register(sub, positive_int) -> None: from dokimi.commands import headroom - headroom.register_eval(eval_sub) # eval headroom + headroom.register_eval(eval_sub, positive_int) # eval headroom # eval panel-score panel_p = eval_sub.add_parser("panel-score", help="Score answers using a multi-judge panel") diff --git a/dokimi/dokimi/commands/headroom.py b/dokimi/dokimi/commands/headroom.py index ef3e65401..a601686f6 100644 --- a/dokimi/dokimi/commands/headroom.py +++ b/dokimi/dokimi/commands/headroom.py @@ -61,7 +61,7 @@ def cmd_eval_headroom(args: argparse.Namespace) -> None: print(f"Wrote headroom artifact to {args.output}") -def register_eval(eval_sub) -> None: +def register_eval(eval_sub, positive_int) -> None: from dokimi.eval.headroom import DEFAULT_CEILING, DEFAULT_FLOOR, DEFAULT_MIN_REPS h = eval_sub.add_parser( @@ -74,7 +74,7 @@ def register_eval(eval_sub) -> None: help=f"Flag an item when the no-skill arm scores at least this, 0-1 (default {DEFAULT_CEILING})") h.add_argument("--floor", type=float, default=DEFAULT_FLOOR, help=f"Flag an item when every arm scores at most this, 0-1 (default {DEFAULT_FLOOR})") - h.add_argument("--min-reps", type=int, default=DEFAULT_MIN_REPS, dest="min_reps", + h.add_argument("--min-reps", type=positive_int, default=DEFAULT_MIN_REPS, dest="min_reps", help=f"Items with fewer runs than this are provisional (default {DEFAULT_MIN_REPS})") h.add_argument("-o", "--output", default=None, help="headroom.v1 JSON to write") h.set_defaults(func=cmd_eval_headroom) diff --git a/dokimi/tests/test_headroom.py b/dokimi/tests/test_headroom.py index 93c2c00a0..71679d77d 100644 --- a/dokimi/tests/test_headroom.py +++ b/dokimi/tests/test_headroom.py @@ -452,3 +452,10 @@ def test_command_exits_2_on_an_input_nested_too_deep_to_read(tmp_path, capsys): cmd_eval_headroom(_args([path])) assert exc.value.code == 2 assert "Error:" in capsys.readouterr().err + + +def test_min_reps_must_be_a_positive_integer_at_the_parser(tmp_path, capsys): + with pytest.raises(SystemExit) as exc: + build_parser().parse_args(["eval", "headroom", str(tmp_path / "a.json"), "--min-reps", "0"]) + assert exc.value.code == 2 + assert "--min-reps" in capsys.readouterr().err From 744e1cac1233ab3f937584f62022d38300f18054 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 17:53:23 -0700 Subject: [PATCH 18/47] fix(headroom): keep a non-ASCII digit question id instead of refusing it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An id like '²' passes isdigit() but int() refuses it, so the whole analysis exited 2 just to sort it. Only ASCII digit ids are read as numbers; any other id sorts as text after them. Thread: PRRT_kwDOVBg7g86xyyVx Signed-off-by: Nikolay Petrov --- dokimi/dokimi/eval/headroom.py | 6 ++---- dokimi/tests/test_headroom.py | 10 +++++----- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/dokimi/dokimi/eval/headroom.py b/dokimi/dokimi/eval/headroom.py index 3cf2bf470..4c7427032 100644 --- a/dokimi/dokimi/eval/headroom.py +++ b/dokimi/dokimi/eval/headroom.py @@ -207,10 +207,8 @@ def _collect(runs: Sequence[Run]) -> Tuple[Dict[str, Dict[str, List[float]]], Di def _sort_key(qid: str): - if qid.isdigit() and not qid.isascii(): - # isdigit() accepts superscripts and other digits int() refuses: name the id, do not crash. - raise ValueError(f"question id {qid!r} uses non-ASCII digits; use ASCII digits or a non-numeric id") - return (0, int(qid), qid) if qid.isdigit() else (1, 0, qid) + # isdigit() also accepts superscripts and other digits int() refuses: those sort as text, not numbers. + return (0, int(qid), qid) if qid.isascii() and qid.isdigit() else (1, 0, qid) def format_table(result: Dict[str, Any]) -> str: diff --git a/dokimi/tests/test_headroom.py b/dokimi/tests/test_headroom.py index 71679d77d..076442b62 100644 --- a/dokimi/tests/test_headroom.py +++ b/dokimi/tests/test_headroom.py @@ -200,11 +200,11 @@ def test_repeated_question_id_in_one_run_is_refused(): headroom([run], no_skill_arm="baseline") -def test_a_non_ascii_digit_question_id_is_refused_by_name(): - # '²'.isdigit() is true and int('²') raises: the id must be named, not crash. - run = _run([("²", {"baseline": 50, "skill": 50})]) - with pytest.raises(ValueError, match="question id '²'"): - headroom([run]) +def test_a_non_ascii_digit_question_id_is_kept_and_sorted_as_text(): + # '²'.isdigit() is true and int('²') raises: the id is an ordinary id, sorted after the numeric ones. + run = _run([("²", {"baseline": 50, "skill": 50}), ("10", {"baseline": 50, "skill": 50}), + ("2", {"baseline": 50, "skill": 50})]) + assert [i["question_id"] for i in headroom([run])["items"]] == ["2", "10", "²"] def test_the_table_and_json_do_not_round_0_89996_up_to_the_ceiling(): From b3a0b785febc5fdb9888d55e212747ba10c3e0b3 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 17:53:37 -0700 Subject: [PATCH 19/47] fix(headroom): refuse -o naming one of the input runs -o pointing at an input arms file replaced that run with the headroom artifact, losing its answers and evaluations. Compare the resolved paths and exit 2 before anything is read into the output. Thread: PRRT_kwDOVBg7g86xyyVs Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/headroom.py | 3 +++ dokimi/tests/test_headroom.py | 10 ++++++++++ 2 files changed, 13 insertions(+) diff --git a/dokimi/dokimi/commands/headroom.py b/dokimi/dokimi/commands/headroom.py index a601686f6..09a1c819d 100644 --- a/dokimi/dokimi/commands/headroom.py +++ b/dokimi/dokimi/commands/headroom.py @@ -23,6 +23,9 @@ def cmd_eval_headroom(args: argparse.Namespace) -> None: from dokimi.artifacts import ArtifactValidationError, save_artifact from dokimi.eval.headroom import format_table, headroom, repeated_ids + if args.output and Path(args.output).resolve() in {Path(p).resolve() for p in args.arms}: + print(f"Error: -o {args.output} is one of the input arms files; it would overwrite that run", file=sys.stderr) + sys.exit(2) runs: list[records.Run] = [] inputs: list[str] = [] first_seen: dict[str, str] = {} diff --git a/dokimi/tests/test_headroom.py b/dokimi/tests/test_headroom.py index 076442b62..c5b4af055 100644 --- a/dokimi/tests/test_headroom.py +++ b/dokimi/tests/test_headroom.py @@ -459,3 +459,13 @@ def test_min_reps_must_be_a_positive_integer_at_the_parser(tmp_path, capsys): build_parser().parse_args(["eval", "headroom", str(tmp_path / "a.json"), "--min-reps", "0"]) assert exc.value.code == 2 assert "--min-reps" in capsys.readouterr().err + + +def test_command_refuses_an_output_that_is_one_of_its_inputs(tmp_path, capsys): + path = _arms_file(tmp_path / "arms.json", {"1": {"baseline": 99, "skill": 99}}) + before = path.read_bytes() + with pytest.raises(SystemExit) as exc: + cmd_eval_headroom(_args([path], output=path)) + assert exc.value.code == 2 + assert path.read_bytes() == before + assert "overwrite" in capsys.readouterr().err From 49a4d544066c642decc07d9dc24490ea50662684 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 17:54:04 -0700 Subject: [PATCH 20/47] fix(headroom): count a re-serialized copy of a run once Repeats were detected by raw bytes, so the same run written with other whitespace or key order counted as a second run and could lift an item out of provisional. Compare a digest of the parsed JSON instead. Thread: PRRT_kwDOVBg7g86xyyVu Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/headroom.py | 13 ++++++++++--- dokimi/tests/test_headroom.py | 14 +++++++++++++- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/dokimi/dokimi/commands/headroom.py b/dokimi/dokimi/commands/headroom.py index 09a1c819d..84ba753ba 100644 --- a/dokimi/dokimi/commands/headroom.py +++ b/dokimi/dokimi/commands/headroom.py @@ -5,7 +5,7 @@ Reads one or more ``arms.v1`` files of the same question set (one per repetition) and writes a ``headroom.v1`` artifact with ``-o``, always printing the human table. A file repeated on the -command line, or byte-identical to an earlier one, is one run: the repeats are skipped with a +command line, or holding the same JSON content as an earlier one, is one run: the repeats are skipped with a note on stderr. See :mod:`dokimi.eval.headroom` for the rules. """ @@ -13,12 +13,19 @@ import argparse import hashlib +import json import sys from pathlib import Path from dokimi import records +def _content_digest(path) -> str: + """Digest of the parsed JSON, so one run re-serialized (whitespace, key order) counts once.""" + data = json.loads(Path(path).read_text(encoding="utf-8")) # records.load has parsed it already + return hashlib.sha256(json.dumps(data, sort_keys=True).encode("utf-8")).hexdigest() + + def cmd_eval_headroom(args: argparse.Namespace) -> None: from dokimi.artifacts import ArtifactValidationError, save_artifact from dokimi.eval.headroom import format_table, headroom, repeated_ids @@ -32,12 +39,12 @@ def cmd_eval_headroom(args: argparse.Namespace) -> None: for path in args.arms: try: run = records.load(path) - digest = hashlib.sha256(Path(path).read_bytes()).hexdigest() + digest = _content_digest(path) except (records.RecordError, OSError, RecursionError) as e: # RecursionError: nesting too deep for json print(f"Error: {e}", file=sys.stderr) sys.exit(2) if digest in first_seen: - print(f"note: skipped {path}: byte-identical to {first_seen[digest]}, counted once", file=sys.stderr) + print(f"note: skipped {path}: same content as {first_seen[digest]}, counted once", file=sys.stderr) continue dupes = repeated_ids(run) if dupes: diff --git a/dokimi/tests/test_headroom.py b/dokimi/tests/test_headroom.py index c5b4af055..6a18ec3c5 100644 --- a/dokimi/tests/test_headroom.py +++ b/dokimi/tests/test_headroom.py @@ -360,7 +360,7 @@ def test_repeated_input_files_count_as_one_run(tmp_path, capsys): assert saved["runs"] == 1 and saved["inputs"] == [str(one)] assert saved["items"][0]["reps"] == 1 and saved["items"][0]["provisional"] is True err = capsys.readouterr().err - assert err.count("counted once") == 3 and f"skipped {copy}: byte-identical to {one}, counted once" in err + assert err.count("counted once") == 3 and f"skipped {copy}: same content as {one}, counted once" in err def test_distinct_input_files_each_count_as_a_run(tmp_path): @@ -469,3 +469,15 @@ def test_command_refuses_an_output_that_is_one_of_its_inputs(tmp_path, capsys): assert exc.value.code == 2 assert path.read_bytes() == before assert "overwrite" in capsys.readouterr().err + + +def test_a_reserialized_copy_of_a_run_counts_once(tmp_path, capsys): + arms = _arms_file(tmp_path / "run1.json", {"1": {"baseline": 95, "skill": 40}}) + copy = tmp_path / "copy.json" + copy.write_text(json.dumps(json.loads(arms.read_text(encoding="utf-8")), indent=2, sort_keys=True), + encoding="utf-8") + out = tmp_path / "headroom.json" + cmd_eval_headroom(_args([arms, copy], output=out)) + saved = json.loads(out.read_text(encoding="utf-8")) + assert saved["runs"] == 1 and saved["items"][0]["reps"] == 1 + assert "counted once" in capsys.readouterr().err From 340879b4a152d3ba9a0d4239848234b37069c679 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 17:54:26 -0700 Subject: [PATCH 21/47] fix(headroom): summarize the unscored-arm note per arm The note listed every 'qid (arm)' pair, so one arm that failed across a 500-question run put 500 entries on one line above the counts. Give one count per arm instead. Thread: PRRT_kwDOVBg7g86xyyVv Signed-off-by: Nikolay Petrov --- dokimi/dokimi/eval/headroom.py | 9 ++++++--- dokimi/tests/test_headroom.py | 11 +++++++++-- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/dokimi/dokimi/eval/headroom.py b/dokimi/dokimi/eval/headroom.py index 4c7427032..8d333b463 100644 --- a/dokimi/dokimi/eval/headroom.py +++ b/dokimi/dokimi/eval/headroom.py @@ -230,10 +230,13 @@ def format_table(result: Dict[str, Any]) -> str: lines = [_line(head, widths)] + [_line(r, widths) for r in rows] if not rows: lines.append("(no item is flagged or provisional)") - unscored = [f"{item['question_id']} ({', '.join(item['unscored_arms'])})" - for item in result["items"] if item["unscored_arms"]] + unscored: Dict[str, int] = {} + for item in result["items"]: + for arm in item["unscored_arms"]: + unscored[arm] = unscored.get(arm, 0) + 1 if unscored: - lines.append("note: no run scored these arms, so the floor test leaves them out: " + "; ".join(unscored)) + counts = [f"{arm} in {n} item{'' if n == 1 else 's'}" for arm, n in sorted(unscored.items())] + lines.append("note: no run scored these arms, so the floor test leaves them out: " + "; ".join(counts)) s = result["summary"] th = result["thresholds"] lines.append("") diff --git a/dokimi/tests/test_headroom.py b/dokimi/tests/test_headroom.py index 6a18ec3c5..2760905b8 100644 --- a/dokimi/tests/test_headroom.py +++ b/dokimi/tests/test_headroom.py @@ -290,7 +290,7 @@ def test_runs_that_declare_different_baselines_need_an_explicit_arm(): def test_table_notes_the_arms_no_run_scored(): run = _run([("1", {"baseline": 40, "skill": 60, "other": None}), ("2", {"baseline": 40, "skill": 60})]) table = format_table(headroom([run], min_reps=1)) - assert "no run scored" in table and "1 (other)" in table and "2 (" not in table + assert "no run scored" in table and "other in 1 item" in table def test_table_names_each_flagged_item(): @@ -442,7 +442,7 @@ def test_command_reads_a_file_nothing_was_judged_in(tmp_path, capsys): encoding="utf-8") cmd_eval_headroom(_args([path])) printed = capsys.readouterr().out - assert "yes" in printed and "no run scored these arms" in printed and "1 (baseline, skill)" in printed + assert "yes" in printed and "no run scored these arms" in printed and "baseline in 1 item; skill in 1 item" in printed def test_command_exits_2_on_an_input_nested_too_deep_to_read(tmp_path, capsys): @@ -481,3 +481,10 @@ def test_a_reserialized_copy_of_a_run_counts_once(tmp_path, capsys): saved = json.loads(out.read_text(encoding="utf-8")) assert saved["runs"] == 1 and saved["items"][0]["reps"] == 1 assert "counted once" in capsys.readouterr().err + + +def test_table_note_counts_the_unscored_items_per_arm_instead_of_listing_them(): + items = [(str(q), {"baseline": 40, "skill": None}) for q in range(1, 501)] + table = format_table(headroom([_run(items)], min_reps=1)) + note = next(line for line in table.splitlines() if line.startswith("note:")) + assert note.endswith("skill in 500 items") and "(" not in note From 2c22659de6c1ddc583a5260e847ec36c0c116245 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 17:54:46 -0700 Subject: [PATCH 22/47] fix(headroom): a floor needs the no-skill arm scored on the item An item whose baseline judge failed was flagged floor when the skill arm scored 0, since the unscored baseline was left out of the floor test. Require the no-skill arm to have a score for the item. Thread: PRRT_kwDOVBg7g86xyyVq Signed-off-by: Nikolay Petrov --- dokimi/dokimi/eval/headroom.py | 8 +++++--- dokimi/tests/test_headroom.py | 6 ++++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/dokimi/dokimi/eval/headroom.py b/dokimi/dokimi/eval/headroom.py index 8d333b463..3e981c21e 100644 --- a/dokimi/dokimi/eval/headroom.py +++ b/dokimi/dokimi/eval/headroom.py @@ -9,8 +9,9 @@ ``floor`` (default 0): no arm answers it, which points at the question, its checks or the grader rather than at any skill. -An item's floor test leaves out an arm no run scored (listed in ``unscored_arms``), and needs at -least one non-baseline arm that did score: the baseline alone at zero shows nothing about a skill. +An item's floor test leaves out an arm no run scored (listed in ``unscored_arms``). It needs the +no-skill arm scored on the item and at least one non-baseline arm that did score: the baseline alone +at zero shows nothing about a skill, and an unscored baseline shows nothing either. An item with fewer than ``min_reps`` runs behind it is ``provisional``: one run's flag may be noise. Its reps are the number of runs in which the no-skill arm scored it, so a single artifact @@ -87,7 +88,8 @@ def _flag( if no_skill is not None and no_skill >= ceiling - _EPS: return CEILING scored = {arm: v for arm, v in arms.items() if v is not None} - if any(arm != control for arm in scored) and all(v <= floor + _EPS for v in scored.values()): + # An unscored no-skill arm cannot show the item is unanswerable: its judge may just have failed. + if no_skill is not None and any(arm != control for arm in scored) and all(v <= floor + _EPS for v in scored.values()): return FLOOR return None diff --git a/dokimi/tests/test_headroom.py b/dokimi/tests/test_headroom.py index 2760905b8..56195c3df 100644 --- a/dokimi/tests/test_headroom.py +++ b/dokimi/tests/test_headroom.py @@ -488,3 +488,9 @@ def test_table_note_counts_the_unscored_items_per_arm_instead_of_listing_them(): table = format_table(headroom([_run(items)], min_reps=1)) note = next(line for line in table.splitlines() if line.startswith("note:")) assert note.endswith("skill in 500 items") and "(" not in note + + +def test_floor_is_not_flagged_when_the_no_skill_arm_was_never_scored_on_the_item(): + # Item 1's baseline judge failed; the skill at 0 alone is not a floor. Item 2 is one. + run = _run([("1", {"baseline": None, "skill": 0}), ("2", {"baseline": 0, "skill": 0})]) + assert _flags(headroom([run], min_reps=1)) == {"1": None, "2": FLOOR} From aa8c3ad2494a670ccfbcd950da9d71ec730e6b4a Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 17:55:25 -0700 Subject: [PATCH 23/47] fix(headroom): refuse a file where no run scored the no-skill arm An unjudged file, or a baseline named differently from the scores, left every item unmeasured and the command still exited 0 with the items counted as unflagged. Exit 2 when items exist and none has a no-skill score, and name the arm and the fix. Thread: PRRT_kwDOVBg7g86xyyVn Signed-off-by: Nikolay Petrov --- dokimi/dokimi/eval/headroom.py | 9 ++++++-- dokimi/tests/test_headroom.py | 38 ++++++++++++++++++++++++---------- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/dokimi/dokimi/eval/headroom.py b/dokimi/dokimi/eval/headroom.py index 3e981c21e..50d373caf 100644 --- a/dokimi/dokimi/eval/headroom.py +++ b/dokimi/dokimi/eval/headroom.py @@ -106,12 +106,17 @@ def headroom( ``no_skill_arm`` defaults to the baseline the runs declare, and must otherwise be an arm the runs list. Raises ValueError for thresholds that cannot be used, a run that is not an - arms run, a question id repeated within a run, an unknown no-skill arm, or runs that - disagree on the baseline when none is given. + arms run, a question id repeated within a run, an unknown no-skill arm, no score at all for + the no-skill arm (when there are items), or runs that disagree on the baseline when none is given. """ check_thresholds(ceiling, floor, min_reps) control = _no_skill_arm(runs, no_skill_arm) scores, arms_of = _collect(runs) + if scores and not any(control in per_arm for per_arm in scores.values()): + # Every item would read as "no ceiling, not a floor": unmeasured, not headroom. Refuse it instead. + raise ValueError( + f"no run scored the no-skill arm {control!r}, so no item can be read for headroom; " + "the runs were not judged, or the baseline is named differently (pass --no-skill-arm)") items: List[ItemHeadroom] = [] for qid in sorted(scores, key=_sort_key): diff --git a/dokimi/tests/test_headroom.py b/dokimi/tests/test_headroom.py index 56195c3df..e0362da1e 100644 --- a/dokimi/tests/test_headroom.py +++ b/dokimi/tests/test_headroom.py @@ -137,8 +137,9 @@ def test_reps_follow_the_baseline_even_when_a_skill_arm_was_scored_in_more_runs( def test_an_item_the_baseline_never_scored_has_no_reps_and_no_flag(): - runs = [_run([("1", {"baseline": None, "skill": 95})]) for _ in range(3)] - item = headroom(runs)["items"][0] + # Item 2 keeps the baseline scored somewhere, so the run is not refused as unjudged. + runs = [_run([("1", {"baseline": None, "skill": 95}), ("2", {"baseline": 40, "skill": 95})]) for _ in range(3)] + item = {i["question_id"]: i for i in headroom(runs)["items"]}["1"] assert (item["reps"], item["no_skill"], item["flag"]) == (0, None, None) @@ -257,13 +258,18 @@ def test_a_run_with_no_questions_has_no_items(): "flagged_provisional": 0, "no_flag": 0} -def test_a_run_with_no_evaluations_flags_nothing_and_has_no_reps(): - # The answers are there but nothing was judged: every score is None. +def test_a_run_nothing_was_judged_in_is_refused(): + # The answers are there but nothing was judged: no item could be read as a ceiling or a floor. run = records.read({"baseline_arm": "baseline", "answers": [ {"question_id": "1", "question_text": "q", "arms": {"baseline": {"answer": "a"}, "skill": {"answer": "b"}}}]}) - item = headroom([run])["items"][0] - assert (item["reps"], item["provisional"], item["flag"], item["no_skill"]) == (0, True, None, None) - assert item["unscored_arms"] == ["baseline", "skill"] + with pytest.raises(ValueError, match="no run scored the no-skill arm 'baseline'"): + headroom([run]) + + +def test_a_baseline_named_differently_from_the_scores_is_refused(): + run = _run([("1", {"no_skill": 100, "skill": 50})], baseline="control") + with pytest.raises(ValueError, match="no run scored the no-skill arm 'control'"): + headroom([run]) def test_binary_answers_are_refused(): @@ -435,14 +441,24 @@ def test_command_reads_a_file_with_no_questions(tmp_path, capsys): assert saved["items"] == [] and saved["summary"]["items"] == 0 and saved["runs"] == 1 -def test_command_reads_a_file_nothing_was_judged_in(tmp_path, capsys): +def test_command_refuses_a_file_nothing_was_judged_in(tmp_path, capsys): path = tmp_path / "unjudged.json" path.write_text(json.dumps({"baseline_arm": "baseline", "answers": [ {"question_id": "1", "question_text": "q", "arms": {"baseline": {"answer": "a"}, "skill": {"answer": "b"}}}]}), encoding="utf-8") - cmd_eval_headroom(_args([path])) - printed = capsys.readouterr().out - assert "yes" in printed and "no run scored these arms" in printed and "baseline in 1 item; skill in 1 item" in printed + out = tmp_path / "headroom.json" + with pytest.raises(SystemExit) as exc: + cmd_eval_headroom(_args([path], output=out)) + assert exc.value.code == 2 and not out.exists() + assert "no run scored the no-skill arm 'baseline'" in capsys.readouterr().err + + +def test_command_refuses_a_wrong_baseline_name_with_exit_2(tmp_path, capsys): + path = _arms_file(tmp_path / "arms.json", {"1": {"no_skill": 100, "skill": 50}}, baseline="control") + with pytest.raises(SystemExit) as exc: + cmd_eval_headroom(_args([path])) + assert exc.value.code == 2 + assert "pass --no-skill-arm" in capsys.readouterr().err def test_command_exits_2_on_an_input_nested_too_deep_to_read(tmp_path, capsys): From 3df832abd94f829f9de9cae42abcb748c247c83c Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 19:29:13 -0700 Subject: [PATCH 24/47] docs(headroom): a repeat is skipped by JSON content, not by bytes The command skips a repeated run by the digest of its parsed JSON (see 49a4d544), so a re-serialized copy also counts once. The section said byte-identical, which understated that. Wording only; no code path changed, so no test applies. Signed-off-by: Nikolay Petrov --- dokimi/docs/evaluating-treatments.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dokimi/docs/evaluating-treatments.md b/dokimi/docs/evaluating-treatments.md index 19f66e8df..cf1c6d9d3 100644 --- a/dokimi/docs/evaluating-treatments.md +++ b/dokimi/docs/evaluating-treatments.md @@ -743,8 +743,9 @@ python cli.py eval headroom run1/arms.json run2/arms.json run3/arms.json \ reps are those runs). Default 3. One file gives every item one rep, so every item is provisional. -A run is one file. A file repeated on the command line, or byte-identical to an earlier -one, is skipped with a note, so it does not add a rep. A question id repeated within one +A run is one file. A file repeated on the command line, or holding the same JSON content as +an earlier one (whitespace and key order do not count), is skipped with a note, so it does +not add a rep. A question id repeated within one file is an error (exit 2): it would count as two runs. The table always goes to stdout: the flagged and provisional items, with the no-skill arm From 95642ebb991f97113844f2ee0056a8f275f4cbe2 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 19:52:22 -0700 Subject: [PATCH 25/47] docs(headroom): keep the tolerance comment to two lines Signed-off-by: Nikolay Petrov --- dokimi/dokimi/eval/headroom.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dokimi/dokimi/eval/headroom.py b/dokimi/dokimi/eval/headroom.py index 50d373caf..8bc719a2a 100644 --- a/dokimi/dokimi/eval/headroom.py +++ b/dokimi/dokimi/eval/headroom.py @@ -82,9 +82,8 @@ def _mean(values: List[float]) -> Optional[float]: def _flag( no_skill: Optional[float], arms: Dict[str, Optional[float]], control: str, ceiling: float, floor: float, ) -> Optional[str]: - # The mean of 0-1 scores is float division: a mean that is exactly the threshold can land a - # few ulps off it. The tolerance is far below one score step (0.01 at the finest), so it only - # absorbs that error. + # A mean exactly at the threshold can land a few ulps off it; the tolerance is far below one + # score step (0.01 at the finest), so it absorbs only that float error. if no_skill is not None and no_skill >= ceiling - _EPS: return CEILING scored = {arm: v for arm, v in arms.items() if v is not None} From 2563839a9936670025312269890ea680c9985085 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Fri, 30 Oct 2026 09:59:52 +0000 Subject: [PATCH 26/47] docs(headroom): a floor needs the no-skill arm scored Signed-off-by: Anton Volkov --- .../20261028_120000_napetrov_feat-eval-headroom.md | 2 +- dokimi/docs/evaluating-treatments.md | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/dokimi/changelog.d/20261028_120000_napetrov_feat-eval-headroom.md b/dokimi/changelog.d/20261028_120000_napetrov_feat-eval-headroom.md index 6cec77380..1e30b4c48 100644 --- a/dokimi/changelog.d/20261028_120000_napetrov_feat-eval-headroom.md +++ b/dokimi/changelog.d/20261028_120000_napetrov_feat-eval-headroom.md @@ -1,3 +1,3 @@ ### Added -- **`eval headroom` flags arms items with no headroom** — reads one or more `arms.v1` files of the same question set and flags an item as a `ceiling` when the no-skill arm's mean is at least `--ceiling` (default 0.9, on 0-1), or as a `floor` when every arm's mean is at most `--floor` (default 0). An item with fewer than `--min-reps` runs (default 3) behind it is `provisional`. The human table goes to stdout; `-o` writes a `headroom.v1` artifact with every item. The flags change no score. +- **`eval headroom` flags arms items with no headroom** — reads one or more `arms.v1` files of the same question set and flags an item as a `ceiling` when the no-skill arm's mean is at least `--ceiling` (default 0.9, on 0-1), or as a `floor` when the no-skill arm was scored, every arm that a run scored has a mean at most `--floor` (default 0), and at least one non-baseline arm scored. An item with fewer than `--min-reps` runs (default 3) behind it is `provisional`. The human table goes to stdout; `-o` writes a `headroom.v1` artifact with every item. The flags change no score. diff --git a/dokimi/docs/evaluating-treatments.md b/dokimi/docs/evaluating-treatments.md index cf1c6d9d3..5cf18d74b 100644 --- a/dokimi/docs/evaluating-treatments.md +++ b/dokimi/docs/evaluating-treatments.md @@ -736,9 +736,10 @@ python cli.py eval headroom run1/arms.json run2/arms.json run3/arms.json \ - **ceiling**: the no-skill arm's mean over the runs is at least `--ceiling` (0-1; the 0-100 aggregate divided by 100). Default 0.9. -- **floor**: every arm that a run scored has a mean at most `--floor`, and at least one - non-baseline arm scored. Default 0. An arm no run scored is left out of this test and - listed in the item's `unscored_arms`; the baseline alone at zero is not a floor. +- **floor**: the no-skill arm was scored, every arm that a run scored has a mean at most + `--floor`, and at least one non-baseline arm scored. Default 0. An arm no run scored is + left out of this test and listed in the item's `unscored_arms`; the baseline alone at zero + is not a floor. - **provisional**: fewer than `--min-reps` runs scored the item with its no-skill arm (the reps are those runs). Default 3. One file gives every item one rep, so every item is provisional. From 9a191ba3ad1a293d7574566560cd123aa715de1d Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Fri, 30 Oct 2026 10:00:10 +0000 Subject: [PATCH 27/47] docs(headroom): list the other exit-2 cases and the score scale Signed-off-by: Anton Volkov --- dokimi/docs/evaluating-treatments.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dokimi/docs/evaluating-treatments.md b/dokimi/docs/evaluating-treatments.md index 5cf18d74b..645e8c9e9 100644 --- a/dokimi/docs/evaluating-treatments.md +++ b/dokimi/docs/evaluating-treatments.md @@ -757,7 +757,9 @@ and shown rounded down, so 0.89996 prints as 0.89. Without `--no-skill-arm`, the declare the same baseline; the default is that baseline. With `--no-skill-arm`, the arm must be one the runs list (exit 2 otherwise, naming the arms that exist), and the declared baselines are not checked. A threshold out of range (`--ceiling` in (0, 1], `--floor` in -[0, ceiling), `--min-reps` at least 1) exits 2. Only arms runs are read. The flags are descriptive: they do not +[0, ceiling), `--min-reps` at least 1) exits 2. It also exits 2 when no run scored the +no-skill arm, or when `-o` is one of the input files. Scores are read as the 0-100 +aggregate divided by 100 and are not range-checked. Only arms runs are read. The flags are descriptive: they do not change any score or any gate. ## Scope and caveats From 7b69f5a5bf41431e2ad16f175baf61dd1b34cd1d Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 30 Oct 2026 08:25:00 -0700 Subject: [PATCH 28/47] test(cli): allowlist the four eval headroom flags with reasons The flag registry ratchet fails on the flags PR #279 adds after merging main. Each row names the PR and the migration that will move the concept. Signed-off-by: Nikolay Petrov --- dokimi/tests/data/cli_flags_allowlist.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dokimi/tests/data/cli_flags_allowlist.txt b/dokimi/tests/data/cli_flags_allowlist.txt index 6e9ed384d..daf322be6 100644 --- a/dokimi/tests/data/cli_flags_allowlist.txt +++ b/dokimi/tests/data/cli_flags_allowlist.txt @@ -220,3 +220,7 @@ --work-output-dir --work-repeats -o +--ceiling # eval headroom (#279) adds it: no registry concept yet; a F1b migration names it +--floor # eval headroom (#279) adds it: no registry concept yet; a F1b migration names it +--min-reps # eval headroom (#279) adds it: no registry concept yet; a F1b migration names it +--no-skill-arm # eval headroom (#279) adds it: no registry concept yet; a F1b migration names it From f259a4fb5dbf28d3cc669060b26950440e87d742 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 30 Oct 2026 08:25:00 -0700 Subject: [PATCH 29/47] fix(headroom): the -o guard survives an unresolvable path; floor wording names the no-skill arm A path that cannot be resolved (a symlink loop on Python 3.12 and older) made resolve() raise RuntimeError out of the guard. It now counts as not an input, and its own open reports the error. The floor test needs an arm other than the no-skill arm to have scored, so the docs and the changelog say so. Signed-off-by: Nikolay Petrov --- ...1028_120000_napetrov_feat-eval-headroom.md | 2 +- dokimi/docs/evaluating-treatments.md | 4 +-- dokimi/dokimi/commands/headroom.py | 11 +++++- dokimi/dokimi/eval/headroom.py | 5 +-- dokimi/tests/test_headroom.py | 34 +++++++++++++++++++ 5 files changed, 50 insertions(+), 6 deletions(-) diff --git a/dokimi/changelog.d/20261028_120000_napetrov_feat-eval-headroom.md b/dokimi/changelog.d/20261028_120000_napetrov_feat-eval-headroom.md index 1e30b4c48..9abe75096 100644 --- a/dokimi/changelog.d/20261028_120000_napetrov_feat-eval-headroom.md +++ b/dokimi/changelog.d/20261028_120000_napetrov_feat-eval-headroom.md @@ -1,3 +1,3 @@ ### Added -- **`eval headroom` flags arms items with no headroom** — reads one or more `arms.v1` files of the same question set and flags an item as a `ceiling` when the no-skill arm's mean is at least `--ceiling` (default 0.9, on 0-1), or as a `floor` when the no-skill arm was scored, every arm that a run scored has a mean at most `--floor` (default 0), and at least one non-baseline arm scored. An item with fewer than `--min-reps` runs (default 3) behind it is `provisional`. The human table goes to stdout; `-o` writes a `headroom.v1` artifact with every item. The flags change no score. +- **`eval headroom` flags arms items with no headroom** — reads one or more `arms.v1` files of the same question set and flags an item as a `ceiling` when the no-skill arm's mean is at least `--ceiling` (default 0.9, on 0-1), or as a `floor` when the no-skill arm was scored, every arm that a run scored has a mean at most `--floor` (default 0), and at least one arm other than the no-skill arm scored. An item with fewer than `--min-reps` runs (default 3) behind it is `provisional`. The human table goes to stdout; `-o` writes a `headroom.v1` artifact with every item. The flags change no score. diff --git a/dokimi/docs/evaluating-treatments.md b/dokimi/docs/evaluating-treatments.md index 12a233e3e..3f4c76138 100644 --- a/dokimi/docs/evaluating-treatments.md +++ b/dokimi/docs/evaluating-treatments.md @@ -738,8 +738,8 @@ python cli.py eval headroom run1/arms.json run2/arms.json run3/arms.json \ - **ceiling**: the no-skill arm's mean over the runs is at least `--ceiling` (0-1; the 0-100 aggregate divided by 100). Default 0.9. - **floor**: the no-skill arm was scored, every arm that a run scored has a mean at most - `--floor`, and at least one non-baseline arm scored. Default 0. An arm no run scored is - left out of this test and listed in the item's `unscored_arms`; the baseline alone at zero + `--floor`, and at least one arm other than the no-skill arm scored. Default 0. An arm no run scored is + left out of this test and listed in the item's `unscored_arms`; the no-skill arm alone at zero is not a floor. - **provisional**: fewer than `--min-reps` runs scored the item with its no-skill arm (the reps are those runs). Default 3. One file gives every item one rep, so every item is diff --git a/dokimi/dokimi/commands/headroom.py b/dokimi/dokimi/commands/headroom.py index 84ba753ba..00114e069 100644 --- a/dokimi/dokimi/commands/headroom.py +++ b/dokimi/dokimi/commands/headroom.py @@ -26,11 +26,20 @@ def _content_digest(path) -> str: return hashlib.sha256(json.dumps(data, sort_keys=True).encode("utf-8")).hexdigest() +def _same_file(a, b) -> bool: + """Whether two paths name one file. A path that cannot be resolved (a symlink loop on some Pythons) + is not taken as an input: its own open reports the error.""" + try: + return Path(a).resolve() == Path(b).resolve() + except (OSError, RuntimeError): + return False + + def cmd_eval_headroom(args: argparse.Namespace) -> None: from dokimi.artifacts import ArtifactValidationError, save_artifact from dokimi.eval.headroom import format_table, headroom, repeated_ids - if args.output and Path(args.output).resolve() in {Path(p).resolve() for p in args.arms}: + if args.output and any(_same_file(args.output, p) for p in args.arms): print(f"Error: -o {args.output} is one of the input arms files; it would overwrite that run", file=sys.stderr) sys.exit(2) runs: list[records.Run] = [] diff --git a/dokimi/dokimi/eval/headroom.py b/dokimi/dokimi/eval/headroom.py index 8bc719a2a..43cce4075 100644 --- a/dokimi/dokimi/eval/headroom.py +++ b/dokimi/dokimi/eval/headroom.py @@ -10,8 +10,9 @@ grader rather than at any skill. An item's floor test leaves out an arm no run scored (listed in ``unscored_arms``). It needs the -no-skill arm scored on the item and at least one non-baseline arm that did score: the baseline alone -at zero shows nothing about a skill, and an unscored baseline shows nothing either. +no-skill arm scored on the item and at least one arm other than the no-skill arm that did score: +the no-skill arm alone at zero shows nothing about a skill, and an unscored no-skill arm shows +nothing either. An item with fewer than ``min_reps`` runs behind it is ``provisional``: one run's flag may be noise. Its reps are the number of runs in which the no-skill arm scored it, so a single artifact diff --git a/dokimi/tests/test_headroom.py b/dokimi/tests/test_headroom.py index e0362da1e..30886fd34 100644 --- a/dokimi/tests/test_headroom.py +++ b/dokimi/tests/test_headroom.py @@ -10,6 +10,8 @@ import argparse import json +import os +import sys import pytest @@ -487,6 +489,38 @@ def test_command_refuses_an_output_that_is_one_of_its_inputs(tmp_path, capsys): assert "overwrite" in capsys.readouterr().err +def test_an_output_whose_path_cannot_be_resolved_does_not_crash_the_guard(tmp_path, monkeypatch): + # Python 3.12 and older raise RuntimeError from resolve() on a symlink loop; 3.13+ do not. + # The guard must not turn that into a traceback: an unresolvable path is not an input. + import pathlib + + arms = _arms_file(tmp_path / "arms.json", {"1": {"baseline": 99, "skill": 99}}) + out = tmp_path / "loop.json" + real_resolve = pathlib.Path.resolve + + def resolve(self, *a, **k): + if self.name == out.name: + raise RuntimeError(f"Symlink loop from {self}") + return real_resolve(self, *a, **k) + + monkeypatch.setattr(pathlib.Path, "resolve", resolve) + cmd_eval_headroom(_args([arms], output=out)) + assert out.exists() + + +@pytest.mark.skipif(sys.platform == "win32", reason="symlinks need privileges on Windows") +def test_an_output_that_is_a_symlink_to_an_input_is_refused(tmp_path, capsys): + path = _arms_file(tmp_path / "arms.json", {"1": {"baseline": 99, "skill": 99}}) + alias = tmp_path / "alias.json" + os.symlink(path.name, alias) + before = path.read_bytes() + with pytest.raises(SystemExit) as exc: + cmd_eval_headroom(_args([path], output=alias)) + assert exc.value.code == 2 + assert path.read_bytes() == before + assert "overwrite" in capsys.readouterr().err + + def test_a_reserialized_copy_of_a_run_counts_once(tmp_path, capsys): arms = _arms_file(tmp_path / "run1.json", {"1": {"baseline": 95, "skill": 40}}) copy = tmp_path / "copy.json" From 9872a50b3fba36468c1c0b8a3b710afa46948210 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 30 Oct 2026 08:27:27 -0700 Subject: [PATCH 30/47] docs(cli): regenerate the CLI snapshot for eval headroom The CI mypy job diffs docs/reference/cli-interface.json against the tree and fails on the four headroom flags and the new command until the snapshot is regenerated. Signed-off-by: Nikolay Petrov --- dokimi/docs/reference/cli-interface.json | 120 +++++++++++++++++++++++ 1 file changed, 120 insertions(+) diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index 1bbc56081..fbb0ffd28 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -4201,6 +4201,126 @@ "subcommand_required": null, "subcommands": {} }, + "headroom": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "arms", + "kind": "argument", + "nargs": "+", + "position": 0, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 0.9, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--ceiling", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "float" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 0.0, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--floor", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "float" + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 3, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--min-reps", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--no-skill-arm", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--output/-o", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi eval headroom", + "subcommand_required": null, + "subcommands": {} + }, "judge-validity": { "allow_abbrev": true, "params": [ From df428e33ed643de9d109b9bd776e6881c592e585 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 30 Oct 2026 10:23:36 -0700 Subject: [PATCH 31/47] fix(headroom): refuse a positive ceiling or floor below one score step A threshold of 0.005 cannot tell two 0.01-step scores apart, so eval headroom refuses it (exit 2) instead of flagging on noise. Zero stays valid for the floor; the ceiling must be at least 0.01. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/eval/headroom.py | 6 ++++++ dokimi/tests/test_headroom.py | 11 +++++++++++ 2 files changed, 17 insertions(+) diff --git a/dokimi/dokimi/eval/headroom.py b/dokimi/dokimi/eval/headroom.py index 43cce4075..d6304adc1 100644 --- a/dokimi/dokimi/eval/headroom.py +++ b/dokimi/dokimi/eval/headroom.py @@ -38,6 +38,8 @@ DEFAULT_MIN_REPS = 3 _EPS = 1e-9 # float error bound for a threshold comparison on a mean of 0-1 scores +# One score step (scores are read to 0.01): a positive threshold below it cannot tell two scores apart. +_MIN_THRESHOLD = 0.01 CEILING = "ceiling" FLOOR = "floor" @@ -71,6 +73,10 @@ def check_thresholds(ceiling: float, floor: float, min_reps: int) -> None: raise ValueError(f"ceiling must be in (0, 1], not {ceiling!r}") if not 0.0 <= floor < ceiling: raise ValueError(f"floor must be in [0, ceiling), not {floor!r} (ceiling {ceiling!r})") + if ceiling < _MIN_THRESHOLD: + raise ValueError(f"ceiling must be at least {_MIN_THRESHOLD}, not {ceiling!r}") + if 0.0 < floor < _MIN_THRESHOLD: + raise ValueError(f"floor must be 0 or at least {_MIN_THRESHOLD}, not {floor!r}") if min_reps < 1: raise ValueError(f"min_reps must be at least 1, not {min_reps!r}") diff --git a/dokimi/tests/test_headroom.py b/dokimi/tests/test_headroom.py index 30886fd34..d5f4259ea 100644 --- a/dokimi/tests/test_headroom.py +++ b/dokimi/tests/test_headroom.py @@ -232,6 +232,8 @@ def test_the_table_lists_the_control_arm_once(): ({"floor": -0.1}, r"floor must be in \[0, ceiling\), not -0\.1"), ({"min_reps": 0}, r"min_reps must be at least 1, not 0"), ({"min_reps": -1}, r"min_reps must be at least 1, not -1"), + ({"ceiling": 0.005}, r"ceiling must be at least 0\.01, not 0\.005"), + ({"floor": 0.005}, r"floor must be 0 or at least 0\.01, not 0\.005"), ]) def test_unusable_thresholds_are_refused(kwargs, message): run = _run([("1", {"baseline": 50, "skill": 50})]) @@ -246,6 +248,7 @@ def test_the_threshold_boundaries_are_accepted(): assert _flags(result) == {"1": CEILING, "2": None} assert result["thresholds"] == {"ceiling": 1.0, "floor": 0.0, "min_reps": 1} assert headroom([run], ceiling=0.5, floor=0.49)["thresholds"]["floor"] == 0.49 + assert headroom([run], ceiling=0.01, floor=0.0, min_reps=1)["thresholds"]["ceiling"] == 0.01 def test_no_runs_are_refused(): @@ -349,6 +352,14 @@ def _arms_file(path, scores, baseline="baseline"): return path +def test_command_exits_2_on_a_positive_threshold_below_one_score_step(tmp_path, capsys): + path = _arms_file(tmp_path / "arms.json", {"1": {"baseline": 99, "skill": 99}}) + with pytest.raises(SystemExit) as exc: + cmd_eval_headroom(_args([path], floor=0.005)) + assert exc.value.code == 2 + assert "floor must be 0 or at least 0.01, not 0.005" in capsys.readouterr().err + + def test_command_refuses_an_unknown_no_skill_arm_and_lists_the_arms(tmp_path, capsys): path = _arms_file(tmp_path / "arms.json", {"1": {"baseline": 99, "skill": 99}}) with pytest.raises(SystemExit) as exc: From 1e164b8decb57cc040cacabd1804063bb8f844a6 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 30 Oct 2026 13:55:52 -0700 Subject: [PATCH 32/47] fix(headroom): guard the .rejected sibling of -o, sort long digit ids, list with_docs arms - -o is refused when its .rejected sibling is an input too: save_artifact deletes that sibling after a good save, so an input under that name was lost. - A digit id past Python's int() digit limit (4300) no longer raises: ASCII digit ids sort by length then digits, the others as text. - An arms record's with_docs and without_docs arms are listed (arm_names() dropped them), so an unscored one shows in unscored_arms and the note. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/headroom.py | 13 +++++++++++-- dokimi/dokimi/eval/headroom.py | 22 +++++++++++++++++----- dokimi/tests/test_headroom.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 7 deletions(-) diff --git a/dokimi/dokimi/commands/headroom.py b/dokimi/dokimi/commands/headroom.py index 00114e069..cabc4c0ab 100644 --- a/dokimi/dokimi/commands/headroom.py +++ b/dokimi/dokimi/commands/headroom.py @@ -35,12 +35,21 @@ def _same_file(a, b) -> bool: return False +def _clobbers_an_input(output, inputs) -> bool: + """Whether a save to ``output`` would overwrite an input: the file itself, or the ``.rejected`` + sibling save_artifact writes on a schema failure and deletes after a good save.""" + out = Path(output) + rejected = out.with_name(out.name + ".rejected") + return any(_same_file(out, p) or _same_file(rejected, p) for p in inputs) + + def cmd_eval_headroom(args: argparse.Namespace) -> None: from dokimi.artifacts import ArtifactValidationError, save_artifact from dokimi.eval.headroom import format_table, headroom, repeated_ids - if args.output and any(_same_file(args.output, p) for p in args.arms): - print(f"Error: -o {args.output} is one of the input arms files; it would overwrite that run", file=sys.stderr) + if args.output and _clobbers_an_input(args.output, args.arms): + print(f"Error: -o {args.output} would overwrite one of the input arms files (or its .rejected sibling); " + "the run would be lost", file=sys.stderr) sys.exit(2) runs: list[records.Run] = [] inputs: list[str] = [] diff --git a/dokimi/dokimi/eval/headroom.py b/dokimi/dokimi/eval/headroom.py index d6304adc1..43dd018fb 100644 --- a/dokimi/dokimi/eval/headroom.py +++ b/dokimi/dokimi/eval/headroom.py @@ -31,7 +31,7 @@ from dataclasses import dataclass from typing import Any, Dict, List, Optional, Sequence, Set, Tuple -from dokimi.records import Run, arm_names +from dokimi.records import Record, Run DEFAULT_CEILING = 0.9 DEFAULT_FLOOR = 0.0 @@ -183,10 +183,18 @@ def _known_arms(runs: Sequence[Run]) -> Set[str]: for run in runs: known.update(run.conditions) for rec in run: - known.update(arm_names(rec)) + known.update(_listed_arms(rec)) return known +def _listed_arms(rec: Record) -> Set[str]: + """The arms an arms record lists, ``null`` ones included. Unlike ``arm_names()``, keeps + ``with_docs`` and ``without_docs``, which arms.v1 allows as arm names; a binary record lists none.""" + if rec.raw.get("arms") is None and rec.raw.get("scores") is None: + return set() + return set(rec.conditions) + + def repeated_ids(run: Run) -> List[str]: """The question ids that more than one record of ``run`` carries, in first-seen order.""" seen: Set[str] = set() @@ -209,7 +217,7 @@ def _collect(runs: Sequence[Run]) -> Tuple[Dict[str, Dict[str, List[float]]], Di f"run {index} lists question_id {dupes[0]!r} more than once; " "a run holds each question once (a repeat would count as a second run)") for rec in run: - names = set(arm_names(rec)) | set(rec.scores) + names = _listed_arms(rec) | set(rec.scores) arms_of.setdefault(rec.key, set()).update(names) per_arm = scores.setdefault(rec.key, {}) for arm in names: @@ -220,8 +228,12 @@ def _collect(runs: Sequence[Run]) -> Tuple[Dict[str, Dict[str, List[float]]], Di def _sort_key(qid: str): - # isdigit() also accepts superscripts and other digits int() refuses: those sort as text, not numbers. - return (0, int(qid), qid) if qid.isascii() and qid.isdigit() else (1, 0, qid) + # ASCII digits sort by value without int(), which refuses over 4300 digits: no leading zeros, + # then length, then digits. Any other id (isdigit() also takes superscripts) sorts after them, as text. + if qid.isascii() and qid.isdigit(): + digits = qid.lstrip("0") or "0" + return (0, len(digits), digits, qid) + return (1, 0, "", qid) def format_table(result: Dict[str, Any]) -> str: diff --git a/dokimi/tests/test_headroom.py b/dokimi/tests/test_headroom.py index d5f4259ea..2e356c4db 100644 --- a/dokimi/tests/test_headroom.py +++ b/dokimi/tests/test_headroom.py @@ -210,6 +210,21 @@ def test_a_non_ascii_digit_question_id_is_kept_and_sorted_as_text(): assert [i["question_id"] for i in headroom([run])["items"]] == ["2", "10", "²"] +def test_a_digit_question_id_past_the_int_limit_is_sorted_not_refused(): + # int() refuses more than 4300 digits (Python's str-to-int limit): a long numeric id still sorts by value. + long_id = "1" * 5000 + run = _run([(long_id, {"baseline": 50, "skill": 50}), ("10", {"baseline": 50, "skill": 50}), + ("0002", {"baseline": 50, "skill": 50})]) + assert [i["question_id"] for i in headroom([run])["items"]] == ["0002", "10", long_id] + + +def test_an_arm_named_with_docs_that_no_run_scored_is_listed_as_unscored(): + # arms.v1 allows with_docs as an arm name; arm_names() drops it, so an unscored one went unlisted. + run = _run([("1", {"baseline": 40, "skill": 60, "with_docs": None})]) + item = headroom([run], min_reps=1)["items"][0] + assert item["unscored_arms"] == ["with_docs"] and item["arms"]["with_docs"] is None + + def test_the_table_and_json_do_not_round_0_89996_up_to_the_ceiling(): run = _run([("1", {"baseline": 89.996, "skill": 99})]) result = headroom([run], min_reps=2) @@ -500,6 +515,19 @@ def test_command_refuses_an_output_that_is_one_of_its_inputs(tmp_path, capsys): assert "overwrite" in capsys.readouterr().err +def test_command_refuses_an_output_whose_rejected_sibling_is_an_input(tmp_path, capsys): + # save_artifact deletes .rejected after a good save: an input under that name would be lost. + run = _arms_file(tmp_path / "run.json", {"1": {"baseline": 99, "skill": 99}}) + stale = tmp_path / "headroom.json.rejected" + stale.write_bytes(run.read_bytes()) + before = stale.read_bytes() + with pytest.raises(SystemExit) as exc: + cmd_eval_headroom(_args([stale], output=tmp_path / "headroom.json")) + assert exc.value.code == 2 + assert stale.read_bytes() == before + assert "overwrite" in capsys.readouterr().err + + def test_an_output_whose_path_cannot_be_resolved_does_not_crash_the_guard(tmp_path, monkeypatch): # Python 3.12 and older raise RuntimeError from resolve() on a symlink loop; 3.13+ do not. # The guard must not turn that into a traceback: an unresolvable path is not an input. From 6a1e7f23c14936a05b78164bbef286b2ac9c8092 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 30 Oct 2026 13:55:55 -0700 Subject: [PATCH 33/47] docs(headroom): state the threshold ranges the code accepts --ceiling and --floor refuse a positive value below one score step (0.01), so the docs now give ranges that match the code. A file gives each item zero or one rep, not always one. Signed-off-by: Nikolay Petrov --- dokimi/docs/evaluating-treatments.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dokimi/docs/evaluating-treatments.md b/dokimi/docs/evaluating-treatments.md index 3f4c76138..8f2086bfc 100644 --- a/dokimi/docs/evaluating-treatments.md +++ b/dokimi/docs/evaluating-treatments.md @@ -742,8 +742,8 @@ python cli.py eval headroom run1/arms.json run2/arms.json run3/arms.json \ left out of this test and listed in the item's `unscored_arms`; the no-skill arm alone at zero is not a floor. - **provisional**: fewer than `--min-reps` runs scored the item with its no-skill arm (the - reps are those runs). Default 3. One file gives every item one rep, so every item is - provisional. + reps are those runs). Default 3. A file gives each item zero or one rep (zero when its + no-skill arm was not scored in that file), so with the default of 3 every item is provisional. A run is one file. A file repeated on the command line, or holding the same JSON content as an earlier one (whitespace and key order do not count), is skipped with a note, so it does @@ -757,8 +757,8 @@ command prints `Error: ...` on stderr and exits 2, after the table. Scores are c and shown rounded down, so 0.89996 prints as 0.89. Without `--no-skill-arm`, the runs must declare the same baseline; the default is that baseline. With `--no-skill-arm`, the arm must be one the runs list (exit 2 otherwise, naming the arms that exist), and the declared -baselines are not checked. A threshold out of range (`--ceiling` in (0, 1], `--floor` in -[0, ceiling), `--min-reps` at least 1) exits 2. It also exits 2 when no run scored the +baselines are not checked. A threshold out of range exits 2: `--ceiling` in [0.01, 1], +`--floor` 0 or in [0.01, ceiling), `--min-reps` at least 1. It also exits 2 when no run scored the no-skill arm, or when `-o` is one of the input files. Scores are read as the 0-100 aggregate divided by 100 and are not range-checked. Only arms runs are read. The flags are descriptive: they do not change any score or any gate. From dfd69466e47e4e58ca9461d5bce2adeeb4d88706 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 30 Oct 2026 13:57:19 -0700 Subject: [PATCH 34/47] test(headroom): pin the -o guard against a dotdot spelling of an input The output sub/../arms.json names the input arms.json; the guard resolves both sides, so it refuses it (exit 2, input bytes unchanged). Without the output-side resolve the test fails. Signed-off-by: Nikolay Petrov --- dokimi/tests/test_headroom.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/dokimi/tests/test_headroom.py b/dokimi/tests/test_headroom.py index 2e356c4db..f09a1e0e5 100644 --- a/dokimi/tests/test_headroom.py +++ b/dokimi/tests/test_headroom.py @@ -416,6 +416,18 @@ def test_command_exits_2_when_the_output_cannot_be_written(tmp_path, capsys): assert capsys.readouterr().err.startswith("Error: ") +def test_command_refuses_an_output_spelled_through_a_dotdot_to_an_input(tmp_path, capsys): + # The same file under another spelling: the input as given, the output as sub/../arms.json. + path = _arms_file(tmp_path / "arms.json", {"1": {"baseline": 99, "skill": 99}}) + (tmp_path / "sub").mkdir() + before = path.read_bytes() + with pytest.raises(SystemExit) as exc: + cmd_eval_headroom(_args([str(path)], output=str(tmp_path / "sub" / ".." / "arms.json"))) + assert exc.value.code == 2 + assert path.read_bytes() == before + assert "overwrite" in capsys.readouterr().err + + def test_command_refuses_a_question_id_repeated_within_one_file(tmp_path, capsys): path = tmp_path / "dup.json" path.write_text(json.dumps({"baseline_arm": "baseline", "answers": [ From 5902edad92c430fcabd3fd9f855d6b52ff7dd3be Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 30 Oct 2026 14:25:42 -0700 Subject: [PATCH 35/47] fix(headroom): refuse a directory as -o with exit 2, not a ValueError -o . and -o ./ made the guard's with_name() raise an uncaught ValueError (a directory has no file name), so the command printed a traceback and exited 1. A directory is now refused before the guard runs, with the usage exit code 2 the -o contract names. Tests pin the three spellings (a directory path, ".", "./"); the two dot spellings fail before the fix. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/headroom.py | 4 ++++ dokimi/tests/test_headroom.py | 14 ++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/dokimi/dokimi/commands/headroom.py b/dokimi/dokimi/commands/headroom.py index cabc4c0ab..9c1280bb1 100644 --- a/dokimi/dokimi/commands/headroom.py +++ b/dokimi/dokimi/commands/headroom.py @@ -47,6 +47,10 @@ def cmd_eval_headroom(args: argparse.Namespace) -> None: from dokimi.artifacts import ArtifactValidationError, save_artifact from dokimi.eval.headroom import format_table, headroom, repeated_ids + if args.output and Path(args.output).is_dir(): + # A directory has no file name for save_artifact's sibling paths ("." has an empty name). + print(f"Error: -o {args.output} is a directory; it needs a file name", file=sys.stderr) + sys.exit(2) if args.output and _clobbers_an_input(args.output, args.arms): print(f"Error: -o {args.output} would overwrite one of the input arms files (or its .rejected sibling); " "the run would be lost", file=sys.stderr) diff --git a/dokimi/tests/test_headroom.py b/dokimi/tests/test_headroom.py index f09a1e0e5..186ca45c0 100644 --- a/dokimi/tests/test_headroom.py +++ b/dokimi/tests/test_headroom.py @@ -416,6 +416,20 @@ def test_command_exits_2_when_the_output_cannot_be_written(tmp_path, capsys): assert capsys.readouterr().err.startswith("Error: ") +@pytest.mark.parametrize("spelling", ["dir", "dot", "dot-slash"]) +def test_command_exits_2_when_the_output_is_a_directory(tmp_path, monkeypatch, capsys, spelling): + # A directory has no file name to write: the save would raise ValueError (for "." and "./") or + # fail on the directory itself. Refuse it up front, as the -o contract says for an unwritable -o. + path = _arms_file(tmp_path / "arms.json", {"1": {"baseline": 95, "skill": 95}}) + (tmp_path / "out").mkdir() + monkeypatch.chdir(tmp_path) + out = {"dir": str(tmp_path / "out"), "dot": ".", "dot-slash": "./"}[spelling] + with pytest.raises(SystemExit) as exc: + cmd_eval_headroom(_args([path], output=out)) + assert exc.value.code == 2 + assert capsys.readouterr().err.startswith("Error: ") + + def test_command_refuses_an_output_spelled_through_a_dotdot_to_an_input(tmp_path, capsys): # The same file under another spelling: the input as given, the output as sub/../arms.json. path = _arms_file(tmp_path / "arms.json", {"1": {"baseline": 99, "skill": 99}}) From 438c6423c985f4f1ed1525ca5ee5fee4f6749c3b Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 30 Oct 2026 14:31:48 -0700 Subject: [PATCH 36/47] fix(headroom): refuse a directory spelled with a trailing slash as -o Path() drops a trailing slash, so -o newdir/ with newdir absent passed the directory check and wrote a plain file named newdir while printing "Wrote headroom artifact to newdir/". The check now also refuses a trailing separator, and "", "." and ".." by name. Red first: the absent-with-slash case wrote the file before this change. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/headroom.py | 13 +++++++++++-- dokimi/tests/test_headroom.py | 11 +++++++---- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/dokimi/dokimi/commands/headroom.py b/dokimi/dokimi/commands/headroom.py index 9c1280bb1..9092ba41a 100644 --- a/dokimi/dokimi/commands/headroom.py +++ b/dokimi/dokimi/commands/headroom.py @@ -14,6 +14,7 @@ import argparse import hashlib import json +import os import sys from pathlib import Path @@ -35,6 +36,13 @@ def _same_file(a, b) -> bool: return False +def _names_a_directory(output) -> bool: + """Whether ``-o`` spells a directory: an existing one, a trailing slash, or ``.``/``..``/empty + (which have no file name to write beside).""" + text = str(output) + return text.endswith(("/", os.sep)) or Path(text).name in ("", ".", "..") or Path(text).is_dir() + + def _clobbers_an_input(output, inputs) -> bool: """Whether a save to ``output`` would overwrite an input: the file itself, or the ``.rejected`` sibling save_artifact writes on a schema failure and deletes after a good save.""" @@ -47,8 +55,9 @@ def cmd_eval_headroom(args: argparse.Namespace) -> None: from dokimi.artifacts import ArtifactValidationError, save_artifact from dokimi.eval.headroom import format_table, headroom, repeated_ids - if args.output and Path(args.output).is_dir(): - # A directory has no file name for save_artifact's sibling paths ("." has an empty name). + if args.output and _names_a_directory(args.output): + # A directory has no file name for save_artifact's sibling paths ("." has an empty name), and + # Path() drops a trailing slash, so "newdir/" would otherwise write a plain file named newdir. print(f"Error: -o {args.output} is a directory; it needs a file name", file=sys.stderr) sys.exit(2) if args.output and _clobbers_an_input(args.output, args.arms): diff --git a/dokimi/tests/test_headroom.py b/dokimi/tests/test_headroom.py index 186ca45c0..26bd1b338 100644 --- a/dokimi/tests/test_headroom.py +++ b/dokimi/tests/test_headroom.py @@ -416,18 +416,21 @@ def test_command_exits_2_when_the_output_cannot_be_written(tmp_path, capsys): assert capsys.readouterr().err.startswith("Error: ") -@pytest.mark.parametrize("spelling", ["dir", "dot", "dot-slash"]) +@pytest.mark.parametrize("spelling", ["dir", "dot", "dot-slash", "absent-with-slash"]) def test_command_exits_2_when_the_output_is_a_directory(tmp_path, monkeypatch, capsys, spelling): - # A directory has no file name to write: the save would raise ValueError (for "." and "./") or - # fail on the directory itself. Refuse it up front, as the -o contract says for an unwritable -o. + # A directory has no file name to write: the save would raise ValueError (for "." and "./"), fail + # on the directory itself, or (for a directory that does not exist yet, spelled with a slash) write + # a plain file under the directory's name. Refuse all of them up front, as the -o contract says. path = _arms_file(tmp_path / "arms.json", {"1": {"baseline": 95, "skill": 95}}) (tmp_path / "out").mkdir() monkeypatch.chdir(tmp_path) - out = {"dir": str(tmp_path / "out"), "dot": ".", "dot-slash": "./"}[spelling] + out = {"dir": str(tmp_path / "out"), "dot": ".", "dot-slash": "./", + "absent-with-slash": str(tmp_path / "newdir") + "/"}[spelling] with pytest.raises(SystemExit) as exc: cmd_eval_headroom(_args([path], output=out)) assert exc.value.code == 2 assert capsys.readouterr().err.startswith("Error: ") + assert not (tmp_path / "newdir").exists() def test_command_refuses_an_output_spelled_through_a_dotdot_to_an_input(tmp_path, capsys): From ebe6575c9be53fc66df0e314fb04598fd477f519 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 30 Oct 2026 14:38:19 -0700 Subject: [PATCH 37/47] fix(headroom): refuse "newdir/." and an unstat-able -o without a crash Two gaps in the directory refusal added for -o: - "newdir/." (absent) ended in ".", not a separator, so its last component was "newdir" and the save wrote a plain file of that name. The check now reads the last component of the spelling: "", "." or ".." refuses. - is_dir() raises PermissionError under a parent without search permission, which escaped the guard as a traceback. An unstat-able path is now not a directory, and the save reports it as before. The message says "names a directory"; the exit-2 list in evaluating-treatments.md names the directory and .rejected refusals. Signed-off-by: Nikolay Petrov --- dokimi/docs/evaluating-treatments.md | 2 +- dokimi/dokimi/commands/headroom.py | 14 ++++++++---- dokimi/tests/test_headroom.py | 34 ++++++++++++++++++++++++---- 3 files changed, 40 insertions(+), 10 deletions(-) diff --git a/dokimi/docs/evaluating-treatments.md b/dokimi/docs/evaluating-treatments.md index 8f2086bfc..6f83507c3 100644 --- a/dokimi/docs/evaluating-treatments.md +++ b/dokimi/docs/evaluating-treatments.md @@ -759,7 +759,7 @@ declare the same baseline; the default is that baseline. With `--no-skill-arm`, be one the runs list (exit 2 otherwise, naming the arms that exist), and the declared baselines are not checked. A threshold out of range exits 2: `--ceiling` in [0.01, 1], `--floor` 0 or in [0.01, ceiling), `--min-reps` at least 1. It also exits 2 when no run scored the -no-skill arm, or when `-o` is one of the input files. Scores are read as the 0-100 +no-skill arm, or when `-o` is one of the input files (or its `.rejected` sibling) or names a directory. Scores are read as the 0-100 aggregate divided by 100 and are not range-checked. Only arms runs are read. The flags are descriptive: they do not change any score or any gate. diff --git a/dokimi/dokimi/commands/headroom.py b/dokimi/dokimi/commands/headroom.py index 9092ba41a..fb7b2019e 100644 --- a/dokimi/dokimi/commands/headroom.py +++ b/dokimi/dokimi/commands/headroom.py @@ -37,10 +37,16 @@ def _same_file(a, b) -> bool: def _names_a_directory(output) -> bool: - """Whether ``-o`` spells a directory: an existing one, a trailing slash, or ``.``/``..``/empty - (which have no file name to write beside).""" + """Whether ``-o`` spells a directory: its last component is empty, ``.`` or ``..`` (a trailing + separator, with or without ``/.``), or it is an existing directory. A path that cannot be + stat-ed is not one: the save reports it, as before.""" text = str(output) - return text.endswith(("/", os.sep)) or Path(text).name in ("", ".", "..") or Path(text).is_dir() + if text.replace(os.sep, "/").rsplit("/", 1)[-1] in ("", ".", ".."): + return True + try: + return Path(text).is_dir() + except (OSError, ValueError): + return False def _clobbers_an_input(output, inputs) -> bool: @@ -58,7 +64,7 @@ def cmd_eval_headroom(args: argparse.Namespace) -> None: if args.output and _names_a_directory(args.output): # A directory has no file name for save_artifact's sibling paths ("." has an empty name), and # Path() drops a trailing slash, so "newdir/" would otherwise write a plain file named newdir. - print(f"Error: -o {args.output} is a directory; it needs a file name", file=sys.stderr) + print(f"Error: -o {args.output} names a directory; it needs a file name", file=sys.stderr) sys.exit(2) if args.output and _clobbers_an_input(args.output, args.arms): print(f"Error: -o {args.output} would overwrite one of the input arms files (or its .rejected sibling); " diff --git a/dokimi/tests/test_headroom.py b/dokimi/tests/test_headroom.py index 26bd1b338..680d39562 100644 --- a/dokimi/tests/test_headroom.py +++ b/dokimi/tests/test_headroom.py @@ -416,23 +416,47 @@ def test_command_exits_2_when_the_output_cannot_be_written(tmp_path, capsys): assert capsys.readouterr().err.startswith("Error: ") -@pytest.mark.parametrize("spelling", ["dir", "dot", "dot-slash", "absent-with-slash"]) +@pytest.mark.parametrize("spelling", ["dir", "dot", "dot-slash", "absent-with-slash", "absent-with-slash-dot"]) def test_command_exits_2_when_the_output_is_a_directory(tmp_path, monkeypatch, capsys, spelling): # A directory has no file name to write: the save would raise ValueError (for "." and "./"), fail - # on the directory itself, or (for a directory that does not exist yet, spelled with a slash) write - # a plain file under the directory's name. Refuse all of them up front, as the -o contract says. + # on the directory itself, or (for a directory that does not exist yet, spelled with a slash or a + # trailing "/.") write a plain file under the directory's name. Refuse all of them up front. path = _arms_file(tmp_path / "arms.json", {"1": {"baseline": 95, "skill": 95}}) (tmp_path / "out").mkdir() monkeypatch.chdir(tmp_path) out = {"dir": str(tmp_path / "out"), "dot": ".", "dot-slash": "./", - "absent-with-slash": str(tmp_path / "newdir") + "/"}[spelling] + "absent-with-slash": str(tmp_path / "newdir") + "/", + "absent-with-slash-dot": str(tmp_path / "newdir") + "/."}[spelling] with pytest.raises(SystemExit) as exc: cmd_eval_headroom(_args([path], output=out)) assert exc.value.code == 2 - assert capsys.readouterr().err.startswith("Error: ") + assert "names a directory" in capsys.readouterr().err assert not (tmp_path / "newdir").exists() +def test_an_output_whose_directory_check_cannot_stat_does_not_crash_the_guard(tmp_path, monkeypatch, capsys): + # A parent without search permission makes is_dir() raise PermissionError on Python 3.12 and 3.13. + # The check must treat that path as not a directory and let the save report it, as before. + import pathlib + + arms = _arms_file(tmp_path / "arms.json", {"1": {"baseline": 99, "skill": 99}}) + out = tmp_path / "locked.json" + real_is_dir = pathlib.Path.is_dir + + def is_dir(self): + if self.name == out.name: + raise PermissionError(13, "Permission denied", str(self)) + return real_is_dir(self) + + monkeypatch.setattr(pathlib.Path, "is_dir", is_dir) + try: + cmd_eval_headroom(_args([arms], output=out)) + code = 0 + except SystemExit as exc: + code = exc.code + assert code in (0, 2) + + def test_command_refuses_an_output_spelled_through_a_dotdot_to_an_input(tmp_path, capsys): # The same file under another spelling: the input as given, the output as sub/../arms.json. path = _arms_file(tmp_path / "arms.json", {"1": {"baseline": 99, "skill": 99}}) From d56fd0d95a629033916cd2ab2c49de52e0bd307d Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 30 Oct 2026 14:44:14 -0700 Subject: [PATCH 38/47] fix(headroom): name the -o refusal as "not a file name" and pin the cases A trailing separator on an existing regular file (-o existing.json/) was refused with "names a directory", which is false for a file. The message now says the spelling is not a file name, which holds for both. The directory test pins the wording and that an existing file is left alone, and the stat-error test asserts the save writes the file when the directory check cannot stat its path. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/headroom.py | 3 ++- dokimi/tests/test_headroom.py | 12 ++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/dokimi/dokimi/commands/headroom.py b/dokimi/dokimi/commands/headroom.py index fb7b2019e..523bba662 100644 --- a/dokimi/dokimi/commands/headroom.py +++ b/dokimi/dokimi/commands/headroom.py @@ -64,7 +64,8 @@ def cmd_eval_headroom(args: argparse.Namespace) -> None: if args.output and _names_a_directory(args.output): # A directory has no file name for save_artifact's sibling paths ("." has an empty name), and # Path() drops a trailing slash, so "newdir/" would otherwise write a plain file named newdir. - print(f"Error: -o {args.output} names a directory; it needs a file name", file=sys.stderr) + print(f"Error: -o {args.output} is not a file name (a directory, or a path ending in a separator)", + file=sys.stderr) sys.exit(2) if args.output and _clobbers_an_input(args.output, args.arms): print(f"Error: -o {args.output} would overwrite one of the input arms files (or its .rejected sibling); " diff --git a/dokimi/tests/test_headroom.py b/dokimi/tests/test_headroom.py index 680d39562..50547ac1d 100644 --- a/dokimi/tests/test_headroom.py +++ b/dokimi/tests/test_headroom.py @@ -416,22 +416,26 @@ def test_command_exits_2_when_the_output_cannot_be_written(tmp_path, capsys): assert capsys.readouterr().err.startswith("Error: ") -@pytest.mark.parametrize("spelling", ["dir", "dot", "dot-slash", "absent-with-slash", "absent-with-slash-dot"]) +@pytest.mark.parametrize("spelling", ["dir", "dot", "dot-slash", "absent-with-slash", "absent-with-slash-dot", + "file-with-slash"]) def test_command_exits_2_when_the_output_is_a_directory(tmp_path, monkeypatch, capsys, spelling): # A directory has no file name to write: the save would raise ValueError (for "." and "./"), fail # on the directory itself, or (for a directory that does not exist yet, spelled with a slash or a # trailing "/.") write a plain file under the directory's name. Refuse all of them up front. path = _arms_file(tmp_path / "arms.json", {"1": {"baseline": 95, "skill": 95}}) (tmp_path / "out").mkdir() + (tmp_path / "existing.json").write_text("keep", encoding="utf-8") monkeypatch.chdir(tmp_path) out = {"dir": str(tmp_path / "out"), "dot": ".", "dot-slash": "./", "absent-with-slash": str(tmp_path / "newdir") + "/", - "absent-with-slash-dot": str(tmp_path / "newdir") + "/."}[spelling] + "absent-with-slash-dot": str(tmp_path / "newdir") + "/.", + "file-with-slash": str(tmp_path / "existing.json") + "/"}[spelling] with pytest.raises(SystemExit) as exc: cmd_eval_headroom(_args([path], output=out)) assert exc.value.code == 2 - assert "names a directory" in capsys.readouterr().err + assert "is not a file name" in capsys.readouterr().err assert not (tmp_path / "newdir").exists() + assert (tmp_path / "existing.json").read_text(encoding="utf-8") == "keep" def test_an_output_whose_directory_check_cannot_stat_does_not_crash_the_guard(tmp_path, monkeypatch, capsys): @@ -454,7 +458,7 @@ def is_dir(self): code = 0 except SystemExit as exc: code = exc.code - assert code in (0, 2) + assert code == 0 and out.exists() # the simulated stat error is not a refusal: the save writes def test_command_refuses_an_output_spelled_through_a_dotdot_to_an_input(tmp_path, capsys): From 8118952a2c83838c20186aa4ac449411f2a089ea Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 30 Oct 2026 14:49:45 -0700 Subject: [PATCH 39/47] docs(headroom): the provisional rule counts reps across files; name the -o refusals "Every item is provisional" held for one file only: reps add up across the files of one run, so three files that score an item give reps=3 and it is not provisional. The exit-2 list also names the path-with-separator refusal. Signed-off-by: Nikolay Petrov --- dokimi/docs/evaluating-treatments.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dokimi/docs/evaluating-treatments.md b/dokimi/docs/evaluating-treatments.md index 6f83507c3..e20ceddf9 100644 --- a/dokimi/docs/evaluating-treatments.md +++ b/dokimi/docs/evaluating-treatments.md @@ -743,7 +743,8 @@ python cli.py eval headroom run1/arms.json run2/arms.json run3/arms.json \ is not a floor. - **provisional**: fewer than `--min-reps` runs scored the item with its no-skill arm (the reps are those runs). Default 3. A file gives each item zero or one rep (zero when its - no-skill arm was not scored in that file), so with the default of 3 every item is provisional. + no-skill arm was not scored in that file), so one file alone makes every item provisional at the + default of 3; the reps of several files add up. A run is one file. A file repeated on the command line, or holding the same JSON content as an earlier one (whitespace and key order do not count), is skipped with a note, so it does @@ -759,7 +760,7 @@ declare the same baseline; the default is that baseline. With `--no-skill-arm`, be one the runs list (exit 2 otherwise, naming the arms that exist), and the declared baselines are not checked. A threshold out of range exits 2: `--ceiling` in [0.01, 1], `--floor` 0 or in [0.01, ceiling), `--min-reps` at least 1. It also exits 2 when no run scored the -no-skill arm, or when `-o` is one of the input files (or its `.rejected` sibling) or names a directory. Scores are read as the 0-100 +no-skill arm, or when `-o` is one of the input files (or its `.rejected` sibling), or names a directory or a path ending in a separator. Scores are read as the 0-100 aggregate divided by 100 and are not range-checked. Only arms runs are read. The flags are descriptive: they do not change any score or any gate. From 6eea8e22f5e1d1fc77369b805a283f9b25daa62f Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 30 Oct 2026 15:13:20 -0700 Subject: [PATCH 40/47] style(test): keep the directory-output test comment to two lines Signed-off-by: Nikolay Petrov --- dokimi/tests/test_headroom.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dokimi/tests/test_headroom.py b/dokimi/tests/test_headroom.py index 50547ac1d..5d3988cab 100644 --- a/dokimi/tests/test_headroom.py +++ b/dokimi/tests/test_headroom.py @@ -419,9 +419,8 @@ def test_command_exits_2_when_the_output_cannot_be_written(tmp_path, capsys): @pytest.mark.parametrize("spelling", ["dir", "dot", "dot-slash", "absent-with-slash", "absent-with-slash-dot", "file-with-slash"]) def test_command_exits_2_when_the_output_is_a_directory(tmp_path, monkeypatch, capsys, spelling): - # A directory has no file name to write: the save would raise ValueError (for "." and "./"), fail - # on the directory itself, or (for a directory that does not exist yet, spelled with a slash or a - # trailing "/.") write a plain file under the directory's name. Refuse all of them up front. + # A directory output would raise ValueError ("."), fail on the directory, or (spelled with a + # slash, not existing yet) write a plain file under the directory's name: refuse all up front. path = _arms_file(tmp_path / "arms.json", {"1": {"baseline": 95, "skill": 95}}) (tmp_path / "out").mkdir() (tmp_path / "existing.json").write_text("keep", encoding="utf-8") From 6399536944900d3cd8a6d8c271d8979e2ac088de Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 30 Oct 2026 17:51:40 -0700 Subject: [PATCH 41/47] feat(eval): eval headroom takes the registry's flag names and a route row --output/-o becomes --out (the primary artifact is always a file; -o and --output are not canonical), and --no-skill-arm becomes --baseline-arm (the baseline concept's spelling). The route table lists eval headroom as free: it reads arms files and makes no model call. Snapshot regenerated. Signed-off-by: Nikolay Petrov --- ...1028_120000_napetrov_feat-eval-headroom.md | 2 +- dokimi/docs/evaluating-treatments.md | 10 +++--- dokimi/docs/reference/cli-interface.json | 34 +++++++++---------- dokimi/dokimi/cli_routes.py | 1 + dokimi/dokimi/commands/headroom.py | 12 +++---- dokimi/dokimi/eval/headroom.py | 4 +-- dokimi/tests/data/cli_flags_allowlist.txt | 1 - dokimi/tests/test_headroom.py | 6 ++-- 8 files changed, 35 insertions(+), 35 deletions(-) diff --git a/dokimi/changelog.d/20261028_120000_napetrov_feat-eval-headroom.md b/dokimi/changelog.d/20261028_120000_napetrov_feat-eval-headroom.md index 9abe75096..bb9cdb5e4 100644 --- a/dokimi/changelog.d/20261028_120000_napetrov_feat-eval-headroom.md +++ b/dokimi/changelog.d/20261028_120000_napetrov_feat-eval-headroom.md @@ -1,3 +1,3 @@ ### Added -- **`eval headroom` flags arms items with no headroom** — reads one or more `arms.v1` files of the same question set and flags an item as a `ceiling` when the no-skill arm's mean is at least `--ceiling` (default 0.9, on 0-1), or as a `floor` when the no-skill arm was scored, every arm that a run scored has a mean at most `--floor` (default 0), and at least one arm other than the no-skill arm scored. An item with fewer than `--min-reps` runs (default 3) behind it is `provisional`. The human table goes to stdout; `-o` writes a `headroom.v1` artifact with every item. The flags change no score. +- **`eval headroom` flags arms items with no headroom** — reads one or more `arms.v1` files of the same question set and flags an item as a `ceiling` when the no-skill arm's mean is at least `--ceiling` (default 0.9, on 0-1), or as a `floor` when the no-skill arm was scored, every arm that a run scored has a mean at most `--floor` (default 0), and at least one arm other than the no-skill arm scored. An item with fewer than `--min-reps` runs (default 3) behind it is `provisional`. The human table goes to stdout; `--out` writes a `headroom.v1` artifact with every item. The flags change no score. diff --git a/dokimi/docs/evaluating-treatments.md b/dokimi/docs/evaluating-treatments.md index 664834a41..e3c25146e 100644 --- a/dokimi/docs/evaluating-treatments.md +++ b/dokimi/docs/evaluating-treatments.md @@ -732,7 +732,7 @@ both, from one or more arms files of the same question set (one file per repetit ```bash python cli.py eval headroom run1/arms.json run2/arms.json run3/arms.json \ - --no-skill-arm baseline --ceiling 0.9 --floor 0 --min-reps 3 -o headroom.json + --baseline-arm baseline --ceiling 0.9 --floor 0 --min-reps 3 --out headroom.json ``` - **ceiling**: the no-skill arm's mean over the runs is at least `--ceiling` (0-1; the @@ -753,14 +753,14 @@ file is an error (exit 2): it would count as two runs. The table always goes to stdout: the flagged and provisional items, with the no-skill arm in its own column, a note naming any arm no run scored, and the counts. The JSON goes to -`-o` only, as a `headroom.v1` artifact with every item; if that file cannot be written the +`--out` only, as a `headroom.v1` artifact with every item; if that file cannot be written the command prints `Error: ...` on stderr and exits 2, after the table. Scores are compared unrounded -and shown rounded down, so 0.89996 prints as 0.89. Without `--no-skill-arm`, the runs must -declare the same baseline; the default is that baseline. With `--no-skill-arm`, the arm must +and shown rounded down, so 0.89996 prints as 0.89. Without `--baseline-arm`, the runs must +declare the same baseline; the default is that baseline. With `--baseline-arm`, the arm must be one the runs list (exit 2 otherwise, naming the arms that exist), and the declared baselines are not checked. A threshold out of range exits 2: `--ceiling` in [0.01, 1], `--floor` 0 or in [0.01, ceiling), `--min-reps` at least 1. It also exits 2 when no run scored the -no-skill arm, or when `-o` is one of the input files (or its `.rejected` sibling), or names a directory or a path ending in a separator. Scores are read as the 0-100 +no-skill arm, or when `--out` is one of the input files (or its `.rejected` sibling), or names a directory or a path ending in a separator. Scores are read as the 0-100 aggregate divided by 100 and are not range-checked. Only arms runs are read. The flags are descriptive: they do not change any score or any gate. diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index d9cfeb6d7..cce58b40d 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -4336,6 +4336,22 @@ "required": true, "type": null }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--baseline-arm", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, { "action": "_StoreAction", "choices": null, @@ -4409,23 +4425,7 @@ "group_required": null, "hidden": false, "is_flag": false, - "key": "--no-skill-arm", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--output/-o", + "key": "--out", "kind": "option", "nargs": null, "position": null, diff --git a/dokimi/dokimi/cli_routes.py b/dokimi/dokimi/cli_routes.py index d6ae4daf2..8926a961d 100644 --- a/dokimi/dokimi/cli_routes.py +++ b/dokimi/dokimi/cli_routes.py @@ -147,6 +147,7 @@ class Tombstone: Leaf(("eval", "pairwise"), PAID), Leaf(("eval", "reference"), PAID), Leaf(("eval", "native-import"), FREE), + Leaf(("eval", "headroom"), FREE), Leaf(("eval", "panel-score"), PAID), Leaf(("dataset", "export"), FREE), Leaf(("ingest", "docling"), FREE), diff --git a/dokimi/dokimi/commands/headroom.py b/dokimi/dokimi/commands/headroom.py index 523bba662..7a37777cc 100644 --- a/dokimi/dokimi/commands/headroom.py +++ b/dokimi/dokimi/commands/headroom.py @@ -4,7 +4,7 @@ """``eval headroom``: flag the ceiling, floor and provisional items of arms runs. Reads one or more ``arms.v1`` files of the same question set (one per repetition) and writes a -``headroom.v1`` artifact with ``-o``, always printing the human table. A file repeated on the +``headroom.v1`` artifact with ``--out``, always printing the human table. A file repeated on the command line, or holding the same JSON content as an earlier one, is one run: the repeats are skipped with a note on stderr. See :mod:`dokimi.eval.headroom` for the rules. """ @@ -37,7 +37,7 @@ def _same_file(a, b) -> bool: def _names_a_directory(output) -> bool: - """Whether ``-o`` spells a directory: its last component is empty, ``.`` or ``..`` (a trailing + """Whether ``--out`` spells a directory: its last component is empty, ``.`` or ``..`` (a trailing separator, with or without ``/.``), or it is an existing directory. A path that cannot be stat-ed is not one: the save reports it, as before.""" text = str(output) @@ -64,11 +64,11 @@ def cmd_eval_headroom(args: argparse.Namespace) -> None: if args.output and _names_a_directory(args.output): # A directory has no file name for save_artifact's sibling paths ("." has an empty name), and # Path() drops a trailing slash, so "newdir/" would otherwise write a plain file named newdir. - print(f"Error: -o {args.output} is not a file name (a directory, or a path ending in a separator)", + print(f"Error: --out {args.output} is not a file name (a directory, or a path ending in a separator)", file=sys.stderr) sys.exit(2) if args.output and _clobbers_an_input(args.output, args.arms): - print(f"Error: -o {args.output} would overwrite one of the input arms files (or its .rejected sibling); " + print(f"Error: --out {args.output} would overwrite one of the input arms files (or its .rejected sibling); " "the run would be lost", file=sys.stderr) sys.exit(2) runs: list[records.Run] = [] @@ -116,7 +116,7 @@ def register_eval(eval_sub, positive_int) -> None: "headroom", help="Flag arms items with no headroom: no-skill ceiling, all-zero floor, provisional reps") h.add_argument("arms", nargs="+", help="arms.v1 JSON, one per repetition of the question set") - h.add_argument("--no-skill-arm", default=None, + h.add_argument("--baseline-arm", default=None, dest="no_skill_arm", help="Arm read as the no-skill control (default: the baseline the runs declare)") h.add_argument("--ceiling", type=float, default=DEFAULT_CEILING, help=f"Flag an item when the no-skill arm scores at least this, 0-1 (default {DEFAULT_CEILING})") @@ -124,5 +124,5 @@ def register_eval(eval_sub, positive_int) -> None: help=f"Flag an item when every arm scores at most this, 0-1 (default {DEFAULT_FLOOR})") h.add_argument("--min-reps", type=positive_int, default=DEFAULT_MIN_REPS, dest="min_reps", help=f"Items with fewer runs than this are provisional (default {DEFAULT_MIN_REPS})") - h.add_argument("-o", "--output", default=None, help="headroom.v1 JSON to write") + h.add_argument("--out", dest="output", default=None, help="headroom.v1 JSON to write") h.set_defaults(func=cmd_eval_headroom) diff --git a/dokimi/dokimi/eval/headroom.py b/dokimi/dokimi/eval/headroom.py index 43dd018fb..f97b50e01 100644 --- a/dokimi/dokimi/eval/headroom.py +++ b/dokimi/dokimi/eval/headroom.py @@ -122,7 +122,7 @@ def headroom( # Every item would read as "no ceiling, not a floor": unmeasured, not headroom. Refuse it instead. raise ValueError( f"no run scored the no-skill arm {control!r}, so no item can be read for headroom; " - "the runs were not judged, or the baseline is named differently (pass --no-skill-arm)") + "the runs were not judged, or the baseline is named differently (pass --baseline-arm)") items: List[ItemHeadroom] = [] for qid in sorted(scores, key=_sort_key): @@ -173,7 +173,7 @@ def _no_skill_arm(runs: Sequence[Run], given: Optional[str]) -> str: declared = {run.baseline for run in runs} only = next(iter(declared)) if len(declared) == 1 else None if only is None: - raise ValueError("the runs do not declare one baseline arm; pass --no-skill-arm") + raise ValueError("the runs do not declare one baseline arm; pass --baseline-arm") return only diff --git a/dokimi/tests/data/cli_flags_allowlist.txt b/dokimi/tests/data/cli_flags_allowlist.txt index 072870d9f..b4cad8f12 100644 --- a/dokimi/tests/data/cli_flags_allowlist.txt +++ b/dokimi/tests/data/cli_flags_allowlist.txt @@ -224,4 +224,3 @@ --ceiling # eval headroom (#279) adds it: no registry concept yet; a F1b migration names it --floor # eval headroom (#279) adds it: no registry concept yet; a F1b migration names it --min-reps # eval headroom (#279) adds it: no registry concept yet; a F1b migration names it ---no-skill-arm # eval headroom (#279) adds it: no registry concept yet; a F1b migration names it diff --git a/dokimi/tests/test_headroom.py b/dokimi/tests/test_headroom.py index 5d3988cab..ed1afb729 100644 --- a/dokimi/tests/test_headroom.py +++ b/dokimi/tests/test_headroom.py @@ -302,7 +302,7 @@ def test_binary_answers_are_refused(): def test_runs_that_declare_different_baselines_need_an_explicit_arm(): a = _run([("1", {"baseline": 50, "skill": 50})], baseline="baseline") b = _run([("1", {"control": 50, "skill": 50})], baseline="control") - with pytest.raises(ValueError, match="--no-skill-arm"): + with pytest.raises(ValueError, match="--baseline-arm"): headroom([a, b]) result = headroom([a, b], no_skill_arm="baseline") assert result["no_skill_arm"] == "baseline" and result["runs"] == 2 @@ -508,7 +508,7 @@ def test_the_registered_options_reach_the_analysis_through_the_real_parser(tmp_p "2": {"baseline": 5, "skill": 15}}) out = tmp_path / "headroom.json" args = build_parser().parse_args(["eval", "headroom", str(path), "--ceiling", "0.5", "--floor", "0.2", - "--min-reps", "1", "--no-skill-arm", "baseline", "-o", str(out)]) + "--min-reps", "1", "--baseline-arm", "baseline", "--out", str(out)]) args.func(args) saved = json.loads(out.read_text(encoding="utf-8")) assert saved["thresholds"] == {"ceiling": 0.5, "floor": 0.2, "min_reps": 1} @@ -542,7 +542,7 @@ def test_command_refuses_a_wrong_baseline_name_with_exit_2(tmp_path, capsys): with pytest.raises(SystemExit) as exc: cmd_eval_headroom(_args([path])) assert exc.value.code == 2 - assert "pass --no-skill-arm" in capsys.readouterr().err + assert "pass --baseline-arm" in capsys.readouterr().err def test_command_exits_2_on_an_input_nested_too_deep_to_read(tmp_path, capsys): From 58891e34f3b3e3e0eacf2f367ede48a58a73642f Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 30 Oct 2026 17:52:14 -0700 Subject: [PATCH 42/47] fix(eval): an empty --out is refused, not skipped The destination checks tested the value's truth, so "" skipped the directory and clobber checks and the save, and the command exited 0 without the artifact. They now test for None; "" is refused with exit 2. Address review: refuse an empty output through the real parser Thread: PRRT_kwDOVBg7g86yKCDW Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/headroom.py | 6 +++--- dokimi/tests/test_headroom.py | 9 +++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/dokimi/dokimi/commands/headroom.py b/dokimi/dokimi/commands/headroom.py index 7a37777cc..c6f6a53bb 100644 --- a/dokimi/dokimi/commands/headroom.py +++ b/dokimi/dokimi/commands/headroom.py @@ -61,13 +61,13 @@ def cmd_eval_headroom(args: argparse.Namespace) -> None: from dokimi.artifacts import ArtifactValidationError, save_artifact from dokimi.eval.headroom import format_table, headroom, repeated_ids - if args.output and _names_a_directory(args.output): + if args.output is not None and _names_a_directory(args.output): # A directory has no file name for save_artifact's sibling paths ("." has an empty name), and # Path() drops a trailing slash, so "newdir/" would otherwise write a plain file named newdir. print(f"Error: --out {args.output} is not a file name (a directory, or a path ending in a separator)", file=sys.stderr) sys.exit(2) - if args.output and _clobbers_an_input(args.output, args.arms): + if args.output is not None and _clobbers_an_input(args.output, args.arms): print(f"Error: --out {args.output} would overwrite one of the input arms files (or its .rejected sibling); " "the run would be lost", file=sys.stderr) sys.exit(2) @@ -100,7 +100,7 @@ def cmd_eval_headroom(args: argparse.Namespace) -> None: sys.exit(2) result["inputs"] = inputs print(format_table(result)) - if args.output: + if args.output is not None: try: save_artifact("headroom", result, args.output) except (OSError, ArtifactValidationError) as e: diff --git a/dokimi/tests/test_headroom.py b/dokimi/tests/test_headroom.py index ed1afb729..e44d4d2b2 100644 --- a/dokimi/tests/test_headroom.py +++ b/dokimi/tests/test_headroom.py @@ -639,3 +639,12 @@ def test_floor_is_not_flagged_when_the_no_skill_arm_was_never_scored_on_the_item # Item 1's baseline judge failed; the skill at 0 alone is not a floor. Item 2 is one. run = _run([("1", {"baseline": None, "skill": 0}), ("2", {"baseline": 0, "skill": 0})]) assert _flags(headroom([run], min_reps=1)) == {"1": None, "2": FLOOR} + + +def test_an_empty_out_is_refused_with_exit_2_through_the_real_parser(tmp_path, capsys): + path = _arms_file(tmp_path / "arms.json", {"1": {"baseline": 95, "skill": 95}}) + args = build_parser().parse_args(["eval", "headroom", str(path), "--out", ""]) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 + assert "not a file name" in capsys.readouterr().err From 156c82daa37434aba4befe63a287ea34d8f53672 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 30 Oct 2026 17:52:51 -0700 Subject: [PATCH 43/47] fix(eval): a failed write of --out exits 1, not 2 Exit 2 is for input and usage errors. A write that the OS refuses is a runtime failure, so eval headroom exits 1 for it, as the other commands do. Signed-off-by: Nikolay Petrov --- dokimi/docs/evaluating-treatments.md | 2 +- dokimi/dokimi/commands/headroom.py | 4 ++-- dokimi/tests/test_headroom.py | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dokimi/docs/evaluating-treatments.md b/dokimi/docs/evaluating-treatments.md index e3c25146e..70c65dc1b 100644 --- a/dokimi/docs/evaluating-treatments.md +++ b/dokimi/docs/evaluating-treatments.md @@ -754,7 +754,7 @@ file is an error (exit 2): it would count as two runs. The table always goes to stdout: the flagged and provisional items, with the no-skill arm in its own column, a note naming any arm no run scored, and the counts. The JSON goes to `--out` only, as a `headroom.v1` artifact with every item; if that file cannot be written the -command prints `Error: ...` on stderr and exits 2, after the table. Scores are compared unrounded +command prints `Error: ...` on stderr and exits 1 (a runtime failure), after the table. Scores are compared unrounded and shown rounded down, so 0.89996 prints as 0.89. Without `--baseline-arm`, the runs must declare the same baseline; the default is that baseline. With `--baseline-arm`, the arm must be one the runs list (exit 2 otherwise, naming the arms that exist), and the declared diff --git a/dokimi/dokimi/commands/headroom.py b/dokimi/dokimi/commands/headroom.py index c6f6a53bb..59175fd78 100644 --- a/dokimi/dokimi/commands/headroom.py +++ b/dokimi/dokimi/commands/headroom.py @@ -103,9 +103,9 @@ def cmd_eval_headroom(args: argparse.Namespace) -> None: if args.output is not None: try: save_artifact("headroom", result, args.output) - except (OSError, ArtifactValidationError) as e: + except (OSError, ArtifactValidationError) as e: # a failed write is runtime, not input: exit 1 print(f"Error: {e}", file=sys.stderr) - sys.exit(2) + sys.exit(1) print(f"Wrote headroom artifact to {args.output}") diff --git a/dokimi/tests/test_headroom.py b/dokimi/tests/test_headroom.py index e44d4d2b2..24c061267 100644 --- a/dokimi/tests/test_headroom.py +++ b/dokimi/tests/test_headroom.py @@ -406,13 +406,13 @@ def test_distinct_input_files_each_count_as_a_run(tmp_path): assert (saved["runs"], item["reps"], item["provisional"], item["flag"]) == (3, 3, False, CEILING) -def test_command_exits_2_when_the_output_cannot_be_written(tmp_path, capsys): +def test_command_exits_1_when_the_output_cannot_be_written(tmp_path, capsys): path = _arms_file(tmp_path / "arms.json", {"1": {"baseline": 95, "skill": 95}}) blocker = tmp_path / "a-file" - blocker.write_text("not a directory, so -o cannot be written beneath it", encoding="utf-8") + blocker.write_text("not a directory, so --out cannot be written beneath it", encoding="utf-8") with pytest.raises(SystemExit) as exc: cmd_eval_headroom(_args([path], output=blocker / "headroom.json")) - assert exc.value.code == 2 + assert exc.value.code == 1 # a failed write is a runtime failure, not an input error assert capsys.readouterr().err.startswith("Error: ") From 6c29745f3dfa697d42c07fda99e8eaaa179d21d8 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 30 Oct 2026 22:03:56 -0700 Subject: [PATCH 44/47] chore(cli): regenerate the CLI snapshot after merging main eval headroom is in the route table as a leaf, so the snapshot lists it. Signed-off-by: Nikolay Petrov --- dokimi/docs/reference/cli-interface.json | 1 + 1 file changed, 1 insertion(+) diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index ccda4bb65..e9afd0837 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -13007,6 +13007,7 @@ "'eval'", "'explore_runs'", "'guidance_reach'", + "'headroom'", "'matrix_rollup'", "'novelty'", "'plan'", From b971c619ea9fe728eacd111c6cdad8d8879ae05c Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 30 Oct 2026 22:17:15 -0700 Subject: [PATCH 45/47] fix(runs): list headroom.v1 in the runs kinds table The runs scan and show tests check every schema version has a row. headroom.v1 was added by this PR without one, so the Tests and Windows smoke jobs failed. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/runs/kinds.py | 1 + dokimi/tests/test_runs_scan.py | 1 + dokimi/tests/test_runs_show.py | 1 + 3 files changed, 3 insertions(+) diff --git a/dokimi/dokimi/runs/kinds.py b/dokimi/dokimi/runs/kinds.py index cb8e49a39..61e5a66f9 100644 --- a/dokimi/dokimi/runs/kinds.py +++ b/dokimi/dokimi/runs/kinds.py @@ -65,6 +65,7 @@ class KindFields: "repo_budget": KindFields(subject=("tree",), counts=("entries",)), "task_inspect": KindFields(subject=("root",), counts=("items",)), "cli_routes": KindFields(counts=("roots", "leaves", "tombstones")), + "headroom": KindFields(counts=("items", "inputs")), } #: Every schema_version an artifact may carry (current and superseded) -> its kind. diff --git a/dokimi/tests/test_runs_scan.py b/dokimi/tests/test_runs_scan.py index 9b5b9997d..c0dd16a45 100644 --- a/dokimi/tests/test_runs_scan.py +++ b/dokimi/tests/test_runs_scan.py @@ -48,6 +48,7 @@ "repo_budget.v1": ({}, "toy", None, None), "task_inspect.v1": ({}, "tests/fixtures/task_formats", None, None), "cli_routes.v1": ({}, None, None, None), + "headroom.v1": ({}, None, None, None), } diff --git a/dokimi/tests/test_runs_show.py b/dokimi/tests/test_runs_show.py index 07ad7a9f5..82023e0fa 100644 --- a/dokimi/tests/test_runs_show.py +++ b/dokimi/tests/test_runs_show.py @@ -37,6 +37,7 @@ "result.v1": {"cells": 6}, "subject_scorecard.v2": {"cells": 0}, "cli_routes.v1": {"roots": 2, "leaves": 2, "tombstones": 1}, + "headroom.v1": {"items": 2, "inputs": 1}, "task_inspect.v1": {"items": 4}, "task_runs.v1": {"tasks": 6, "harnesses": 2, "results": 24}, "tool_preference.v1": {"items": 16, "results": 16}, From 97f0f7a3f693cb5757c4b2d1271547e39e85021e Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Fri, 30 Oct 2026 22:14:33 -0700 Subject: [PATCH 46/47] fix(eval): exit 2 when eval headroom cannot write --out The exit-code table in docs/quickstart.md gives exit 2 to an output that cannot be written, and exit_on_unwritable implements it. Use it; a schema refusal of the payload stays exit 1. Signed-off-by: Alexander Rybkin --- dokimi/docs/evaluating-treatments.md | 2 +- dokimi/dokimi/commands/headroom.py | 6 ++++-- dokimi/tests/test_headroom.py | 6 +++--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/dokimi/docs/evaluating-treatments.md b/dokimi/docs/evaluating-treatments.md index 70c65dc1b..5d049069a 100644 --- a/dokimi/docs/evaluating-treatments.md +++ b/dokimi/docs/evaluating-treatments.md @@ -754,7 +754,7 @@ file is an error (exit 2): it would count as two runs. The table always goes to stdout: the flagged and provisional items, with the no-skill arm in its own column, a note naming any arm no run scored, and the counts. The JSON goes to `--out` only, as a `headroom.v1` artifact with every item; if that file cannot be written the -command prints `Error: ...` on stderr and exits 1 (a runtime failure), after the table. Scores are compared unrounded +command prints `Error: cannot write ...` on stderr and exits 2, after the table. Scores are compared unrounded and shown rounded down, so 0.89996 prints as 0.89. Without `--baseline-arm`, the runs must declare the same baseline; the default is that baseline. With `--baseline-arm`, the arm must be one the runs list (exit 2 otherwise, naming the arms that exist), and the declared diff --git a/dokimi/dokimi/commands/headroom.py b/dokimi/dokimi/commands/headroom.py index 59175fd78..0bcd1b326 100644 --- a/dokimi/dokimi/commands/headroom.py +++ b/dokimi/dokimi/commands/headroom.py @@ -19,6 +19,7 @@ from pathlib import Path from dokimi import records +from dokimi.commands._output import exit_on_unwritable def _content_digest(path) -> str: @@ -102,8 +103,9 @@ def cmd_eval_headroom(args: argparse.Namespace) -> None: print(format_table(result)) if args.output is not None: try: - save_artifact("headroom", result, args.output) - except (OSError, ArtifactValidationError) as e: # a failed write is runtime, not input: exit 1 + with exit_on_unwritable(args.output): # an unwritable output exits 2, as docs/quickstart.md says + save_artifact("headroom", result, args.output) + except ArtifactValidationError as e: # a payload the schema refuses is an internal failure: exit 1 print(f"Error: {e}", file=sys.stderr) sys.exit(1) print(f"Wrote headroom artifact to {args.output}") diff --git a/dokimi/tests/test_headroom.py b/dokimi/tests/test_headroom.py index 24c061267..52dae8e2c 100644 --- a/dokimi/tests/test_headroom.py +++ b/dokimi/tests/test_headroom.py @@ -406,14 +406,14 @@ def test_distinct_input_files_each_count_as_a_run(tmp_path): assert (saved["runs"], item["reps"], item["provisional"], item["flag"]) == (3, 3, False, CEILING) -def test_command_exits_1_when_the_output_cannot_be_written(tmp_path, capsys): +def test_command_exits_2_when_the_output_cannot_be_written(tmp_path, capsys): path = _arms_file(tmp_path / "arms.json", {"1": {"baseline": 95, "skill": 95}}) blocker = tmp_path / "a-file" blocker.write_text("not a directory, so --out cannot be written beneath it", encoding="utf-8") with pytest.raises(SystemExit) as exc: cmd_eval_headroom(_args([path], output=blocker / "headroom.json")) - assert exc.value.code == 1 # a failed write is a runtime failure, not an input error - assert capsys.readouterr().err.startswith("Error: ") + assert exc.value.code == 2 # the exit-code table in docs/quickstart.md: an unwritable output is 2 + assert capsys.readouterr().err.startswith(f"Error: cannot write {blocker / 'headroom.json'}: ") @pytest.mark.parametrize("spelling", ["dir", "dot", "dot-slash", "absent-with-slash", "absent-with-slash-dot", From 976f06a22f2298bd8be754d464e0873bea541d21 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Fri, 30 Oct 2026 22:15:01 -0700 Subject: [PATCH 47/47] fix(eval): refuse eval headroom inputs of different configurations Inputs count as repetitions, but runs of different models, judges, run configs or libraries were averaged per question id and lifted the provisional flag. Compare report.model_compare.replicate_key and the library, as model compare does, and exit 2 on a mismatch. Signed-off-by: Alexander Rybkin --- ...1028_120000_napetrov_feat-eval-headroom.md | 2 +- dokimi/docs/evaluating-treatments.md | 4 ++- dokimi/dokimi/commands/headroom.py | 15 ++++++++++ dokimi/tests/test_headroom.py | 30 +++++++++++++++++++ 4 files changed, 49 insertions(+), 2 deletions(-) diff --git a/dokimi/changelog.d/20261028_120000_napetrov_feat-eval-headroom.md b/dokimi/changelog.d/20261028_120000_napetrov_feat-eval-headroom.md index bb9cdb5e4..8a9c7edc0 100644 --- a/dokimi/changelog.d/20261028_120000_napetrov_feat-eval-headroom.md +++ b/dokimi/changelog.d/20261028_120000_napetrov_feat-eval-headroom.md @@ -1,3 +1,3 @@ ### Added -- **`eval headroom` flags arms items with no headroom** — reads one or more `arms.v1` files of the same question set and flags an item as a `ceiling` when the no-skill arm's mean is at least `--ceiling` (default 0.9, on 0-1), or as a `floor` when the no-skill arm was scored, every arm that a run scored has a mean at most `--floor` (default 0), and at least one arm other than the no-skill arm scored. An item with fewer than `--min-reps` runs (default 3) behind it is `provisional`. The human table goes to stdout; `--out` writes a `headroom.v1` artifact with every item. The flags change no score. +- **`eval headroom` flags arms items with no headroom** — reads one or more `arms.v1` files of the same question set and flags an item as a `ceiling` when the no-skill arm's mean is at least `--ceiling` (default 0.9, on 0-1), or as a `floor` when the no-skill arm was scored, every arm that a run scored has a mean at most `--floor` (default 0), and at least one arm other than the no-skill arm scored. An item with fewer than `--min-reps` runs (default 3) behind it is `provisional`. The human table goes to stdout; `--out` writes a `headroom.v1` artifact with every item. The flags change no score. Inputs that are not repetitions of one configuration (model, judge, run config other than the seed, harness, arms or library differ) exit 2. diff --git a/dokimi/docs/evaluating-treatments.md b/dokimi/docs/evaluating-treatments.md index 5d049069a..dfebee7c1 100644 --- a/dokimi/docs/evaluating-treatments.md +++ b/dokimi/docs/evaluating-treatments.md @@ -756,7 +756,9 @@ in its own column, a note naming any arm no run scored, and the counts. The JSON `--out` only, as a `headroom.v1` artifact with every item; if that file cannot be written the command prints `Error: cannot write ...` on stderr and exits 2, after the table. Scores are compared unrounded and shown rounded down, so 0.89996 prints as 0.89. Without `--baseline-arm`, the runs must -declare the same baseline; the default is that baseline. With `--baseline-arm`, the arm must +declare the same baseline; the default is that baseline. The runs must also be repetitions of one +configuration: a file whose model, judge, run config (the seed aside), harness, arms or library +differs from the first file's exits 2. With `--baseline-arm`, the arm must be one the runs list (exit 2 otherwise, naming the arms that exist), and the declared baselines are not checked. A threshold out of range exits 2: `--ceiling` in [0.01, 1], `--floor` 0 or in [0.01, ceiling), `--min-reps` at least 1. It also exits 2 when no run scored the diff --git a/dokimi/dokimi/commands/headroom.py b/dokimi/dokimi/commands/headroom.py index 0bcd1b326..2e7e758c9 100644 --- a/dokimi/dokimi/commands/headroom.py +++ b/dokimi/dokimi/commands/headroom.py @@ -58,6 +58,20 @@ def _clobbers_an_input(output, inputs) -> bool: return any(_same_file(out, p) or _same_file(rejected, p) for p in inputs) +def _exit_unless_replicates(runs, inputs) -> None: + """Exit 2 on the first input that is not a repetition of the first one's configuration (models, + judge, run_config but the seed, harness, plugins, cell, arms: report.model_compare.replicate_key) + or library: averaging such runs per question id would mix different measurements.""" + from dokimi.report.model_compare import replicate_key + + keys = [(replicate_key(dict(run.meta)), run.meta.get("library_name")) for run in runs] + odd = next((path for key, path in zip(keys[1:], inputs[1:], strict=True) if key != keys[0]), None) + if odd is not None: + print(f"Error: {odd} is not a repetition of {inputs[0]}: the model, judge, run config, harness, " + "arms or library differ, and runs of different configurations are not pooled", file=sys.stderr) + sys.exit(2) + + def cmd_eval_headroom(args: argparse.Namespace) -> None: from dokimi.artifacts import ArtifactValidationError, save_artifact from dokimi.eval.headroom import format_table, headroom, repeated_ids @@ -93,6 +107,7 @@ def cmd_eval_headroom(args: argparse.Namespace) -> None: first_seen[digest] = str(path) runs.append(run) inputs.append(str(path)) + _exit_unless_replicates(runs, inputs) try: result = headroom(runs, no_skill_arm=args.no_skill_arm, ceiling=args.ceiling, floor=args.floor, min_reps=args.min_reps) diff --git a/dokimi/tests/test_headroom.py b/dokimi/tests/test_headroom.py index 52dae8e2c..732f2c50f 100644 --- a/dokimi/tests/test_headroom.py +++ b/dokimi/tests/test_headroom.py @@ -406,6 +406,36 @@ def test_distinct_input_files_each_count_as_a_run(tmp_path): assert (saved["runs"], item["reps"], item["provisional"], item["flag"]) == (3, 3, False, CEILING) +@pytest.mark.parametrize("field, value", [("model", "other-model"), ("judge_model", "other-judge"), + ("library_name", "other-library"), ("run_config", {"max_tokens": 9})]) +def test_command_refuses_runs_of_different_configurations(tmp_path, capsys, field, value): + one = _arms_file(tmp_path / "run1.json", {"1": {"baseline": 95, "skill": 95}}) + two = _arms_file(tmp_path / "run2.json", {"1": {"baseline": 94, "skill": 95}}) + for path, extra in ((one, {}), (two, {field: value})): + data = json.loads(path.read_text(encoding="utf-8")) + path.write_text(json.dumps({**data, "model": "m", "judge_model": "j", "library_name": "lib", + "run_config": {"max_tokens": 5, "seed": 1}, **extra}), encoding="utf-8") + out = tmp_path / "headroom.json" + with pytest.raises(SystemExit) as exc: + cmd_eval_headroom(_args([one, two], output=out)) + assert exc.value.code == 2 and not out.exists() + assert f"Error: {two} is not a repetition of {one}" in capsys.readouterr().err + + +def test_runs_that_differ_only_in_the_seed_are_repetitions(tmp_path): + paths = [] + for seed in (1, 2): + path = _arms_file(tmp_path / f"run{seed}.json", {"1": {"baseline": 95, "skill": 95 - seed}}) + data = json.loads(path.read_text(encoding="utf-8")) + path.write_text(json.dumps({**data, "model": "m", "run_config": {"max_tokens": 5, "seed": seed}}), + encoding="utf-8") + paths.append(path) + out = tmp_path / "headroom.json" + cmd_eval_headroom(_args(paths, output=out)) + saved = json.loads(out.read_text(encoding="utf-8")) + assert (saved["runs"], saved["items"][0]["reps"]) == (2, 2) + + def test_command_exits_2_when_the_output_cannot_be_written(tmp_path, capsys): path = _arms_file(tmp_path / "arms.json", {"1": {"baseline": 95, "skill": 95}}) blocker = tmp_path / "a-file"