diff --git a/dokimi/changelog.d/20261028_120000_napetrov_feat-eval-headroom.md b/dokimi/changelog.d/20261028_120000_napetrov_feat-eval-headroom.md new file mode 100644 index 000000000..8a9c7edc0 --- /dev/null +++ b/dokimi/changelog.d/20261028_120000_napetrov_feat-eval-headroom.md @@ -0,0 +1,3 @@ +### Added + +- **`eval headroom` flags arms items with no headroom** — reads one or more `arms.v1` files of the same question set and flags an item as a `ceiling` when the no-skill arm's mean is at least `--ceiling` (default 0.9, on 0-1), or as a `floor` when the no-skill arm was scored, every arm that a run scored has a mean at most `--floor` (default 0), and at least one arm other than the no-skill arm scored. An item with fewer than `--min-reps` runs (default 3) behind it is `provisional`. The human table goes to stdout; `--out` writes a `headroom.v1` artifact with every item. The flags change no score. Inputs that are not repetitions of one configuration (model, judge, run config other than the seed, harness, arms or library differ) exit 2. diff --git a/dokimi/docs/evaluating-treatments.md b/dokimi/docs/evaluating-treatments.md index 7219d8f44..dfebee7c1 100644 --- a/dokimi/docs/evaluating-treatments.md +++ b/dokimi/docs/evaluating-treatments.md @@ -724,6 +724,48 @@ Two gates are deliberately statistical rather than absolute: those questions. It pools questions; a mean of per-cell means differs slightly when the sets have different sizes — say which one a report uses. +## Headroom: items a comparison cannot learn from + +A skill cannot raise an item the no-skill arm already answers, and an item no arm +answers points at the question or its grader, not at the skill. `eval headroom` flags +both, from one or more arms files of the same question set (one file per repetition): + +```bash +python cli.py eval headroom run1/arms.json run2/arms.json run3/arms.json \ + --baseline-arm baseline --ceiling 0.9 --floor 0 --min-reps 3 --out headroom.json +``` + +- **ceiling**: the no-skill arm's mean over the runs is at least `--ceiling` (0-1; the + 0-100 aggregate divided by 100). Default 0.9. +- **floor**: the no-skill arm was scored, every arm that a run scored has a mean at most + `--floor`, and at least one arm other than the no-skill arm scored. Default 0. An arm no run scored is + left out of this test and listed in the item's `unscored_arms`; the no-skill arm alone at zero + is not a floor. +- **provisional**: fewer than `--min-reps` runs scored the item with its no-skill arm (the + reps are those runs). Default 3. A file gives each item zero or one rep (zero when its + no-skill arm was not scored in that file), so one file alone makes every item provisional at the + default of 3; the reps of several files add up. + +A run is one file. A file repeated on the command line, or holding the same JSON content as +an earlier one (whitespace and key order do not count), is skipped with a note, so it does +not add a rep. A question id repeated within one +file is an error (exit 2): it would count as two runs. + +The table always goes to stdout: the flagged and provisional items, with the no-skill arm +in its own column, a note naming any arm no run scored, and the counts. The JSON goes to +`--out` only, as a `headroom.v1` artifact with every item; if that file cannot be written the +command prints `Error: cannot write ...` on stderr and exits 2, after the table. Scores are compared unrounded +and shown rounded down, so 0.89996 prints as 0.89. Without `--baseline-arm`, the runs must +declare the same baseline; the default is that baseline. The runs must also be repetitions of one +configuration: a file whose model, judge, run config (the seed aside), harness, arms or library +differs from the first file's exits 2. With `--baseline-arm`, the arm must +be one the runs list (exit 2 otherwise, naming the arms that exist), and the declared +baselines are not checked. A threshold out of range exits 2: `--ceiling` in [0.01, 1], +`--floor` 0 or in [0.01, ceiling), `--min-reps` at least 1. It also exits 2 when no run scored the +no-skill arm, or when `--out` is one of the input files (or its `.rejected` sibling), or names a directory or a path ending in a separator. Scores are read as the 0-100 +aggregate divided by 100 and are not range-checked. Only arms runs are read. The flags are descriptive: they do not +change any score or any gate. + ## Scope and caveats - **Three levels of fidelity for skills/MCP.** (1) `skill:` / `mcp:` inject diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index c9eb437d1..e9afd0837 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -4369,6 +4369,126 @@ "subcommand_required": null, "subcommands": {} }, + "headroom": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "arms", + "kind": "argument", + "nargs": "+", + "position": 0, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--baseline-arm", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 0.9, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--ceiling", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "float" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 0.0, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--floor", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "float" + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 3, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--min-reps", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--out", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi eval headroom", + "subcommand_required": null, + "subcommands": {} + }, "judge-validity": { "allow_abbrev": true, "params": [ @@ -12887,6 +13007,7 @@ "'eval'", "'explore_runs'", "'guidance_reach'", + "'headroom'", "'matrix_rollup'", "'novelty'", "'plan'", diff --git a/dokimi/dokimi/artifacts.py b/dokimi/dokimi/artifacts.py index 0088ff5eb..36df23f1d 100644 --- a/dokimi/dokimi/artifacts.py +++ b/dokimi/dokimi/artifacts.py @@ -57,6 +57,7 @@ "result": "result.v1", "repo_budget": "repo_budget.v1", "task_inspect": "task_inspect.v1", + "headroom": "headroom.v1", "cli_routes": "cli_routes.v1", } diff --git a/dokimi/dokimi/cli_routes.py b/dokimi/dokimi/cli_routes.py index 47008bcc9..98f940245 100644 --- a/dokimi/dokimi/cli_routes.py +++ b/dokimi/dokimi/cli_routes.py @@ -149,6 +149,7 @@ class Tombstone: Leaf(("eval", "pairwise"), PAID), Leaf(("eval", "reference"), PAID), Leaf(("eval", "native-import"), FREE), + Leaf(("eval", "headroom"), FREE), Leaf(("eval", "panel-score"), PAID), Leaf(("dataset", "export"), FREE), Leaf(("ingest", "docling"), FREE), diff --git a/dokimi/dokimi/commands/evaluate.py b/dokimi/dokimi/commands/evaluate.py index 62c4897b2..8f5ca1bda 100644 --- a/dokimi/dokimi/commands/evaluate.py +++ b/dokimi/dokimi/commands/evaluate.py @@ -570,6 +570,10 @@ def register(sub, positive_int) -> None: answer_formats.register_eval(eval_sub, positive_int) # eval reference, eval native-import + from dokimi.commands import headroom + + headroom.register_eval(eval_sub, positive_int) # eval headroom + # eval panel-score panel_p = eval_sub.add_parser("panel-score", help="Score answers using a multi-judge panel") panel_p.add_argument("--answers", required=True, help="Path to answers JSON file") diff --git a/dokimi/dokimi/commands/headroom.py b/dokimi/dokimi/commands/headroom.py new file mode 100644 index 000000000..2e7e758c9 --- /dev/null +++ b/dokimi/dokimi/commands/headroom.py @@ -0,0 +1,145 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""``eval headroom``: flag the ceiling, floor and provisional items of arms runs. + +Reads one or more ``arms.v1`` files of the same question set (one per repetition) and writes a +``headroom.v1`` artifact with ``--out``, always printing the human table. A file repeated on the +command line, or holding the same JSON content as an earlier one, is one run: the repeats are skipped with a +note on stderr. See :mod:`dokimi.eval.headroom` for the rules. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import sys +from pathlib import Path + +from dokimi import records +from dokimi.commands._output import exit_on_unwritable + + +def _content_digest(path) -> str: + """Digest of the parsed JSON, so one run re-serialized (whitespace, key order) counts once.""" + data = json.loads(Path(path).read_text(encoding="utf-8")) # records.load has parsed it already + return hashlib.sha256(json.dumps(data, sort_keys=True).encode("utf-8")).hexdigest() + + +def _same_file(a, b) -> bool: + """Whether two paths name one file. A path that cannot be resolved (a symlink loop on some Pythons) + is not taken as an input: its own open reports the error.""" + try: + return Path(a).resolve() == Path(b).resolve() + except (OSError, RuntimeError): + return False + + +def _names_a_directory(output) -> bool: + """Whether ``--out`` spells a directory: its last component is empty, ``.`` or ``..`` (a trailing + separator, with or without ``/.``), or it is an existing directory. A path that cannot be + stat-ed is not one: the save reports it, as before.""" + text = str(output) + if text.replace(os.sep, "/").rsplit("/", 1)[-1] in ("", ".", ".."): + return True + try: + return Path(text).is_dir() + except (OSError, ValueError): + return False + + +def _clobbers_an_input(output, inputs) -> bool: + """Whether a save to ``output`` would overwrite an input: the file itself, or the ``.rejected`` + sibling save_artifact writes on a schema failure and deletes after a good save.""" + out = Path(output) + rejected = out.with_name(out.name + ".rejected") + return any(_same_file(out, p) or _same_file(rejected, p) for p in inputs) + + +def _exit_unless_replicates(runs, inputs) -> None: + """Exit 2 on the first input that is not a repetition of the first one's configuration (models, + judge, run_config but the seed, harness, plugins, cell, arms: report.model_compare.replicate_key) + or library: averaging such runs per question id would mix different measurements.""" + from dokimi.report.model_compare import replicate_key + + keys = [(replicate_key(dict(run.meta)), run.meta.get("library_name")) for run in runs] + odd = next((path for key, path in zip(keys[1:], inputs[1:], strict=True) if key != keys[0]), None) + if odd is not None: + print(f"Error: {odd} is not a repetition of {inputs[0]}: the model, judge, run config, harness, " + "arms or library differ, and runs of different configurations are not pooled", file=sys.stderr) + sys.exit(2) + + +def cmd_eval_headroom(args: argparse.Namespace) -> None: + from dokimi.artifacts import ArtifactValidationError, save_artifact + from dokimi.eval.headroom import format_table, headroom, repeated_ids + + if args.output is not None and _names_a_directory(args.output): + # A directory has no file name for save_artifact's sibling paths ("." has an empty name), and + # Path() drops a trailing slash, so "newdir/" would otherwise write a plain file named newdir. + print(f"Error: --out {args.output} is not a file name (a directory, or a path ending in a separator)", + file=sys.stderr) + sys.exit(2) + if args.output is not None and _clobbers_an_input(args.output, args.arms): + print(f"Error: --out {args.output} would overwrite one of the input arms files (or its .rejected sibling); " + "the run would be lost", file=sys.stderr) + sys.exit(2) + runs: list[records.Run] = [] + inputs: list[str] = [] + first_seen: dict[str, str] = {} + for path in args.arms: + try: + run = records.load(path) + digest = _content_digest(path) + except (records.RecordError, OSError, RecursionError) as e: # RecursionError: nesting too deep for json + print(f"Error: {e}", file=sys.stderr) + sys.exit(2) + if digest in first_seen: + print(f"note: skipped {path}: same content as {first_seen[digest]}, counted once", file=sys.stderr) + continue + dupes = repeated_ids(run) + if dupes: + print(f"Error: {path}: question_id {dupes[0]!r} appears more than once; " + "a file holds each question once", file=sys.stderr) + sys.exit(2) + first_seen[digest] = str(path) + runs.append(run) + inputs.append(str(path)) + _exit_unless_replicates(runs, inputs) + try: + result = headroom(runs, no_skill_arm=args.no_skill_arm, ceiling=args.ceiling, + floor=args.floor, min_reps=args.min_reps) + except ValueError as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(2) + result["inputs"] = inputs + print(format_table(result)) + if args.output is not None: + try: + with exit_on_unwritable(args.output): # an unwritable output exits 2, as docs/quickstart.md says + save_artifact("headroom", result, args.output) + except ArtifactValidationError as e: # a payload the schema refuses is an internal failure: exit 1 + print(f"Error: {e}", file=sys.stderr) + sys.exit(1) + print(f"Wrote headroom artifact to {args.output}") + + +def register_eval(eval_sub, positive_int) -> None: + from dokimi.eval.headroom import DEFAULT_CEILING, DEFAULT_FLOOR, DEFAULT_MIN_REPS + + h = eval_sub.add_parser( + "headroom", + help="Flag arms items with no headroom: no-skill ceiling, all-zero floor, provisional reps") + h.add_argument("arms", nargs="+", help="arms.v1 JSON, one per repetition of the question set") + h.add_argument("--baseline-arm", default=None, dest="no_skill_arm", + help="Arm read as the no-skill control (default: the baseline the runs declare)") + h.add_argument("--ceiling", type=float, default=DEFAULT_CEILING, + help=f"Flag an item when the no-skill arm scores at least this, 0-1 (default {DEFAULT_CEILING})") + h.add_argument("--floor", type=float, default=DEFAULT_FLOOR, + help=f"Flag an item when every arm scores at most this, 0-1 (default {DEFAULT_FLOOR})") + h.add_argument("--min-reps", type=positive_int, default=DEFAULT_MIN_REPS, dest="min_reps", + help=f"Items with fewer runs than this are provisional (default {DEFAULT_MIN_REPS})") + h.add_argument("--out", dest="output", default=None, help="headroom.v1 JSON to write") + h.set_defaults(func=cmd_eval_headroom) diff --git a/dokimi/dokimi/eval/headroom.py b/dokimi/dokimi/eval/headroom.py new file mode 100644 index 000000000..f97b50e01 --- /dev/null +++ b/dokimi/dokimi/eval/headroom.py @@ -0,0 +1,283 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Headroom: the items of an arms comparison that cannot tell a skill apart. + +An item is a **ceiling** when its no-skill arm already scores at least ``ceiling`` (0-1: the +judge's or the grader's 0-100 aggregate divided by 100). A skill cannot raise it, so the item +measures nothing about the skill. An item is a **floor** when every arm scores at most +``floor`` (default 0): no arm answers it, which points at the question, its checks or the +grader rather than at any skill. + +An item's floor test leaves out an arm no run scored (listed in ``unscored_arms``). It needs the +no-skill arm scored on the item and at least one arm other than the no-skill arm that did score: +the no-skill arm alone at zero shows nothing about a skill, and an unscored no-skill arm shows +nothing either. + +An item with fewer than ``min_reps`` runs behind it is ``provisional``: one run's flag may be +noise. Its reps are the number of runs in which the no-skill arm scored it, so a single artifact +gives every item one rep. Each arm's score is the mean over the runs that scored it. + +A run is one ``arms.v1`` file (or any artifact :mod:`dokimi.records` reads as an arms run), one +per repetition of the same question set. Each ``Run`` passed here counts as one run, so callers +drop repeated inputs first (the CLI does, by content digest). A question id repeated within one +run is refused: it would count as two runs. Nothing here draws a confidence interval; the paired +statistics of a comparison stay in :mod:`dokimi.eval.stats`. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Sequence, Set, Tuple + +from dokimi.records import Record, Run + +DEFAULT_CEILING = 0.9 +DEFAULT_FLOOR = 0.0 +DEFAULT_MIN_REPS = 3 + +_EPS = 1e-9 # float error bound for a threshold comparison on a mean of 0-1 scores +# One score step (scores are read to 0.01): a positive threshold below it cannot tell two scores apart. +_MIN_THRESHOLD = 0.01 + +CEILING = "ceiling" +FLOOR = "floor" + + +@dataclass(frozen=True) +class ItemHeadroom: + question_id: str + reps: int + provisional: bool + no_skill: Optional[float] + arms: Dict[str, Optional[float]] + unscored_arms: List[str] + flag: Optional[str] + + def as_dict(self) -> Dict[str, Any]: + return { + "question_id": self.question_id, + "reps": self.reps, + "provisional": self.provisional, + "no_skill": self.no_skill, + "arms": dict(self.arms), + "unscored_arms": list(self.unscored_arms), + "flag": self.flag, + } + + +def check_thresholds(ceiling: float, floor: float, min_reps: int) -> None: + """Refuse thresholds that cannot flag anything sensibly (ValueError).""" + if not 0.0 < ceiling <= 1.0: + raise ValueError(f"ceiling must be in (0, 1], not {ceiling!r}") + if not 0.0 <= floor < ceiling: + raise ValueError(f"floor must be in [0, ceiling), not {floor!r} (ceiling {ceiling!r})") + if ceiling < _MIN_THRESHOLD: + raise ValueError(f"ceiling must be at least {_MIN_THRESHOLD}, not {ceiling!r}") + if 0.0 < floor < _MIN_THRESHOLD: + raise ValueError(f"floor must be 0 or at least {_MIN_THRESHOLD}, not {floor!r}") + if min_reps < 1: + raise ValueError(f"min_reps must be at least 1, not {min_reps!r}") + + +def _mean(values: List[float]) -> Optional[float]: + # Unrounded: the thresholds compare the exact mean (a 89.996 item is not a ceiling). + return sum(values) / len(values) if values else None + + +def _flag( + no_skill: Optional[float], arms: Dict[str, Optional[float]], control: str, ceiling: float, floor: float, +) -> Optional[str]: + # A mean exactly at the threshold can land a few ulps off it; the tolerance is far below one + # score step (0.01 at the finest), so it absorbs only that float error. + if no_skill is not None and no_skill >= ceiling - _EPS: + return CEILING + scored = {arm: v for arm, v in arms.items() if v is not None} + # An unscored no-skill arm cannot show the item is unanswerable: its judge may just have failed. + if no_skill is not None and any(arm != control for arm in scored) and all(v <= floor + _EPS for v in scored.values()): + return FLOOR + return None + + +def headroom( + runs: Sequence[Run], + *, + no_skill_arm: Optional[str] = None, + ceiling: float = DEFAULT_CEILING, + floor: float = DEFAULT_FLOOR, + min_reps: int = DEFAULT_MIN_REPS, +) -> Dict[str, Any]: + """Flag every item of ``runs`` (arms runs of one question set) by its headroom. + + ``no_skill_arm`` defaults to the baseline the runs declare, and must otherwise be an arm + the runs list. Raises ValueError for thresholds that cannot be used, a run that is not an + arms run, a question id repeated within a run, an unknown no-skill arm, no score at all for + the no-skill arm (when there are items), or runs that disagree on the baseline when none is given. + """ + check_thresholds(ceiling, floor, min_reps) + control = _no_skill_arm(runs, no_skill_arm) + scores, arms_of = _collect(runs) + if scores and not any(control in per_arm for per_arm in scores.values()): + # Every item would read as "no ceiling, not a floor": unmeasured, not headroom. Refuse it instead. + raise ValueError( + f"no run scored the no-skill arm {control!r}, so no item can be read for headroom; " + "the runs were not judged, or the baseline is named differently (pass --baseline-arm)") + + items: List[ItemHeadroom] = [] + for qid in sorted(scores, key=_sort_key): + per_arm = scores[qid] + arms = {arm: _mean(per_arm.get(arm, [])) for arm in sorted(arms_of[qid])} + reps = len(per_arm.get(control, [])) + no_skill = arms.get(control) + items.append(ItemHeadroom( + question_id=qid, + reps=reps, + provisional=reps < min_reps, + no_skill=no_skill, + arms=arms, + unscored_arms=[arm for arm, value in arms.items() if value is None], + flag=_flag(no_skill, arms, control, ceiling, floor), + )) + + summary = { + "items": len(items), + "ceiling": sum(1 for i in items if i.flag == CEILING), + "floor": sum(1 for i in items if i.flag == FLOOR), + "provisional": sum(1 for i in items if i.provisional), + "flagged_provisional": sum(1 for i in items if i.flag and i.provisional), + "no_flag": sum(1 for i in items if i.flag is None), + } + return { + "no_skill_arm": control, + "thresholds": {"ceiling": ceiling, "floor": floor, "min_reps": min_reps}, + "runs": len(runs), + "items": [i.as_dict() for i in items], + "summary": summary, + } + + +def _no_skill_arm(runs: Sequence[Run], given: Optional[str]) -> str: + """The control arm: ``given``, or the one baseline all runs declare. Refuses anything else.""" + if not runs: + raise ValueError("no runs to read") + for run in runs: + if run.shape != "arms": + raise ValueError("headroom reads arms runs only; a binary answers or eval file was given") + if given is not None: + known = _known_arms(runs) + if given not in known: + listed = ", ".join(sorted(known)) or "none" + raise ValueError(f"no-skill arm {given!r} is not an arm of these runs; the arms are: {listed}") + return given + declared = {run.baseline for run in runs} + only = next(iter(declared)) if len(declared) == 1 else None + if only is None: + raise ValueError("the runs do not declare one baseline arm; pass --baseline-arm") + return only + + +def _known_arms(runs: Sequence[Run]) -> Set[str]: + """Every arm any run declares or any record lists, ``null`` ones included.""" + known: Set[str] = set() + for run in runs: + known.update(run.conditions) + for rec in run: + known.update(_listed_arms(rec)) + return known + + +def _listed_arms(rec: Record) -> Set[str]: + """The arms an arms record lists, ``null`` ones included. Unlike ``arm_names()``, keeps + ``with_docs`` and ``without_docs``, which arms.v1 allows as arm names; a binary record lists none.""" + if rec.raw.get("arms") is None and rec.raw.get("scores") is None: + return set() + return set(rec.conditions) + + +def repeated_ids(run: Run) -> List[str]: + """The question ids that more than one record of ``run`` carries, in first-seen order.""" + seen: Set[str] = set() + repeated: List[str] = [] + for rec in run: + if rec.key in seen and rec.key not in repeated: + repeated.append(rec.key) + seen.add(rec.key) + return repeated + + +def _collect(runs: Sequence[Run]) -> Tuple[Dict[str, Dict[str, List[float]]], Dict[str, Set[str]]]: + """Question id -> arm -> the per-run scores (0-1), and question id -> the arms it listed.""" + scores: Dict[str, Dict[str, List[float]]] = {} + arms_of: Dict[str, Set[str]] = {} + for index, run in enumerate(runs, start=1): + dupes = repeated_ids(run) + if dupes: + raise ValueError( + f"run {index} lists question_id {dupes[0]!r} more than once; " + "a run holds each question once (a repeat would count as a second run)") + for rec in run: + names = _listed_arms(rec) | set(rec.scores) + arms_of.setdefault(rec.key, set()).update(names) + per_arm = scores.setdefault(rec.key, {}) + for arm in names: + value = rec.score(arm) + if value is not None: + per_arm.setdefault(arm, []).append(float(value) / 100.0) + return scores, arms_of + + +def _sort_key(qid: str): + # ASCII digits sort by value without int(), which refuses over 4300 digits: no leading zeros, + # then length, then digits. Any other id (isdigit() also takes superscripts) sorts after them, as text. + if qid.isascii() and qid.isdigit(): + digits = qid.lstrip("0") or "0" + return (0, len(digits), digits, qid) + return (1, 0, "", qid) + + +def format_table(result: Dict[str, Any]) -> str: + """The human-readable table: one row per flagged or provisional item, then the counts.""" + control = result["no_skill_arm"] + arm_names_ = sorted({a for item in result["items"] for a in item["arms"]} - {control}) + head = ["question", "reps", "no-skill", *arm_names_, "flag", "provisional"] + rows = [] + for item in result["items"]: + if item["flag"] is None and not item["provisional"]: + continue + cells = [ + str(item["question_id"]), str(item["reps"]), _fmt(item["no_skill"]), + *[_fmt(item["arms"].get(a)) for a in arm_names_], + item["flag"] or "-", "yes" if item["provisional"] else "no", + ] + rows.append(cells) + widths = [max(len(r[c]) for r in [head, *rows]) for c in range(len(head))] if rows else [len(h) for h in head] + lines = [_line(head, widths)] + [_line(r, widths) for r in rows] + if not rows: + lines.append("(no item is flagged or provisional)") + unscored: Dict[str, int] = {} + for item in result["items"]: + for arm in item["unscored_arms"]: + unscored[arm] = unscored.get(arm, 0) + 1 + if unscored: + counts = [f"{arm} in {n} item{'' if n == 1 else 's'}" for arm, n in sorted(unscored.items())] + lines.append("note: no run scored these arms, so the floor test leaves them out: " + "; ".join(counts)) + s = result["summary"] + th = result["thresholds"] + lines.append("") + lines.append( + f"{s['items']} items; no-skill arm {result['no_skill_arm']!r}; ceiling >= {th['ceiling']:g}, " + f"floor <= {th['floor']:g}, provisional below {th['min_reps']} reps") + lines.append( + f"ceiling {s['ceiling']}, floor {s['floor']}, provisional {s['provisional']} " + f"({s['flagged_provisional']} of them flagged), unflagged {s['no_flag']}") + return "\n".join(lines) + + +def _fmt(value: Optional[float]) -> str: + # Rounded down, with the flag's tolerance: 0.89996 shows 0.89, an exact 0.9 shows 0.90. + return "-" if value is None else f"{math.floor((value + _EPS) * 100) / 100:.2f}" + + +def _line(cells: List[str], widths: List[int]) -> str: + return " ".join(c.ljust(w) for c, w in zip(cells, widths, strict=True)).rstrip() diff --git a/dokimi/dokimi/runs/kinds.py b/dokimi/dokimi/runs/kinds.py index cb8e49a39..61e5a66f9 100644 --- a/dokimi/dokimi/runs/kinds.py +++ b/dokimi/dokimi/runs/kinds.py @@ -65,6 +65,7 @@ class KindFields: "repo_budget": KindFields(subject=("tree",), counts=("entries",)), "task_inspect": KindFields(subject=("root",), counts=("items",)), "cli_routes": KindFields(counts=("roots", "leaves", "tombstones")), + "headroom": KindFields(counts=("items", "inputs")), } #: Every schema_version an artifact may carry (current and superseded) -> its kind. diff --git a/dokimi/dokimi/schemas/headroom.v1.json b/dokimi/dokimi/schemas/headroom.v1.json new file mode 100644 index 000000000..fe2587187 --- /dev/null +++ b/dokimi/dokimi/schemas/headroom.v1.json @@ -0,0 +1,55 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/intel/dokimi/schemas/headroom.v1.json", + "title": "Arms headroom v1", + "description": "Which items of an arms comparison have no headroom: the no-skill arm already at the ceiling (or every arm at the floor), and items with too few repetitions to trust a flag. Written by `eval headroom`; reads arms.v1 files only.", + "type": "object", + "required": ["schema_version", "no_skill_arm", "thresholds", "runs", "items", "summary", "inputs"], + "additionalProperties": false, + "properties": { + "schema_version": {"const": "headroom.v1"}, + "no_skill_arm": {"type": "string", "description": "The control arm read as no-skill."}, + "thresholds": { + "type": "object", + "required": ["ceiling", "floor", "min_reps"], + "additionalProperties": false, + "properties": { + "ceiling": {"type": "number", "exclusiveMinimum": 0, "maximum": 1}, + "floor": {"type": "number", "minimum": 0}, + "min_reps": {"type": "integer", "minimum": 1} + } + }, + "runs": {"type": "integer", "minimum": 1, "description": "Number of distinct arms.v1 runs; a repeated input file counts once."}, + "inputs": {"type": "array", "items": {"type": "string"}, "description": "The arms.v1 files read, repeats dropped."}, + "items": { + "type": "array", + "items": { + "type": "object", + "required": ["question_id", "reps", "provisional", "no_skill", "arms", "unscored_arms", "flag"], + "additionalProperties": false, + "properties": { + "question_id": {"type": "string"}, + "reps": {"type": "integer", "minimum": 0, "description": "Number of runs in which the no-skill arm scored the item."}, + "provisional": {"type": "boolean", "description": "True when reps is below thresholds.min_reps."}, + "no_skill": {"type": ["number", "null"], "description": "Unrounded mean over runs of the no-skill arm, 0-1."}, + "arms": {"type": "object", "additionalProperties": {"type": ["number", "null"]}, "description": "Mean over runs of each arm, 0-1; null when no run scored it."}, + "unscored_arms": {"type": "array", "items": {"type": "string"}, "uniqueItems": true, "description": "The arms no run scored (null in arms), sorted; the floor test leaves them out."}, + "flag": {"enum": ["ceiling", "floor", null]} + } + } + }, + "summary": { + "type": "object", + "required": ["items", "ceiling", "floor", "provisional", "flagged_provisional", "no_flag"], + "additionalProperties": false, + "properties": { + "items": {"type": "integer", "minimum": 0}, + "ceiling": {"type": "integer", "minimum": 0}, + "floor": {"type": "integer", "minimum": 0}, + "provisional": {"type": "integer", "minimum": 0}, + "flagged_provisional": {"type": "integer", "minimum": 0}, + "no_flag": {"type": "integer", "minimum": 0} + } + } + } +} diff --git a/dokimi/tests/data/cli_flags_allowlist.txt b/dokimi/tests/data/cli_flags_allowlist.txt index f8e037fda..941acfeb0 100644 --- a/dokimi/tests/data/cli_flags_allowlist.txt +++ b/dokimi/tests/data/cli_flags_allowlist.txt @@ -222,3 +222,6 @@ --work-output-dir --work-repeats -o +--ceiling # eval headroom (#279) adds it: no registry concept yet; a F1b migration names it +--floor # eval headroom (#279) adds it: no registry concept yet; a F1b migration names it +--min-reps # eval headroom (#279) adds it: no registry concept yet; a F1b migration names it diff --git a/dokimi/tests/fixtures/artifacts/headroom.v1.json b/dokimi/tests/fixtures/artifacts/headroom.v1.json new file mode 100644 index 000000000..385000cec --- /dev/null +++ b/dokimi/tests/fixtures/artifacts/headroom.v1.json @@ -0,0 +1,35 @@ +{ + "schema_version": "headroom.v1", + "no_skill_arm": "baseline", + "thresholds": {"ceiling": 0.9, "floor": 0.0, "min_reps": 3}, + "runs": 1, + "inputs": ["arms.json"], + "items": [ + { + "question_id": "1", + "reps": 1, + "provisional": true, + "no_skill": 0.95, + "arms": {"baseline": 0.95, "skill": 0.94}, + "unscored_arms": [], + "flag": "ceiling" + }, + { + "question_id": "2", + "reps": 1, + "provisional": true, + "no_skill": 0.4, + "arms": {"baseline": 0.4, "skill": 0.7}, + "unscored_arms": [], + "flag": null + } + ], + "summary": { + "items": 2, + "ceiling": 1, + "floor": 0, + "provisional": 2, + "flagged_provisional": 1, + "no_flag": 1 + } +} diff --git a/dokimi/tests/test_headroom.py b/dokimi/tests/test_headroom.py new file mode 100644 index 000000000..732f2c50f --- /dev/null +++ b/dokimi/tests/test_headroom.py @@ -0,0 +1,680 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for ``dokimi.eval.headroom`` and ``eval headroom``: ceiling, floor and provisional items. + +Runs are built as arms.v1 payloads in memory; no model is called. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys + +import pytest + +from dokimi import records +from dokimi.cli import build_parser +from dokimi.commands.headroom import cmd_eval_headroom +from dokimi.eval.headroom import ( + CEILING, DEFAULT_CEILING, DEFAULT_FLOOR, DEFAULT_MIN_REPS, FLOOR, format_table, headroom, +) + + +def _run(items, baseline="baseline"): + """An arms.v1 payload: ``items`` is ``[(question_id, {arm: aggregate-or-None})]``.""" + answers, evaluations = [], [] + for qid, scores in items: + answers.append({"question_id": qid, "question_text": f"question {qid}", + "arms": {arm: {"answer": "x"} for arm in scores}}) + evaluations.append({"question_id": qid, + "scores": {arm: ({"aggregate": v} if v is not None else None) + for arm, v in scores.items()}}) + return records.read({"baseline_arm": baseline, "answers": answers, "evaluations": evaluations}, + source="test") + + +def _flags(result): + return {i["question_id"]: i["flag"] for i in result["items"]} + + +def test_no_skill_at_the_ceiling_is_flagged_and_just_below_is_not(): + run = _run([("1", {"baseline": 90, "skill": 95}), ("2", {"baseline": 89.9, "skill": 99})]) + assert _flags(headroom([run])) == {"1": CEILING, "2": None} + + +def test_the_threshold_compares_the_unrounded_mean(): + # 89.996 is below 0.9; rounding the mean to 4 decimals would have flagged it. + run = _run([("1", {"baseline": 89.996, "skill": 99})]) + assert _flags(headroom([run])) == {"1": None} + + +def test_a_mean_exactly_at_the_ceiling_is_flagged_though_float_division_undershoots(): + # Two runs at 85 and 95 average to exactly 90; 0.85 + 0.95 over 2 is 0.8999999999999999 in floats. + runs = [_run([("1", {"baseline": 85, "skill": 85})]), _run([("1", {"baseline": 95, "skill": 95})])] + assert _flags(headroom(runs, min_reps=1)) == {"1": CEILING} + + +def test_a_mean_exactly_at_the_floor_is_flagged_though_float_division_overshoots(): + # Two runs at 20 and 40 average to exactly 30; every arm at 30 is at the floor 0.3. + runs = [_run([("1", {"baseline": 20, "skill": 20})]), _run([("1", {"baseline": 40, "skill": 40})])] + assert _flags(headroom(runs, floor=0.3, min_reps=1)) == {"1": FLOOR} + + +def test_floor_needs_every_arm_at_zero_not_just_the_baseline(): + run = _run([("1", {"baseline": 0, "skill": 0}), ("2", {"baseline": 0, "skill": 40})]) + assert _flags(headroom([run])) == {"1": FLOOR, "2": None} + + +def test_floor_needs_the_baseline_at_the_floor_too_not_just_the_skill_arms(): + # The baseline answers question 1 (40) though the skill arm scores 0: no floor. + run = _run([("1", {"baseline": 40, "skill": 0}), ("2", {"baseline": 0, "skill": 0})]) + assert _flags(headroom([run])) == {"1": None, "2": FLOOR} + + +def test_thresholds_are_configurable(): + run = _run([("1", {"baseline": 60, "skill": 10}), ("2", {"baseline": 5, "skill": 15})]) + result = headroom([run], ceiling=0.5, floor=0.2) + assert _flags(result) == {"1": CEILING, "2": FLOOR} + assert result["thresholds"] == {"ceiling": 0.5, "floor": 0.2, "min_reps": 3} + + +def test_a_single_run_is_provisional_and_three_runs_are_not(): + one = _run([("1", {"baseline": 95, "skill": 95})]) + item = headroom([one])["items"][0] + assert (item["reps"], item["provisional"]) == (1, True) + three = [_run([("1", {"baseline": 95, "skill": 95})]) for _ in range(3)] + item = headroom(three)["items"][0] + assert (item["reps"], item["provisional"], item["flag"]) == (3, False, CEILING) + + +def test_provisional_flag_is_kept_next_to_the_ceiling_flag(): + result = headroom([_run([("1", {"baseline": 95, "skill": 95})])]) + assert result["summary"] == {"items": 1, "ceiling": 1, "floor": 0, "provisional": 1, + "flagged_provisional": 1, "no_flag": 0} + + +def test_summary_counts_each_kind_of_item_apart(): + # Three runs. "f1", "f2" and "ok" are in all three; "cp" and "up" only in the first, so they are provisional. + steady = [("f1", {"baseline": 0, "skill": 0}), ("f2", {"baseline": 0, "skill": 0}), + ("ok", {"baseline": 50, "skill": 60})] + first = [("cp", {"baseline": 95, "skill": 95}), ("up", {"baseline": 50, "skill": 60}), *steady] + result = headroom([_run(first), _run(steady), _run(steady)]) + assert _flags(result) == {"cp": CEILING, "up": None, "f1": FLOOR, "f2": FLOOR, "ok": None} + assert result["summary"] == {"items": 5, "ceiling": 1, "floor": 2, "provisional": 2, + "flagged_provisional": 1, "no_flag": 2} + + +def test_items_list_numeric_ids_in_numeric_order_then_the_other_ids(): + scores = {"baseline": 50, "skill": 60} + run = _run([(qid, scores) for qid in ["q1", "10", "b", "2", "a", "1"]]) + assert [i["question_id"] for i in headroom([run])["items"]] == ["1", "2", "10", "a", "b", "q1"] + + +def test_the_ceiling_reads_the_mean_over_runs(): + # One run at the top does not make a ceiling: the mean (0.85) is below 0.9. + runs = [_run([("1", {"baseline": 100, "skill": 100})]), _run([("1", {"baseline": 70, "skill": 70})])] + item = headroom(runs, min_reps=1)["items"][0] + assert item["no_skill"] == 0.85 and item["flag"] is None + runs[1] = _run([("1", {"baseline": 90, "skill": 90})]) + item = headroom(runs, min_reps=1)["items"][0] + assert item["no_skill"] == 0.95 and item["flag"] == CEILING + + +def test_reps_count_the_runs_that_scored_the_baseline(): + # The skill arm missed one run; the baseline scored all three, so the item has three reps. + runs = [_run([("1", {"baseline": 50, "skill": 50})]) for _ in range(3)] + runs[2] = _run([("1", {"baseline": 50, "skill": None})]) + item = headroom(runs)["items"][0] + assert item["reps"] == 3 and item["provisional"] is False + + +def test_reps_follow_the_baseline_even_when_a_skill_arm_was_scored_in_more_runs(): + runs = [_run([("1", {"baseline": 50, "skill": 50})]), _run([("1", {"baseline": None, "skill": 50})]), + _run([("1", {"baseline": None, "skill": 50})])] + item = headroom(runs)["items"][0] + assert item["reps"] == 1 and item["provisional"] is True + + +def test_an_item_the_baseline_never_scored_has_no_reps_and_no_flag(): + # Item 2 keeps the baseline scored somewhere, so the run is not refused as unjudged. + runs = [_run([("1", {"baseline": None, "skill": 95}), ("2", {"baseline": 40, "skill": 95})]) for _ in range(3)] + item = {i["question_id"]: i for i in headroom(runs)["items"]}["1"] + assert (item["reps"], item["no_skill"], item["flag"]) == (0, None, None) + + +def test_an_arm_no_run_scored_is_listed_as_unscored(): + run = _run([("1", {"baseline": 40, "skill": 60, "other": None})]) + item = headroom([run], min_reps=1)["items"][0] + assert item["unscored_arms"] == ["other"] and item["arms"]["other"] is None + + +def test_an_arm_scored_in_some_runs_is_not_unscored(): + runs = [_run([("1", {"baseline": 40, "skill": 60, "other": 50})]), + _run([("1", {"baseline": 40, "skill": 60, "other": None})])] + item = headroom(runs, min_reps=1)["items"][0] + assert item["unscored_arms"] == [] and item["arms"]["other"] == 0.5 + + +def test_floor_leaves_out_an_arm_no_run_scored(): + # "other" never answered; the arms that did are all at zero, so the item is a floor. + run = _run([("1", {"baseline": 0, "skill": 0, "other": None})]) + assert _flags(headroom([run])) == {"1": FLOOR} + + +def test_floor_needs_a_non_baseline_arm_that_scored(): + # Only the baseline answered (at zero): nothing shows the question is unanswerable for a skill. + assert _flags(headroom([_run([("1", {"baseline": 0})])])) == {"1": None} + assert _flags(headroom([_run([("1", {"baseline": 0, "skill": None})])])) == {"1": None} + + +def test_a_given_no_skill_arm_replaces_the_declared_baseline(): + # The runs declare "baseline"; the caller reads "skill" as the control. Every verdict follows "skill". + runs = [_run([("1", {"baseline": 10, "skill": 95}), ("2", {"baseline": 30, "skill": 0}), + ("3", {"baseline": 0, "skill": 0}), ("4", {"baseline": 40, "skill": None})]) + for _ in range(2)] + runs.append(_run([("1", {"baseline": 10, "skill": None}), ("2", {"baseline": 30, "skill": 0}), + ("3", {"baseline": 0, "skill": 0}), ("4", {"baseline": 40, "skill": None})])) + declared = headroom(runs) + assert declared["no_skill_arm"] == "baseline" and _flags(declared) == {"1": None, "2": None, "3": FLOOR, "4": None} + given = headroom(runs, no_skill_arm="skill") + assert given["no_skill_arm"] == "skill" + assert _flags(given) == {"1": CEILING, "2": None, "3": FLOOR, "4": None} + by_id = {i["question_id"]: i for i in given["items"]} + assert (by_id["1"]["no_skill"], by_id["1"]["reps"], by_id["1"]["provisional"]) == (0.95, 2, True) + assert (by_id["2"]["reps"], by_id["2"]["provisional"]) == (3, False) + assert (by_id["4"]["no_skill"], by_id["4"]["reps"]) == (None, 0) + + +def test_unknown_no_skill_arm_is_refused_and_lists_the_arms(): + # A mistyped arm used to read as "no ceiling anywhere" with exit 0. + run = _run([("1", {"baseline": 99, "skill": 99})]) + with pytest.raises(ValueError, match="'nope'.*the arms are: baseline, skill"): + headroom([run], no_skill_arm="nope") + + +def test_repeated_question_id_in_one_run_is_refused(): + run = records.read({"baseline_arm": "baseline", "answers": [ + {"question_id": "1", "question_text": "q", "arms": {"baseline": {"answer": "a"}}}, + {"question_id": "1", "question_text": "q", "arms": {"baseline": {"answer": "b"}}}]}) + with pytest.raises(ValueError, match="question_id '1' more than once"): + headroom([run], no_skill_arm="baseline") + + +def test_a_non_ascii_digit_question_id_is_kept_and_sorted_as_text(): + # '²'.isdigit() is true and int('²') raises: the id is an ordinary id, sorted after the numeric ones. + run = _run([("²", {"baseline": 50, "skill": 50}), ("10", {"baseline": 50, "skill": 50}), + ("2", {"baseline": 50, "skill": 50})]) + assert [i["question_id"] for i in headroom([run])["items"]] == ["2", "10", "²"] + + +def test_a_digit_question_id_past_the_int_limit_is_sorted_not_refused(): + # int() refuses more than 4300 digits (Python's str-to-int limit): a long numeric id still sorts by value. + long_id = "1" * 5000 + run = _run([(long_id, {"baseline": 50, "skill": 50}), ("10", {"baseline": 50, "skill": 50}), + ("0002", {"baseline": 50, "skill": 50})]) + assert [i["question_id"] for i in headroom([run])["items"]] == ["0002", "10", long_id] + + +def test_an_arm_named_with_docs_that_no_run_scored_is_listed_as_unscored(): + # arms.v1 allows with_docs as an arm name; arm_names() drops it, so an unscored one went unlisted. + run = _run([("1", {"baseline": 40, "skill": 60, "with_docs": None})]) + item = headroom([run], min_reps=1)["items"][0] + assert item["unscored_arms"] == ["with_docs"] and item["arms"]["with_docs"] is None + + +def test_the_table_and_json_do_not_round_0_89996_up_to_the_ceiling(): + run = _run([("1", {"baseline": 89.996, "skill": 99})]) + result = headroom([run], min_reps=2) + assert result["items"][0]["no_skill"] == pytest.approx(0.89996) and result["items"][0]["flag"] is None + table = format_table(result) + assert "0.89" in table and "0.90" not in table + + +def test_the_table_lists_the_control_arm_once(): + result = headroom([_run([("1", {"baseline": 40, "skill": 60})])], min_reps=2) + header = format_table(result).splitlines()[0].split() + assert header == ["question", "reps", "no-skill", "skill", "flag", "provisional"] + + +@pytest.mark.parametrize("kwargs, message", [ + ({"ceiling": 0.0}, r"ceiling must be in \(0, 1\], not 0\.0"), + ({"ceiling": -0.1}, r"ceiling must be in \(0, 1\], not -0\.1"), + ({"ceiling": 1.5}, r"ceiling must be in \(0, 1\], not 1\.5"), + ({"floor": 0.9}, r"floor must be in \[0, ceiling\), not 0\.9 \(ceiling 0\.9\)"), + ({"floor": -0.1}, r"floor must be in \[0, ceiling\), not -0\.1"), + ({"min_reps": 0}, r"min_reps must be at least 1, not 0"), + ({"min_reps": -1}, r"min_reps must be at least 1, not -1"), + ({"ceiling": 0.005}, r"ceiling must be at least 0\.01, not 0\.005"), + ({"floor": 0.005}, r"floor must be 0 or at least 0\.01, not 0\.005"), +]) +def test_unusable_thresholds_are_refused(kwargs, message): + run = _run([("1", {"baseline": 50, "skill": 50})]) + with pytest.raises(ValueError, match=message): + headroom([run], **kwargs) + + +def test_the_threshold_boundaries_are_accepted(): + # ceiling 1.0 is the top of (0, 1]; floor 0 is the bottom of [0, ceiling); one rep is the least. + run = _run([("1", {"baseline": 100, "skill": 100}), ("2", {"baseline": 99, "skill": 99})]) + result = headroom([run], ceiling=1.0, floor=0.0, min_reps=1) + assert _flags(result) == {"1": CEILING, "2": None} + assert result["thresholds"] == {"ceiling": 1.0, "floor": 0.0, "min_reps": 1} + assert headroom([run], ceiling=0.5, floor=0.49)["thresholds"]["floor"] == 0.49 + assert headroom([run], ceiling=0.01, floor=0.0, min_reps=1)["thresholds"]["ceiling"] == 0.01 + + +def test_no_runs_are_refused(): + with pytest.raises(ValueError, match="no runs to read"): + headroom([]) + + +def test_a_run_with_no_questions_has_no_items(): + result = headroom([_run([])]) + assert result["items"] == [] and result["runs"] == 1 + assert result["summary"] == {"items": 0, "ceiling": 0, "floor": 0, "provisional": 0, + "flagged_provisional": 0, "no_flag": 0} + + +def test_a_run_nothing_was_judged_in_is_refused(): + # The answers are there but nothing was judged: no item could be read as a ceiling or a floor. + run = records.read({"baseline_arm": "baseline", "answers": [ + {"question_id": "1", "question_text": "q", "arms": {"baseline": {"answer": "a"}, "skill": {"answer": "b"}}}]}) + with pytest.raises(ValueError, match="no run scored the no-skill arm 'baseline'"): + headroom([run]) + + +def test_a_baseline_named_differently_from_the_scores_is_refused(): + run = _run([("1", {"no_skill": 100, "skill": 50})], baseline="control") + with pytest.raises(ValueError, match="no run scored the no-skill arm 'control'"): + headroom([run]) + + +def test_binary_answers_are_refused(): + binary = records.read({"answers": [{"question_id": "1", "question_text": "q", + "with_docs": {"answer": "a"}, "without_docs": {"answer": "b"}}]}) + with pytest.raises(ValueError, match="arms runs only"): + headroom([binary], no_skill_arm="without_docs") + + +def test_runs_that_declare_different_baselines_need_an_explicit_arm(): + a = _run([("1", {"baseline": 50, "skill": 50})], baseline="baseline") + b = _run([("1", {"control": 50, "skill": 50})], baseline="control") + with pytest.raises(ValueError, match="--baseline-arm"): + headroom([a, b]) + result = headroom([a, b], no_skill_arm="baseline") + assert result["no_skill_arm"] == "baseline" and result["runs"] == 2 + item = result["items"][0] + # "baseline" was scored in run a only, so one rep; "control" is just another arm here. + assert (item["reps"], item["provisional"], item["no_skill"], item["flag"]) == (1, True, 0.5, None) + assert item["arms"] == {"baseline": 0.5, "control": 0.5, "skill": 0.5} + assert headroom([a, b], no_skill_arm="control")["items"][0]["reps"] == 1 + + +def test_table_notes_the_arms_no_run_scored(): + run = _run([("1", {"baseline": 40, "skill": 60, "other": None}), ("2", {"baseline": 40, "skill": 60})]) + table = format_table(headroom([run], min_reps=1)) + assert "no run scored" in table and "other in 1 item" in table + + +def test_table_names_each_flagged_item(): + result = headroom([_run([("7", {"baseline": 97, "skill": 98}), ("8", {"baseline": 40, "skill": 60})])], + min_reps=1) + table = format_table(result) + assert "ceiling" in table and "\n7 " in table and "\n8 " not in table + assert "ceiling 1, floor 0" in table + + +def _args(paths, **overrides): + base = {"arms": [str(p) for p in paths], "no_skill_arm": None, "ceiling": 0.9, "floor": 0.0, + "min_reps": 3, "output": None} + return argparse.Namespace(**{**base, **overrides}) + + +def test_command_writes_a_headroom_artifact_and_prints_the_table(tmp_path, capsys): + arms_path = tmp_path / "arms.json" + arms_path.write_text(json.dumps({"baseline_arm": "baseline", "answers": [ + {"question_id": "1", "question_text": "q", "arms": {"baseline": {"answer": "a"}, "skill": {"answer": "b"}}}], + "evaluations": [{"question_id": "1", "scores": {"baseline": {"aggregate": 93}, "skill": {"aggregate": 94}}}]})) + out = tmp_path / "headroom.json" + cmd_eval_headroom(_args([arms_path], output=out)) + saved = json.loads(out.read_text(encoding="utf-8")) + assert saved["schema_version"] == "headroom.v1" + assert saved["items"][0]["flag"] == CEILING and saved["inputs"] == [str(arms_path)] + printed = capsys.readouterr().out + assert "ceiling" in printed and f"Wrote headroom artifact to {out}" in printed + + +def test_command_exits_2_on_an_unreadable_input(tmp_path, capsys): + with pytest.raises(SystemExit) as exc: + cmd_eval_headroom(_args([tmp_path / "missing.json"])) + assert exc.value.code == 2 + assert "file not found" in capsys.readouterr().err + + +def _arms_file(path, scores, baseline="baseline"): + """An arms.v1 file on disk: ``scores`` is ``{question_id: {arm: aggregate}}``.""" + payload = {"baseline_arm": baseline, + "answers": [{"question_id": qid, "question_text": f"question {qid}", + "arms": {arm: {"answer": "x"} for arm in arms}} + for qid, arms in scores.items()], + "evaluations": [{"question_id": qid, + "scores": {arm: {"aggregate": v} for arm, v in arms.items()}} + for qid, arms in scores.items()]} + path.write_text(json.dumps(payload), encoding="utf-8") + return path + + +def test_command_exits_2_on_a_positive_threshold_below_one_score_step(tmp_path, capsys): + path = _arms_file(tmp_path / "arms.json", {"1": {"baseline": 99, "skill": 99}}) + with pytest.raises(SystemExit) as exc: + cmd_eval_headroom(_args([path], floor=0.005)) + assert exc.value.code == 2 + assert "floor must be 0 or at least 0.01, not 0.005" in capsys.readouterr().err + + +def test_command_refuses_an_unknown_no_skill_arm_and_lists_the_arms(tmp_path, capsys): + path = _arms_file(tmp_path / "arms.json", {"1": {"baseline": 99, "skill": 99}}) + with pytest.raises(SystemExit) as exc: + cmd_eval_headroom(_args([path], no_skill_arm="basline")) + assert exc.value.code == 2 + err = capsys.readouterr().err + assert "'basline'" in err and "the arms are: baseline, skill" in err + + +def test_repeated_input_files_count_as_one_run(tmp_path, capsys): + one = _arms_file(tmp_path / "run1.json", {"1": {"baseline": 95, "skill": 95}}) + copy = tmp_path / "copy.json" + copy.write_bytes(one.read_bytes()) + out = tmp_path / "headroom.json" + cmd_eval_headroom(_args([one, one, one, copy], output=out)) + saved = json.loads(out.read_text(encoding="utf-8")) + assert saved["runs"] == 1 and saved["inputs"] == [str(one)] + assert saved["items"][0]["reps"] == 1 and saved["items"][0]["provisional"] is True + err = capsys.readouterr().err + assert err.count("counted once") == 3 and f"skipped {copy}: same content as {one}, counted once" in err + + +def test_distinct_input_files_each_count_as_a_run(tmp_path): + paths = [_arms_file(tmp_path / f"run{i}.json", {"1": {"baseline": 95 - i, "skill": 95}}) for i in range(3)] + out = tmp_path / "headroom.json" + cmd_eval_headroom(_args(paths, output=out)) + saved = json.loads(out.read_text(encoding="utf-8")) + item = saved["items"][0] + assert (saved["runs"], item["reps"], item["provisional"], item["flag"]) == (3, 3, False, CEILING) + + +@pytest.mark.parametrize("field, value", [("model", "other-model"), ("judge_model", "other-judge"), + ("library_name", "other-library"), ("run_config", {"max_tokens": 9})]) +def test_command_refuses_runs_of_different_configurations(tmp_path, capsys, field, value): + one = _arms_file(tmp_path / "run1.json", {"1": {"baseline": 95, "skill": 95}}) + two = _arms_file(tmp_path / "run2.json", {"1": {"baseline": 94, "skill": 95}}) + for path, extra in ((one, {}), (two, {field: value})): + data = json.loads(path.read_text(encoding="utf-8")) + path.write_text(json.dumps({**data, "model": "m", "judge_model": "j", "library_name": "lib", + "run_config": {"max_tokens": 5, "seed": 1}, **extra}), encoding="utf-8") + out = tmp_path / "headroom.json" + with pytest.raises(SystemExit) as exc: + cmd_eval_headroom(_args([one, two], output=out)) + assert exc.value.code == 2 and not out.exists() + assert f"Error: {two} is not a repetition of {one}" in capsys.readouterr().err + + +def test_runs_that_differ_only_in_the_seed_are_repetitions(tmp_path): + paths = [] + for seed in (1, 2): + path = _arms_file(tmp_path / f"run{seed}.json", {"1": {"baseline": 95, "skill": 95 - seed}}) + data = json.loads(path.read_text(encoding="utf-8")) + path.write_text(json.dumps({**data, "model": "m", "run_config": {"max_tokens": 5, "seed": seed}}), + encoding="utf-8") + paths.append(path) + out = tmp_path / "headroom.json" + cmd_eval_headroom(_args(paths, output=out)) + saved = json.loads(out.read_text(encoding="utf-8")) + assert (saved["runs"], saved["items"][0]["reps"]) == (2, 2) + + +def test_command_exits_2_when_the_output_cannot_be_written(tmp_path, capsys): + path = _arms_file(tmp_path / "arms.json", {"1": {"baseline": 95, "skill": 95}}) + blocker = tmp_path / "a-file" + blocker.write_text("not a directory, so --out cannot be written beneath it", encoding="utf-8") + with pytest.raises(SystemExit) as exc: + cmd_eval_headroom(_args([path], output=blocker / "headroom.json")) + assert exc.value.code == 2 # the exit-code table in docs/quickstart.md: an unwritable output is 2 + assert capsys.readouterr().err.startswith(f"Error: cannot write {blocker / 'headroom.json'}: ") + + +@pytest.mark.parametrize("spelling", ["dir", "dot", "dot-slash", "absent-with-slash", "absent-with-slash-dot", + "file-with-slash"]) +def test_command_exits_2_when_the_output_is_a_directory(tmp_path, monkeypatch, capsys, spelling): + # A directory output would raise ValueError ("."), fail on the directory, or (spelled with a + # slash, not existing yet) write a plain file under the directory's name: refuse all up front. + path = _arms_file(tmp_path / "arms.json", {"1": {"baseline": 95, "skill": 95}}) + (tmp_path / "out").mkdir() + (tmp_path / "existing.json").write_text("keep", encoding="utf-8") + monkeypatch.chdir(tmp_path) + out = {"dir": str(tmp_path / "out"), "dot": ".", "dot-slash": "./", + "absent-with-slash": str(tmp_path / "newdir") + "/", + "absent-with-slash-dot": str(tmp_path / "newdir") + "/.", + "file-with-slash": str(tmp_path / "existing.json") + "/"}[spelling] + with pytest.raises(SystemExit) as exc: + cmd_eval_headroom(_args([path], output=out)) + assert exc.value.code == 2 + assert "is not a file name" in capsys.readouterr().err + assert not (tmp_path / "newdir").exists() + assert (tmp_path / "existing.json").read_text(encoding="utf-8") == "keep" + + +def test_an_output_whose_directory_check_cannot_stat_does_not_crash_the_guard(tmp_path, monkeypatch, capsys): + # A parent without search permission makes is_dir() raise PermissionError on Python 3.12 and 3.13. + # The check must treat that path as not a directory and let the save report it, as before. + import pathlib + + arms = _arms_file(tmp_path / "arms.json", {"1": {"baseline": 99, "skill": 99}}) + out = tmp_path / "locked.json" + real_is_dir = pathlib.Path.is_dir + + def is_dir(self): + if self.name == out.name: + raise PermissionError(13, "Permission denied", str(self)) + return real_is_dir(self) + + monkeypatch.setattr(pathlib.Path, "is_dir", is_dir) + try: + cmd_eval_headroom(_args([arms], output=out)) + code = 0 + except SystemExit as exc: + code = exc.code + assert code == 0 and out.exists() # the simulated stat error is not a refusal: the save writes + + +def test_command_refuses_an_output_spelled_through_a_dotdot_to_an_input(tmp_path, capsys): + # The same file under another spelling: the input as given, the output as sub/../arms.json. + path = _arms_file(tmp_path / "arms.json", {"1": {"baseline": 99, "skill": 99}}) + (tmp_path / "sub").mkdir() + before = path.read_bytes() + with pytest.raises(SystemExit) as exc: + cmd_eval_headroom(_args([str(path)], output=str(tmp_path / "sub" / ".." / "arms.json"))) + assert exc.value.code == 2 + assert path.read_bytes() == before + assert "overwrite" in capsys.readouterr().err + + +def test_command_refuses_a_question_id_repeated_within_one_file(tmp_path, capsys): + path = tmp_path / "dup.json" + path.write_text(json.dumps({"baseline_arm": "baseline", "answers": [ + {"question_id": "1", "question_text": "q", "arms": {"baseline": {"answer": "a"}}}, + {"question_id": "1", "question_text": "q", "arms": {"baseline": {"answer": "b"}}}]})) + with pytest.raises(SystemExit) as exc: + cmd_eval_headroom(_args([path])) + assert exc.value.code == 2 + err = capsys.readouterr().err + assert str(path) in err and "question_id '1' appears more than once" in err + + +def test_command_passes_non_default_thresholds_to_the_analysis(tmp_path): + # With the defaults neither item is flagged: 0.6 < 0.9, and 0.15 > 0.0. + path = _arms_file(tmp_path / "arms.json", {"1": {"baseline": 60, "skill": 10}, + "2": {"baseline": 5, "skill": 15}}) + out = tmp_path / "headroom.json" + cmd_eval_headroom(_args([path], ceiling=0.5, floor=0.2, min_reps=1, output=out)) + saved = json.loads(out.read_text(encoding="utf-8")) + assert {i["question_id"]: i["flag"] for i in saved["items"]} == {"1": CEILING, "2": FLOOR} + assert saved["thresholds"] == {"ceiling": 0.5, "floor": 0.2, "min_reps": 1} + assert [i["provisional"] for i in saved["items"]] == [False, False] + + +def test_the_registered_options_default_to_the_documented_thresholds(tmp_path): + args = build_parser().parse_args(["eval", "headroom", str(tmp_path / "a.json")]) + assert (args.ceiling, args.floor, args.min_reps) == (DEFAULT_CEILING, DEFAULT_FLOOR, DEFAULT_MIN_REPS) + assert (args.no_skill_arm, args.output) == (None, None) + assert args.func is cmd_eval_headroom + + +def test_the_registered_options_reach_the_analysis_through_the_real_parser(tmp_path): + path = _arms_file(tmp_path / "arms.json", {"1": {"baseline": 60, "skill": 10}, + "2": {"baseline": 5, "skill": 15}}) + out = tmp_path / "headroom.json" + args = build_parser().parse_args(["eval", "headroom", str(path), "--ceiling", "0.5", "--floor", "0.2", + "--min-reps", "1", "--baseline-arm", "baseline", "--out", str(out)]) + args.func(args) + saved = json.loads(out.read_text(encoding="utf-8")) + assert saved["thresholds"] == {"ceiling": 0.5, "floor": 0.2, "min_reps": 1} + assert {i["question_id"]: i["flag"] for i in saved["items"]} == {"1": CEILING, "2": FLOOR} + + +def test_command_reads_a_file_with_no_questions(tmp_path, capsys): + path = tmp_path / "empty.json" + path.write_text(json.dumps({"baseline_arm": "baseline", "answers": [], "evaluations": []}), encoding="utf-8") + out = tmp_path / "headroom.json" + cmd_eval_headroom(_args([path], output=out)) + assert "(no item is flagged or provisional)" in capsys.readouterr().out + saved = json.loads(out.read_text(encoding="utf-8")) + assert saved["items"] == [] and saved["summary"]["items"] == 0 and saved["runs"] == 1 + + +def test_command_refuses_a_file_nothing_was_judged_in(tmp_path, capsys): + path = tmp_path / "unjudged.json" + path.write_text(json.dumps({"baseline_arm": "baseline", "answers": [ + {"question_id": "1", "question_text": "q", "arms": {"baseline": {"answer": "a"}, "skill": {"answer": "b"}}}]}), + encoding="utf-8") + out = tmp_path / "headroom.json" + with pytest.raises(SystemExit) as exc: + cmd_eval_headroom(_args([path], output=out)) + assert exc.value.code == 2 and not out.exists() + assert "no run scored the no-skill arm 'baseline'" in capsys.readouterr().err + + +def test_command_refuses_a_wrong_baseline_name_with_exit_2(tmp_path, capsys): + path = _arms_file(tmp_path / "arms.json", {"1": {"no_skill": 100, "skill": 50}}, baseline="control") + with pytest.raises(SystemExit) as exc: + cmd_eval_headroom(_args([path])) + assert exc.value.code == 2 + assert "pass --baseline-arm" in capsys.readouterr().err + + +def test_command_exits_2_on_an_input_nested_too_deep_to_read(tmp_path, capsys): + path = tmp_path / "deep.json" + path.write_text("[" * 100000, encoding="utf-8") + with pytest.raises(SystemExit) as exc: + cmd_eval_headroom(_args([path])) + assert exc.value.code == 2 + assert "Error:" in capsys.readouterr().err + + +def test_min_reps_must_be_a_positive_integer_at_the_parser(tmp_path, capsys): + with pytest.raises(SystemExit) as exc: + build_parser().parse_args(["eval", "headroom", str(tmp_path / "a.json"), "--min-reps", "0"]) + assert exc.value.code == 2 + assert "--min-reps" in capsys.readouterr().err + + +def test_command_refuses_an_output_that_is_one_of_its_inputs(tmp_path, capsys): + path = _arms_file(tmp_path / "arms.json", {"1": {"baseline": 99, "skill": 99}}) + before = path.read_bytes() + with pytest.raises(SystemExit) as exc: + cmd_eval_headroom(_args([path], output=path)) + assert exc.value.code == 2 + assert path.read_bytes() == before + assert "overwrite" in capsys.readouterr().err + + +def test_command_refuses_an_output_whose_rejected_sibling_is_an_input(tmp_path, capsys): + # save_artifact deletes .rejected after a good save: an input under that name would be lost. + run = _arms_file(tmp_path / "run.json", {"1": {"baseline": 99, "skill": 99}}) + stale = tmp_path / "headroom.json.rejected" + stale.write_bytes(run.read_bytes()) + before = stale.read_bytes() + with pytest.raises(SystemExit) as exc: + cmd_eval_headroom(_args([stale], output=tmp_path / "headroom.json")) + assert exc.value.code == 2 + assert stale.read_bytes() == before + assert "overwrite" in capsys.readouterr().err + + +def test_an_output_whose_path_cannot_be_resolved_does_not_crash_the_guard(tmp_path, monkeypatch): + # Python 3.12 and older raise RuntimeError from resolve() on a symlink loop; 3.13+ do not. + # The guard must not turn that into a traceback: an unresolvable path is not an input. + import pathlib + + arms = _arms_file(tmp_path / "arms.json", {"1": {"baseline": 99, "skill": 99}}) + out = tmp_path / "loop.json" + real_resolve = pathlib.Path.resolve + + def resolve(self, *a, **k): + if self.name == out.name: + raise RuntimeError(f"Symlink loop from {self}") + return real_resolve(self, *a, **k) + + monkeypatch.setattr(pathlib.Path, "resolve", resolve) + cmd_eval_headroom(_args([arms], output=out)) + assert out.exists() + + +@pytest.mark.skipif(sys.platform == "win32", reason="symlinks need privileges on Windows") +def test_an_output_that_is_a_symlink_to_an_input_is_refused(tmp_path, capsys): + path = _arms_file(tmp_path / "arms.json", {"1": {"baseline": 99, "skill": 99}}) + alias = tmp_path / "alias.json" + os.symlink(path.name, alias) + before = path.read_bytes() + with pytest.raises(SystemExit) as exc: + cmd_eval_headroom(_args([path], output=alias)) + assert exc.value.code == 2 + assert path.read_bytes() == before + assert "overwrite" in capsys.readouterr().err + + +def test_a_reserialized_copy_of_a_run_counts_once(tmp_path, capsys): + arms = _arms_file(tmp_path / "run1.json", {"1": {"baseline": 95, "skill": 40}}) + copy = tmp_path / "copy.json" + copy.write_text(json.dumps(json.loads(arms.read_text(encoding="utf-8")), indent=2, sort_keys=True), + encoding="utf-8") + out = tmp_path / "headroom.json" + cmd_eval_headroom(_args([arms, copy], output=out)) + saved = json.loads(out.read_text(encoding="utf-8")) + assert saved["runs"] == 1 and saved["items"][0]["reps"] == 1 + assert "counted once" in capsys.readouterr().err + + +def test_table_note_counts_the_unscored_items_per_arm_instead_of_listing_them(): + items = [(str(q), {"baseline": 40, "skill": None}) for q in range(1, 501)] + table = format_table(headroom([_run(items)], min_reps=1)) + note = next(line for line in table.splitlines() if line.startswith("note:")) + assert note.endswith("skill in 500 items") and "(" not in note + + +def test_floor_is_not_flagged_when_the_no_skill_arm_was_never_scored_on_the_item(): + # Item 1's baseline judge failed; the skill at 0 alone is not a floor. Item 2 is one. + run = _run([("1", {"baseline": None, "skill": 0}), ("2", {"baseline": 0, "skill": 0})]) + assert _flags(headroom([run], min_reps=1)) == {"1": None, "2": FLOOR} + + +def test_an_empty_out_is_refused_with_exit_2_through_the_real_parser(tmp_path, capsys): + path = _arms_file(tmp_path / "arms.json", {"1": {"baseline": 95, "skill": 95}}) + args = build_parser().parse_args(["eval", "headroom", str(path), "--out", ""]) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 + assert "not a file name" in capsys.readouterr().err diff --git a/dokimi/tests/test_runs_scan.py b/dokimi/tests/test_runs_scan.py index 9b5b9997d..c0dd16a45 100644 --- a/dokimi/tests/test_runs_scan.py +++ b/dokimi/tests/test_runs_scan.py @@ -48,6 +48,7 @@ "repo_budget.v1": ({}, "toy", None, None), "task_inspect.v1": ({}, "tests/fixtures/task_formats", None, None), "cli_routes.v1": ({}, None, None, None), + "headroom.v1": ({}, None, None, None), } diff --git a/dokimi/tests/test_runs_show.py b/dokimi/tests/test_runs_show.py index 07ad7a9f5..82023e0fa 100644 --- a/dokimi/tests/test_runs_show.py +++ b/dokimi/tests/test_runs_show.py @@ -37,6 +37,7 @@ "result.v1": {"cells": 6}, "subject_scorecard.v2": {"cells": 0}, "cli_routes.v1": {"roots": 2, "leaves": 2, "tombstones": 1}, + "headroom.v1": {"items": 2, "inputs": 1}, "task_inspect.v1": {"items": 4}, "task_runs.v1": {"tasks": 6, "harnesses": 2, "results": 24}, "tool_preference.v1": {"items": 16, "results": 16},