From 30030ed1d0201159419a312d980e03bbc2c0cece Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Thu, 29 Oct 2026 17:01:50 +0000 Subject: [PATCH 01/16] feat(repo): compare records start context, filters tasks, flags rows An evaluation of AGENTS.md guidance files (26 review_replay tasks, two arms, three reps) showed three gaps in reading a batch with repo compare. Start context: a batch did not say what each cell started with, so whether an arm's guidance was delivered had to be read by hand from every agent.stdout. The Transcript now carries first_prompt_tokens, the input + cache_creation + cache_read tokens of the first main-thread call, read with dokimi.context.trace (one call per message id, subagent calls apart). repo run writes it as result.json agent.first_prompt_tokens (null until set, and for a trace without such a call), and repo compare compares it as a descriptive measure next to cost_usd and turns. Like a process count, it is compared only when both arms report it, so a batch written before the field has no row rather than a row of zeros. Task selection: repo compare --task ID (repeatable) compares only those task ids. An id no finished cell has is refused through CompareError; the tasks left out are not listed as unpaired nor their cells as missing; the summary records the selection as task_filter and the text header prints "tasks: N of M (--task)". Reading notes, with no number changed: a gate or score row of 2+ tasks whose interval includes 0 gets an "inconclusive" note with mde, the smallest effect 80% power detects at that spread, (1.959964 + 0.841621) * sd / sqrt(n) (mde is in every row of --json); a row whose interval and sign-flip p disagree gets interval_p_disagree and a note naming the two methods, saying for gate and score that p is the one to read; descriptive rows with p < 0.05 are named on one line as uncorrected and outside Holm's family. Contract sections 4 and 9, docs/context-window.md and the compare module docstring describe the field, the flag and the notes. Signed-off-by: Anton Volkov --- ...61029_150000_antonwolfy_compare-reading.md | 15 ++ dokimi/docs/context-window.md | 4 + dokimi/docs/repo-evaluation-contract.md | 35 +++- dokimi/dokimi/commands/repo_run.py | 4 +- dokimi/dokimi/repos/compare.py | 87 +++++++++- dokimi/dokimi/repos/events.py | 9 +- dokimi/dokimi/repos/runner.py | 3 +- dokimi/tests/repo_eval_fixture.py | 3 +- dokimi/tests/test_repo_compare_reading.py | 157 ++++++++++++++++++ dokimi/tests/test_repo_reach.py | 18 ++ dokimi/tests/test_repo_run.py | 6 +- 11 files changed, 323 insertions(+), 18 deletions(-) create mode 100644 dokimi/changelog.d/20261029_150000_antonwolfy_compare-reading.md create mode 100644 dokimi/tests/test_repo_compare_reading.py diff --git a/dokimi/changelog.d/20261029_150000_antonwolfy_compare-reading.md b/dokimi/changelog.d/20261029_150000_antonwolfy_compare-reading.md new file mode 100644 index 000000000..3246881de --- /dev/null +++ b/dokimi/changelog.d/20261029_150000_antonwolfy_compare-reading.md @@ -0,0 +1,15 @@ +### Added + +- **`repo run` records each cell's start context, and `repo compare` compares it** — `result.json` + `agent.first_prompt_tokens` is what the first main-thread model call was sent (input plus cache + creation plus cache read tokens, one call per message id, as `context trace` reads it): start-up and + the task. It is `null` for a trace without that call and for the `text` format. `repo compare` adds a + descriptive `first_prompt_tokens` row; a batch written before the field has no such row. +- **`repo compare --task ID`** — repeatable: compares only those task ids. An id no finished cell of the + batches has exits 1 and names it; the tasks left out are not listed as unpaired, and the `--json` + summary (`task_filter`) and the text header (`tasks: N of M (--task)`) record the selection. +- **Reading notes under a `repo compare` comparison** — a gate or score row of 2+ tasks whose interval + includes 0 says it is inconclusive and gives `mde`, the smallest effect 80% power detects at that + spread (`mde` is in every `--json` row); a row whose interval and p disagree says so and why + (`interval_p_disagree`), and for gate and score that p is the one to read; descriptive rows with + p below 0.05 are named as uncorrected and outside Holm's family. No number changes. diff --git a/dokimi/docs/context-window.md b/dokimi/docs/context-window.md index 1469a6ecf..1d093fef6 100644 --- a/dokimi/docs/context-window.md +++ b/dokimi/docs/context-window.md @@ -79,3 +79,7 @@ point `CLAUDE_CONFIG_DIR` at an empty directory to leave it out. The `surface` field of `--json` (the tool, skill and agent counts, and the names of the MCP servers and loaded plugins) shows what start-up offered, so two runs that differ there are not the same condition. + +`repo run` records **first** for every cell as `result.json` +`agent.first_prompt_tokens`, and `repo compare` compares it between arms as +the descriptive measure `first_prompt_tokens`. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 38d9c96d8..9b2dd1c08 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -419,6 +419,12 @@ reads the upstream base (`base_sha`) and the workspace start commit Code's `modelUsage`), `main` when the harness reported the parent's only, and `null` when it reported none. `subagent_calls` counts the parent's and subagents' `Agent`/`Task` calls; `null` when the harness wrote no result record. +`first_prompt_tokens` is what the first main-thread model call was sent +(`input + cache_creation + cache_read` tokens of that call, counted once per +message id, as `context trace` reads it): everything start-up loaded plus the +task, so two arms that differ only in guidance differ there by what start-up +loaded of it. It is `null` when the trace has no main-thread call with +`usage`, and for a trace format that carries none (`text`). `result.json` `process` holds what the framework counts in the events, the parent's and subagents' together, so a grader need not: @@ -1315,15 +1321,34 @@ python cli.py repo compare repo-runs/replay --baseline guidance:none --metric re - `repo compare` reduces cells to one value per (task, arm, model) first: repetitions share a task's difficulty and are not paired across arms, so the task is the unit. Per treatment arm, model and measure (gate share, score, - cost, turns, each `--metric` from `grade.json`, and each `process` count a + cost, turns, `first_prompt_tokens`, each `--metric` from `grade.json`, and each `process` count a compared cell reports as `process:` or `counter:`) it gives the mean per-task delta against the baseline with a 95% interval over tasks (a bootstrap for - gate and score, a t interval for cost, turns, metrics and process counts, `-` for a single + gate and score, a t interval for the descriptive measures, `-` for a single task), the tasks up, down and tied, and a two-sided sign-flip permutation p (exact up to 16 changed tasks; at one cell per task and arm it equals McNemar's exact test). - A `process` count only one arm reports for the shared tasks of a - (treatment, model) is not compared; a warning names it. Holm's correction runs over the gate and score comparisons of one call; the - rest are descriptive. A gate or score comparison carries `headroom`: + A `process` count or `first_prompt_tokens` only one arm reports for the shared tasks of a + (treatment, model) is not compared; a warning names it. When neither arm + reports it (a batch run before `repo run` recorded it), the row is absent. + Holm's correction runs over the gate and score comparisons of one call; the + rest are descriptive. Three notes under a comparison keep a row from being + misread, and change no number. A gate or score row of 2+ tasks whose + interval includes 0 is inconclusive, not evidence of no effect: the note + gives `mde`, the smallest mean delta a two-sided 5% test finds with 80% power + at that spread, `(1.959964 + 0.841621) * sd / sqrt(n)` over the per-task + deltas (every row has `mde` in `--json`; `null` for one task or deltas with + no spread, and then no note). A row whose interval and p disagree (the + interval excludes 0 while p is 0.05 or more, or includes 0 while p is below + 0.05) has `interval_p_disagree` `true` and a note: they come from two + methods, and for gate and score p is the one to read, since a percentile + bootstrap runs narrow over few tasks; the inconclusive note gives way to + it. Descriptive rows with p below 0.05 are named on one line: their p is + uncorrected and outside Holm's family. `--task ID` (repeatable) compares only + those tasks: an id no finished cell of the batches has is refused, the tasks + left out are not listed as unpaired nor their cells as missing, the `--json` summary records the + selection as `task_filter` (`tasks`, and `of`, the task count of the + batches; `null` without `--task`), and the text header says `tasks: N of M + (--task)`. A gate or score comparison carries `headroom`: `ceiling` (both arms at 1.0 or the task's `score.max` on every paired task), `floor` (both at 0 on every one), `near_ceiling` (the baseline at the top on all but at most one of 2+ tasks and the treatment nowhere below diff --git a/dokimi/dokimi/commands/repo_run.py b/dokimi/dokimi/commands/repo_run.py index 035933761..03aae1112 100644 --- a/dokimi/dokimi/commands/repo_run.py +++ b/dokimi/dokimi/commands/repo_run.py @@ -401,7 +401,7 @@ def cmd_repo_compare(args: argparse.Namespace) -> None: arms = [a for spec in args.arm for a in spec.split(",") if a] if args.arm else None try: summary = compare(dirs, args.baseline, arms, args.model, tuple(args.metric or ()), - pass_k=args.pass_k, path_stability=args.path_stability) + pass_k=args.pass_k, path_stability=args.path_stability, tasks=args.task) except CompareError as exc: print(f"✗ {exc}", file=sys.stderr) sys.exit(1) @@ -546,6 +546,8 @@ def register(repo_sub, positive_int, source_help: str, cache_help: str) -> None: help="Treatment arm (ARM or BATCH/ARM); repeatable or comma-separated " "(default: every other arm)") compare_p.add_argument("--model", action="append", default=None, help="Only this model; repeatable") + compare_p.add_argument("--task", action="append", default=None, + help="Only this task id; repeatable (default: every task of the batches)") compare_p.add_argument("--metric", action="append", default=None, help="Also compare this numeric or boolean grade.json metric; repeatable") compare_p.add_argument("--pass-k", type=positive_int, default=None, dest="pass_k", metavar="K", diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index 510a5af5e..915c74c52 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -12,6 +12,9 @@ - **score**: the mean of ``score.metric`` for score-only tasks (a grader failure counts as 0); - **cost_usd**, **turns**: the per-cell means the harness reported; +- **first_prompt_tokens**: the mean of ``result.json`` ``agent.first_prompt_tokens`` (what the first + main-thread call was sent: start-up plus the task) over the cells that have it. Like a process count, it is + compared only when both arms report it for some shared task: a batch run before it was recorded has none; - **metric:**: the mean of ``grade.json`` ``metrics.`` over the cells that report it (``true`` is 1), for ``--metric``; - **pass^k**: for ``--pass-k K``, the chance that K cells drawn from the @@ -37,6 +40,18 @@ the gate and score comparisons of one invocation; cost, turns and metrics are descriptive and keep their raw p. +Three notes keep a row from being misread; none changes a number. A gate or score row of 2+ tasks whose +interval includes 0 is inconclusive, not "no effect": the note gives ``mde``, the smallest mean delta a +two-sided 5% test finds with 80% power at the row's spread, ``(z_0.975 + z_0.80) * sd / sqrt(n)`` over the +per-task deltas (in every row; ``None`` for one task or no spread). A row whose interval and p disagree (the +interval excludes 0 while p is 0.05 or more, or includes it while p is below) has ``interval_p_disagree``: +the two come from different methods, and for gate and score p is the one to read, since the percentile +bootstrap runs narrow over few tasks (the inconclusive note then gives way to this one). A descriptive row +with p below 0.05 is named on a line saying its p is uncorrected and outside Holm's family. + +``--task`` limits a comparison to some task ids: the others are left out, not listed as unpaired, and the +summary's ``task_filter`` records the selection. + A gate or score comparison also says whether its tasks had room to move (``headroom``), since a zero delta on a task set nobody can fail, or nobody can pass, is not evidence of no effect. A task's top is 1.0 for gate and the task's ``score.max`` for score (recorded in ``result.json``; a batch without it has no known @@ -79,6 +94,7 @@ import json import math import random +import statistics from collections import defaultdict from pathlib import Path from typing import Any @@ -98,6 +114,11 @@ EPS = 1e-9 #: Measures Holm's correction runs over. PRIMARY = ("gate", "score") +#: The level the notes read p and the interval at, and the normal quantiles of ``mde`` (two-sided 5%, 80% power). +ALPHA = 0.05 +Z_ALPHA, Z_POWER = 1.959964, 0.841621 +#: Descriptive measures compared only when both arms report them for some shared task (an older batch has none). +REPORTED = ("process:", "counter:", "first_prompt_tokens") #: Batch stamp keys that must agree between the arms of a comparison; ``tree`` is shown but may differ (another #: revision of the repository is a treatment of its own). STAMP_KEYS = ("eval_dir", "image", "graders") @@ -151,18 +172,20 @@ def batch_stamp(batch_dir: Path) -> dict[str, str]: def compare(batch_dirs: list[Path], baseline: str, treatments: list[str] | None = None, models: list[str] | None = None, metrics: tuple[str, ...] = (), seed: int = 0, - pass_k: int | None = None, path_stability: bool = False) -> dict[str, Any]: + pass_k: int | None = None, path_stability: bool = False, tasks: list[str] | None = None) -> dict[str, Any]: metrics = tuple(dict.fromkeys(metrics)) if pass_k is not None and pass_k < 1: raise CompareError(f"--pass-k must be at least 1, got {pass_k}") cells, missing = load_batches(batch_dirs, metrics, path_stability) + cells, missing, task_filter = _select_tasks(cells, missing, tasks) _check_metrics(metrics, cells) base_sel = parse_selector(baseline) if not any(selector_matches(base_sel, c) for c in cells): raise CompareError(f"no finished cell matches the baseline {baseline!r}") treatments = _treatments(cells, baseline, base_sel, treatments) measures = ["gate", "score", *([f"pass^{pass_k}"] if pass_k else []), - *(["path_distance"] if path_stability else []), "cost_usd", "turns", *(f"metric:{m}" for m in metrics)] + *(["path_distance"] if path_stability else []), "cost_usd", "turns", "first_prompt_tokens", + *(f"metric:{m}" for m in metrics)] base_vals = _task_values([c for c in cells if selector_matches(base_sel, c)], metrics, pass_k, path_stability) rows: list[dict[str, Any]] = [] unpaired: list[dict[str, Any]] = [] @@ -183,10 +206,25 @@ def compare(batch_dirs: list[Path], baseline: str, treatments: list[str] | None provenance = _provenance(batch_dirs, cells, baseline, treatments, models) provenance["warnings"] += one_arm return {"batches": [str(d) for d in batch_dirs], "baseline": baseline, "treatments": treatments, - "cells": len(cells), "missing": missing, **provenance, + "task_filter": task_filter, "cells": len(cells), "missing": missing, **provenance, "comparisons": rows, "unpaired": unpaired} +def _select_tasks(cells: list[dict[str, Any]], missing: list[str], + tasks: list[str] | None) -> tuple[list[dict[str, Any]], list[str], dict[str, Any] | None]: + """The cells and missing cells of ``tasks`` (``--task``), and the selection as the summary records it. + + No ``tasks`` keeps every cell and records ``None``. An id no finished cell of the batches has is refused.""" + if not tasks: + return cells, missing, None + known = {c["task"] for c in cells} + unknown = [t for t in dict.fromkeys(tasks) if t not in known] + if unknown: + raise CompareError(f"no finished cell of the compared batches has --task {', '.join(unknown)}") + kept = [m for m in missing if any(m.partition("/")[2].startswith(f"{t}__") for t in tasks)] # BATCH/__... + return [c for c in cells if c["task"] in tasks], kept, {"tasks": sorted(set(tasks)), "of": len(known)} + + def _one_arm_models(treatment: str, base_vals: dict[tuple[str, str], dict[str, Any]], treat_vals: dict[tuple[str, str], dict[str, Any]], models: list[str] | None) -> list[dict[str, Any]]: """The tasks of a model only one arm ran: it has no task to pair, so every task is unpaired.""" @@ -326,7 +364,7 @@ def _compare_model(treatment: str, model: str, base_vals: dict[tuple[str, str], rows: list[dict[str, Any]], unpaired: list[dict[str, Any]]) -> list[str]: """Append one row per measure of one (treatment, model), and the tasks only one arm has. - Returns the process measures only one arm reports for any shared task: they are skipped, not unpaired. + Returns the :data:`REPORTED` measures only one arm reports for any shared task: they are skipped, not unpaired. """ base_tasks = {t for m, t in base_vals if m == model} treat_tasks = {t for m, t in treat_vals if m == model} @@ -340,10 +378,10 @@ def _compare_model(treatment: str, model: str, base_vals: dict[tuple[str, str], unpaired.append(_unpaired(treatment, model, task, None, f"{b} task in baseline, {t} in treatment")) skipped = [] for measure in measures: - if measure.startswith(("process:", "counter:")): + if measure.startswith(REPORTED): has = [any(vals[(model, t)]["values"].get(measure) is not None for t in shared) for vals in (base_vals, treat_vals)] - if not all(has): # an arm that reports no such count at all (an older batch) leaves no task out of a pair + if not all(has): # an arm that reports none of it at all (an older batch) leaves no task out of a pair if any(has): skipped.append(measure) continue @@ -484,17 +522,22 @@ def paired_summary(pairs: list[dict[str, Any]], measure: str = "gate", seed: int permutation test. The interval is a percentile bootstrap for the bounded gate and score shares and a t interval (:func:`~dokimi.eval.stats.mean_ci`) for the continuous measures, where the bootstrap under-covers at these task counts; one task has no interval (``lo``/``hi`` None), not a zero-width one. + ``mde`` and ``interval_p_disagree`` are the reading aids of the module docstring. """ deltas = [p["delta"] for p in pairs] ci = (bootstrap_ci if measure in PRIMARY else mean_ci)(deltas, seed=seed) lo, hi = (float(ci["lo"]), float(ci["hi"])) if len(deltas) > 1 else (None, None) # mean_ci gives numpy floats + p = sign_flip_p(deltas, seed=seed) + sd = statistics.stdev(deltas) if len(deltas) > 1 else 0.0 return { "n_tasks": len(pairs), "baseline_mean": _round(_mean([p["baseline"] for p in pairs])), "treatment_mean": _round(_mean([p["treatment"] for p in pairs])), "delta": {"mean": ci["mean"], "lo": lo, "hi": hi}, - "p": sign_flip_p(deltas, seed=seed), + "p": p, "p_holm": None, + "mde": _round(min_detectable(sd, len(deltas))) if sd > EPS else None, + "interval_p_disagree": lo is not None and hi is not None and p is not None and (lo <= 0 <= hi) == (p < ALPHA), "up": sum(d > EPS for d in deltas), "down": sum(d < -EPS for d in deltas), "tied": sum(abs(d) <= EPS for d in deltas), @@ -503,6 +546,11 @@ def paired_summary(pairs: list[dict[str, Any]], measure: str = "gate", seed: int } +def min_detectable(sd: float, n: int) -> float: + """The smallest mean delta a two-sided test at :data:`ALPHA` finds with 80% power over ``n`` tasks of spread ``sd``.""" + return (Z_ALPHA + Z_POWER) * sd / math.sqrt(n) + + def sign_flip_p(deltas: list[float], seed: int = 0, n_resamples: int = N_RESAMPLES) -> float | None: """Two-sided p of ``|sum(deltas)|`` when each delta's sign is a fair coin under the null. @@ -545,6 +593,7 @@ def format_compare(summary: dict[str, Any]) -> str: f"{t['task']} {t['delta']:+.2f}" for t in sorted(moved, key=lambda t: t["delta"]))) if r["headroom"]: lines.append(f" {r['measure']} {_headroom_note(r)}") + lines += _reading_notes(rows) if summary["unpaired"]: shown = summary["unpaired"][:10] lines.append("unpaired: " + ", ".join( @@ -558,6 +607,27 @@ def format_compare(summary: dict[str, Any]) -> str: return "\n".join(lines) +def _reading_notes(rows: list[dict[str, Any]]) -> list[str]: + """One comparison's notes against a misreading (module docstring); no number changes.""" + out = [] + for r in rows: + d, primary = r["delta"], r["measure"] in PRIMARY + if r["interval_p_disagree"]: + inside = d["lo"] <= 0 <= d["hi"] + how = "a percentile bootstrap, too narrow over few tasks," if primary else "a t interval" + out.append(f" {r['measure']}: the interval [{d['lo']:+.3f}, {d['hi']:+.3f}] " + f"{'includes' if inside else 'excludes'} 0 but p {r['p']:.4g} is {'<' if inside else '≥'} " + f"{ALPHA}; they come from two methods, {how} and a sign-flip test" + ("; read p" if primary else "")) + elif primary and r["mde"] is not None and d["lo"] <= 0 <= d["hi"]: + out.append(f" {r['measure']} inconclusive: the interval includes 0; with this spread the smallest effect " + f"80% power detects is about ±{r['mde']:.3f} ({r['n_tasks']} tasks)") + low = [r["measure"] for r in rows if r["measure"] not in PRIMARY and r["p"] is not None and r["p"] < ALPHA] + if low: + out.append(f" p < {ALPHA} on {', '.join(low)}: descriptive rows, their p uncorrected and outside Holm's " + "family, which covers gate and score only") + return out + + def _headroom_note(row: dict[str, Any]) -> str: n, gate = row["n_tasks"], row["measure"] == "gate" if row["headroom"] == "ceiling": @@ -572,6 +642,8 @@ def _headroom_note(row: dict[str, Any]) -> str: def _format_header(summary: dict[str, Any]) -> list[str]: lines = [f"baseline {summary['baseline']}: {summary['cells']} finished cells in " f"{len(summary['batches'])} batch(es), {len(summary['missing'])} missing"] + if summary.get("task_filter"): + lines.append(f" tasks: {len(summary['task_filter']['tasks'])} of {summary['task_filter']['of']} (--task)") for text, batches in summary["sources"].items(): lines.append(f" {text}: {', '.join(batches)}") for name, stamp in summary["stamps"].items(): @@ -591,6 +663,7 @@ def _task_values(cells: list[dict[str, Any]], metrics: tuple[str, ...], pass_k: score_task = any(r["score_task"] for r in recs) found = {"cost_usd": [r["agent"].get("cost_usd") for r in recs], "turns": [r["agent"].get("num_turns") for r in recs], + "first_prompt_tokens": [r["agent"].get("first_prompt_tokens") for r in recs], **{f"metric:{name}": [r["metrics"].get(name) for r in recs] for name in metrics}} found.update(_process_lists(recs)) if score_task: diff --git a/dokimi/dokimi/repos/events.py b/dokimi/dokimi/repos/events.py index c9375866d..c86154b7d 100644 --- a/dokimi/dokimi/repos/events.py +++ b/dokimi/dokimi/repos/events.py @@ -20,7 +20,9 @@ Token counts cover the same scope as ``cost_usd``: summed over ``modelUsage`` (parent, subagents and helper models) when the result has it, else the parent's -``usage``; ``usage_scope`` says which (``tree`` or ``main``). +``usage``; ``usage_scope`` says which (``tree`` or ``main``). ``first_prompt_tokens`` is +the prompt of the first main-thread model call, everything start-up loaded plus the +task, read as :mod:`dokimi.context.trace` reads it (one call per message id). """ from __future__ import annotations @@ -32,6 +34,8 @@ from pathlib import PurePosixPath, PureWindowsPath from typing import Any +from dokimi.context.trace import parse_trace + TRACE_FORMATS = ("claude-stream-json", "text") OUTPUT_LIMIT = 8192 OUTPUT_KEEP = 4096 @@ -56,6 +60,7 @@ class Transcript: cache_read_tokens: int | None = None cache_write_tokens: int | None = None usage_scope: str | None = None # "tree": tokens include subagents and helper models; "main": parent only + first_prompt_tokens: int | None = None # the first main-thread call's prompt; None for a format without usage subagent_calls: int | None = None # counted once the harness's result record is seen is_error: bool = False harness_version: str | None = None @@ -112,6 +117,8 @@ def _claude_stream_json(raw: str, workdir: str | None) -> Transcript: _apply(out, rec, t, by_id, workdir) except (AttributeError, TypeError) as exc: # valid JSON of an unexpected shape out.errors.append(f"line {n}: unexpected shape ({exc})") + main = parse_trace(raw).main_calls() + out.first_prompt_tokens = main[0]["prompt_tokens"] if main else None return out diff --git a/dokimi/dokimi/repos/runner.py b/dokimi/dokimi/repos/runner.py index 594dbbe5d..1677aaa45 100644 --- a/dokimi/dokimi/repos/runner.py +++ b/dokimi/dokimi/repos/runner.py @@ -277,7 +277,7 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, # A killed harness reports no cost; the batch budget counts the cell's cap instead of $0. "charged_usd": caps["budget_usd"], "num_turns": None, "is_error": None, "result_subtype": None, "input_tokens": None, "output_tokens": None, "cache_read_tokens": None, - "cache_write_tokens": None, "usage_scope": None, "subagent_calls": None, + "cache_write_tokens": None, "usage_scope": None, "first_prompt_tokens": None, "subagent_calls": None, "plugins": [], "over_budget": False, "caps": caps, "network": network, "trace_errors": []} process: dict[str, Any] | None = None # null when the trace could not be read added = meta.get("arm_patch") or {} @@ -301,6 +301,7 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, result_subtype=transcript.result_subtype, input_tokens=transcript.input_tokens, output_tokens=transcript.output_tokens, cache_read_tokens=transcript.cache_read_tokens, cache_write_tokens=transcript.cache_write_tokens, usage_scope=transcript.usage_scope, + first_prompt_tokens=transcript.first_prompt_tokens, subagent_calls=transcript.subagent_calls, plugins=transcript.plugins, over_budget=cost is not None and cost > caps["budget_usd"], trace_errors=transcript.errors[:5]) if events is not None: # after the cost is recorded: a metric that fails must not cost the cell its record diff --git a/dokimi/tests/repo_eval_fixture.py b/dokimi/tests/repo_eval_fixture.py index 8682e3e49..5f0e7038d 100644 --- a/dokimi/tests/repo_eval_fixture.py +++ b/dokimi/tests/repo_eval_fixture.py @@ -128,7 +128,8 @@ "usage": {"input_tokens": 5, "output_tokens": 2, "cache_read_input_tokens": 100, "cache_creation_input_tokens": 40}} if os.environ.get("FAKE_USAGE"): # a subagent (Agent tool) and the token counts of the whole tree - lines.append({"type": "assistant", "message": {"content": [ + lines.append({"type": "assistant", "message": {"id": "msg_1", "usage": { + "input_tokens": 3, "cache_creation_input_tokens": 1200, "cache_read_input_tokens": 15000}, "content": [ {"type": "tool_use", "id": "t2", "name": "Agent", "input": {"prompt": "look around"}}]}}) result.update(usage={"input_tokens": 7, "output_tokens": 2}, modelUsage={"m-main": {"inputTokens": 7, "outputTokens": 2, "cacheReadInputTokens": 100}, diff --git a/dokimi/tests/test_repo_compare_reading.py b/dokimi/tests/test_repo_compare_reading.py new file mode 100644 index 000000000..a99f993ef --- /dev/null +++ b/dokimi/tests/test_repo_compare_reading.py @@ -0,0 +1,157 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""`repo compare`: the start context of each arm, `--task`, and the notes that keep a row from being misread.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from dokimi.cli import build_parser +from dokimi.repos.compare import CompareError, compare, format_compare, min_detectable + +from test_repo_compare import _cell, _row + + +def _first(batch: Path, task: str, arm: str, rep: int, tokens: int | None) -> None: + """A finished gate cell whose ``agent`` records ``first_prompt_tokens`` (left out for ``None``: an older batch).""" + _cell(batch, task, arm, rep) + path = batch / f"{task}__{arm.replace(':', '-')}__m-1__r{rep}" / "result.json" + rec = json.loads(path.read_text()) + if tokens is not None: + rec["agent"]["first_prompt_tokens"] = tokens + path.write_text(json.dumps(rec)) + + +def test_first_prompt_tokens_is_compared_per_task_over_the_cells_that_have_it(tmp_path): + b = tmp_path / "b" + for task, base, treat in (("t1", (16_310, 16_320), (19_370, 19_390)), ("t2", (16_300, None), (19_400, 19_380))): + for rep, (x, y) in enumerate(zip(base, treat, strict=True), 1): + _first(b, task, "guidance:none", rep, x) + _first(b, task, "guidance:raw", rep, y) + row = _row(compare([b], "guidance:none"), "first_prompt_tokens") + by_task = {t["task"]: t for t in row["tasks"]} + assert by_task["t1"]["baseline"] == 16_315 and by_task["t1"]["treatment"] == 19_380 + assert by_task["t2"]["baseline"] == 16_300 and by_task["t2"]["n_baseline"] == 1 # the cell without it is left out + assert row["delta"]["mean"] == pytest.approx((3_065 + 3_090) / 2) and row["p_holm"] is None + + +def test_first_prompt_tokens_of_an_older_batch_is_no_row_not_zero(tmp_path): + old, mixed = tmp_path / "old", tmp_path / "mixed" + for rep in (1, 2): + _first(old, "t1", "guidance:none", rep, None) + _first(old, "t1", "guidance:raw", rep, None) + _first(mixed, "t1", "guidance:none", rep, None) + _first(mixed, "t1", "guidance:raw", rep, 19_380) + s = compare([old], "guidance:none") + assert "first_prompt_tokens" not in {r["measure"] for r in s["comparisons"]} + assert not s["unpaired"] and not s["warnings"] + s = compare([mixed], "guidance:none") + assert "first_prompt_tokens" not in {r["measure"] for r in s["comparisons"]} and not s["unpaired"] + assert s["warnings"] == ["guidance:raw m-1: only one arm reports first_prompt_tokens for any shared task; " + "not compared"] + + +def _three_tasks(b: Path) -> None: + for task, base_ok in (("t1", False), ("t2", True), ("t3", False)): + _cell(b, task, "guidance:none", 1, success=base_ok) + _cell(b, task, "guidance:raw", 1) + _cell(b, "t4", "guidance:none", 1) # only one arm ran it: unpaired, unless --task leaves it out + (b / "t4__guidance-raw__m-1__r1").mkdir() + (b / "t4__guidance-raw__m-1__r1" / "meta.json").write_text("{}") # started, never finished: missing + + +def test_task_compares_only_the_named_tasks_and_says_so(tmp_path): + b = tmp_path / "b" + _three_tasks(b) + whole = compare([b], "guidance:none") + assert whole["task_filter"] is None and _row(whole, "gate")["n_tasks"] == 3 + assert [u["task"] for u in whole["unpaired"]] == ["t4"] and len(whole["missing"]) == 1 + s = compare([b], "guidance:none", tasks=["t1", "t2"]) + gate = _row(s, "gate") + assert [t["task"] for t in gate["tasks"]] == ["t1", "t2"] and gate["up"] == 1 + assert s["unpaired"] == [] and s["missing"] == [] and s["cells"] == 4 + assert s["task_filter"] == {"tasks": ["t1", "t2"], "of": 4} + assert " tasks: 2 of 4 (--task)" in format_compare(s).splitlines() + assert "(--task)" not in format_compare(whole) + + +def test_cli_task_is_repeatable_and_lands_in_the_json(tmp_path, capsys): + b = tmp_path / "b" + _three_tasks(b) + args = build_parser().parse_args(["repo", "compare", str(b), "--baseline", "guidance:none", + "--task", "t3", "--task", "t4", "--json"]) + args.func(args) + out = json.loads(capsys.readouterr().out) + assert out["task_filter"] == {"tasks": ["t3", "t4"], "of": 4} + assert [t["task"] for t in _row(out, "gate")["tasks"]] == ["t3"] + assert [u["task"] for u in out["unpaired"]] == ["t4"] and len(out["missing"]) == 1 + + +def test_task_refuses_an_id_no_cell_has(tmp_path, capsys): + b = tmp_path / "b" + _three_tasks(b) + with pytest.raises(CompareError, match=r"has --task t9, fix_x$"): + compare([b], "guidance:none", tasks=["t1", "t9", "fix_x"]) + args = build_parser().parse_args(["repo", "compare", str(b), "--baseline", "guidance:none", "--task", "t9"]) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 1 and "no finished cell of the compared batches has --task t9" in capsys.readouterr().err + + +def test_min_detectable_effect_arithmetic(): + # The recall row that read as "no gain": SD 0.222 of the per-task deltas over 26 tasks + assert min_detectable(0.222, 26) == pytest.approx(0.122, abs=5e-4) + assert min_detectable(0.222, 26) == pytest.approx(min_detectable(0.444, 104)) # half the spread or 4x the tasks + + +def _score_tasks(b: Path, deltas: list[float], base: float) -> None: + for i, d in enumerate(deltas): + _cell(b, f"review_{i}", "guidance:none", 1, score=base) + _cell(b, f"review_{i}", "guidance:raw", 1, score=base + d) + + +def test_an_interval_across_zero_is_inconclusive_with_its_mde(tmp_path): + _score_tasks(tmp_path / "b", [0.5, -0.5, 0.25, -0.25], 0.5) # sample SD 0.456435 + s = compare([tmp_path / "b"], "guidance:none") + score = _row(s, "score") + assert score["mde"] == pytest.approx(0.639371, abs=1e-6) and score["interval_p_disagree"] is False + text = format_compare(s) + assert (" score inconclusive: the interval includes 0; with this spread the smallest effect 80% power " + "detects is about ±0.639 (4 tasks)") in text.splitlines() + assert "no effect" not in text + _score_tasks(tmp_path / "one", [0.5], 0.5) + one = compare([tmp_path / "one"], "guidance:none") + assert _row(one, "score")["mde"] is None and "inconclusive" not in format_compare(one) + + +def test_a_bootstrap_interval_that_excludes_zero_against_p_025_is_flagged(tmp_path): + _score_tasks(tmp_path / "b", [0.0, 0.0, 0.333, 0.167, 0.0, 0.333], 0.0) + s = compare([tmp_path / "b"], "guidance:none") + score = _row(s, "score") + assert score["delta"]["lo"] > 0 and score["p"] == 0.25 # three non-zero deltas, all positive: p cannot go lower + assert score["interval_p_disagree"] is True + notes = [line for line in format_compare(s).splitlines() if line.startswith(" score")] + assert notes[-1].startswith(f" score: the interval [{score['delta']['lo']:+.3f}, +0.250] excludes 0 but p 0.25 " + "is ≥ 0.05; they come from two methods, a percentile bootstrap") + assert notes[-1].endswith("; read p") and not any("inconclusive" in line for line in notes) + + +def test_a_descriptive_p_below_005_is_named_as_uncorrected(tmp_path): + b = tmp_path / "b" + for i, turns in enumerate([11, 11, 11, 11, 11, 30]): # every task up, one far up: sign-flip p 2/64 + _cell(b, f"fix_{i}", "guidance:none", 1, turns=10) + _cell(b, f"fix_{i}", "guidance:raw", 1, turns=turns) + s = compare([b], "guidance:none") + turns = _row(s, "turns") + assert turns["p"] == pytest.approx(2 / 64) and turns["delta"]["lo"] < 0 < turns["delta"]["hi"] + assert turns["interval_p_disagree"] is True and _row(s, "gate")["interval_p_disagree"] is False + lines = format_compare(s).splitlines() + disagree = [line for line in lines if line.startswith(" turns: the interval")] + assert len(disagree) == 1 and "includes 0 but p 0.03125 is < 0.05" in disagree[0] + assert "a t interval and a sign-flip test" in disagree[0] and "read p" not in disagree[0] + assert (" p < 0.05 on turns: descriptive rows, their p uncorrected and outside Holm's family, which covers " + "gate and score only") in lines diff --git a/dokimi/tests/test_repo_reach.py b/dokimi/tests/test_repo_reach.py index 3e774169c..373926a78 100644 --- a/dokimi/tests/test_repo_reach.py +++ b/dokimi/tests/test_repo_reach.py @@ -450,6 +450,24 @@ def test_tokens_include_task_tool_and_cache_fields(): assert t.usage_scope == "main" and t.subagent_calls == 1 +def test_first_prompt_tokens_is_the_first_main_thread_call_counted_once(): + def msg(mid, usage, parent=None): + return {"type": "assistant", "parent_tool_use_id": parent, + "message": {"id": mid, "usage": usage, "content": [{"type": "text", "text": "x"}]}} + lines = [ # a subagent's call comes first; the main call's two content blocks share one message id, + # and its first block's usage is sparse: the call's prompt is the merge of both, not the first line's + msg("s1", {"input_tokens": 9, "cache_read_input_tokens": 50_000}, parent="t1"), + msg("m1", {"input_tokens": 4, "cache_read_input_tokens": 16_315}), + msg("m1", {"input_tokens": 4, "cache_creation_input_tokens": 3_061, "cache_read_input_tokens": 16_315, + "output_tokens": 80}), + msg("m2", {"input_tokens": 1, "cache_read_input_tokens": 19_500}), + ] + assert convert("claude-stream-json", "\n".join(json.dumps(x) for x in lines)).first_prompt_tokens == 19_380 + no_usage = {"type": "assistant", "message": {"content": [{"type": "text", "text": "hi"}]}} + assert convert("claude-stream-json", json.dumps(no_usage)).first_prompt_tokens is None + assert convert("text", "hi").first_prompt_tokens is None + + @pytest.mark.parametrize("line", [ {"type": "assistant", "message": "hi"}, {"type": "result", "result": "x", "usage": "n/a"}, diff --git a/dokimi/tests/test_repo_run.py b/dokimi/tests/test_repo_run.py index bc214de79..68c18d2cb 100644 --- a/dokimi/tests/test_repo_run.py +++ b/dokimi/tests/test_repo_run.py @@ -401,7 +401,8 @@ def test_agent_timeout_is_killed_and_still_graded(setup, monkeypatch, capsys): rec = json.loads((setup["out"] / "b1/fix_add__guidance-raw__m-1__r1/result.json").read_text()) assert rec["agent"]["timed_out"] is True and rec["agent"]["rc"] is None and rec["agent"]["wall_s"] < 30 assert rec["grade"]["status"] == "graded" and rec["grade"]["success"] is False # the bug is still there - unmeasured = {k: rec["agent"][k] for k in ("input_tokens", "cache_read_tokens", "usage_scope", "subagent_calls")} + unmeasured = {k: rec["agent"][k] for k in ("input_tokens", "cache_read_tokens", "usage_scope", "subagent_calls", + "first_prompt_tokens")} assert unmeasured == {k: None for k in unmeasured} @@ -703,6 +704,7 @@ def test_result_json_carries_the_token_fields_of_the_whole_tree(setup, monkeypat assert (agent["input_tokens"], agent["output_tokens"], agent["cache_read_tokens"], agent["cache_write_tokens"]) == (37, 11, 100, 5) assert agent["usage_scope"] == "tree" and agent["subagent_calls"] == 1 + assert agent["first_prompt_tokens"] == 3 + 1200 + 15000 # the first main-thread call, not the run's total def test_a_host_error_after_the_agent_is_a_charged_grader_failure(setup, monkeypatch, capsys): @@ -729,7 +731,7 @@ def boom(*a, **kw): rec = _only_cell(setup) assert rec["grade"]["status"] == "grader_failure" and "unreadable trace" in rec["grade"]["reasons"][0] assert rec["agent"]["cost_usd"] is None and rec["agent"]["charged_usd"] == 0.5 - assert all(rec["agent"][k] is None for k in ("input_tokens", "usage_scope", "subagent_calls")) + assert all(rec["agent"][k] is None for k in ("input_tokens", "usage_scope", "subagent_calls", "first_prompt_tokens")) assert "1 ran" in (out := capsys.readouterr().out) and "$0.5000" in out From 00f9b403f9561e43ea42495eda49439fc2c58892 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Thu, 29 Oct 2026 19:09:44 +0000 Subject: [PATCH 02/16] test(repo): pin the one-arm skip both ways and the usage merge Review comment 4415626666: the one-arm skip of first_prompt_tokens was tested only with the baseline lacking it. The test now also compares a batch whose treatment lacks it and asserts the same warning and no unpaired entries, so a skip that checks only the baseline arm fails. Review comment 4416413441: the main call's message id now has a third, sparser usage line after the full one, so the first prompt is 19_380 only when the lines merge by maximum; "last line wins" gives 16_319 and fails the test, as "first line wins" already did. Signed-off-by: Anton Volkov --- dokimi/tests/test_repo_compare_reading.py | 13 ++++++++----- dokimi/tests/test_repo_reach.py | 5 +++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/dokimi/tests/test_repo_compare_reading.py b/dokimi/tests/test_repo_compare_reading.py index a99f993ef..7933e98b6 100644 --- a/dokimi/tests/test_repo_compare_reading.py +++ b/dokimi/tests/test_repo_compare_reading.py @@ -40,19 +40,22 @@ def test_first_prompt_tokens_is_compared_per_task_over_the_cells_that_have_it(tm def test_first_prompt_tokens_of_an_older_batch_is_no_row_not_zero(tmp_path): - old, mixed = tmp_path / "old", tmp_path / "mixed" + old, mixed, reverse = tmp_path / "old", tmp_path / "mixed", tmp_path / "reverse" for rep in (1, 2): _first(old, "t1", "guidance:none", rep, None) _first(old, "t1", "guidance:raw", rep, None) _first(mixed, "t1", "guidance:none", rep, None) _first(mixed, "t1", "guidance:raw", rep, 19_380) + _first(reverse, "t1", "guidance:none", rep, 16_315) + _first(reverse, "t1", "guidance:raw", rep, None) s = compare([old], "guidance:none") assert "first_prompt_tokens" not in {r["measure"] for r in s["comparisons"]} assert not s["unpaired"] and not s["warnings"] - s = compare([mixed], "guidance:none") - assert "first_prompt_tokens" not in {r["measure"] for r in s["comparisons"]} and not s["unpaired"] - assert s["warnings"] == ["guidance:raw m-1: only one arm reports first_prompt_tokens for any shared task; " - "not compared"] + for batch in (mixed, reverse): # the arm without it is the baseline, then the treatment + s = compare([batch], "guidance:none") + assert "first_prompt_tokens" not in {r["measure"] for r in s["comparisons"]} and not s["unpaired"] + assert s["warnings"] == ["guidance:raw m-1: only one arm reports first_prompt_tokens for any shared task; " + "not compared"] def _three_tasks(b: Path) -> None: diff --git a/dokimi/tests/test_repo_reach.py b/dokimi/tests/test_repo_reach.py index a8083547c..bea0759fa 100644 --- a/dokimi/tests/test_repo_reach.py +++ b/dokimi/tests/test_repo_reach.py @@ -454,12 +454,13 @@ def test_first_prompt_tokens_is_the_first_main_thread_call_counted_once(): def msg(mid, usage, parent=None): return {"type": "assistant", "parent_tool_use_id": parent, "message": {"id": mid, "usage": usage, "content": [{"type": "text", "text": "x"}]}} - lines = [ # a subagent's call comes first; the main call's two content blocks share one message id, - # and its first block's usage is sparse: the call's prompt is the merge of both, not the first line's + lines = [ # a subagent's call comes first; the main call's three content blocks share one message id, + # and its first and last blocks' usage is sparse: the call's prompt is the merge, not the first or last line's msg("s1", {"input_tokens": 9, "cache_read_input_tokens": 50_000}, parent="t1"), msg("m1", {"input_tokens": 4, "cache_read_input_tokens": 16_315}), msg("m1", {"input_tokens": 4, "cache_creation_input_tokens": 3_061, "cache_read_input_tokens": 16_315, "output_tokens": 80}), + msg("m1", {"input_tokens": 4, "cache_read_input_tokens": 16_315, "output_tokens": 80}), msg("m2", {"input_tokens": 1, "cache_read_input_tokens": 19_500}), ] assert convert("claude-stream-json", "\n".join(json.dumps(x) for x in lines)).first_prompt_tokens == 19_380 From 0c7d61f6e3ea8b42b89ff233c43ef31d54589800 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 19:29:46 -0700 Subject: [PATCH 03/16] fix(repo): compare's mde uses the t quantiles of its interval The inconclusive note gave mde with normal quantiles (1.96 + 0.84), but the interval beside it is a t interval at n-1 degrees of freedom. At 4 tasks the note said +-0.639 where the t quantiles give +-0.950; at 26 the gap is small. The contract's formula is updated to match. Signed-off-by: Nikolay Petrov --- dokimi/docs/repo-evaluation-contract.md | 5 +++-- dokimi/dokimi/repos/compare.py | 16 ++++++++++------ dokimi/tests/test_repo_compare_reading.py | 16 +++++++++++----- 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 9b2dd1c08..72bbc43d5 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1335,8 +1335,9 @@ python cli.py repo compare repo-runs/replay --baseline guidance:none --metric re misread, and change no number. A gate or score row of 2+ tasks whose interval includes 0 is inconclusive, not evidence of no effect: the note gives `mde`, the smallest mean delta a two-sided 5% test finds with 80% power - at that spread, `(1.959964 + 0.841621) * sd / sqrt(n)` over the per-task - deltas (every row has `mde` in `--json`; `null` for one task or deltas with + at that spread, `(t(0.975, n-1) + t(0.80, n-1)) * sd / sqrt(n)` over the + per-task deltas, the t quantiles at `n-1` degrees of freedom (every row has + `mde` in `--json`; `null` for one task or deltas with no spread, and then no note). A row whose interval and p disagree (the interval excludes 0 while p is 0.05 or more, or includes 0 while p is below 0.05) has `interval_p_disagree` `true` and a note: they come from two diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index 3001003be..bc8c27374 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -42,8 +42,8 @@ Three notes keep a row from being misread; none changes a number. A gate or score row of 2+ tasks whose interval includes 0 is inconclusive, not "no effect": the note gives ``mde``, the smallest mean delta a -two-sided 5% test finds with 80% power at the row's spread, ``(z_0.975 + z_0.80) * sd / sqrt(n)`` over the -per-task deltas (in every row; ``None`` for one task or no spread). A row whose interval and p disagree (the +two-sided 5% test finds with 80% power at the row's spread, ``(t_0.975 + t_0.80) * sd / sqrt(n)`` over the +per-task deltas, the t quantiles at ``n - 1`` degrees of freedom (in every row; ``None`` for one task or no spread). A row whose interval and p disagree (the interval excludes 0 while p is 0.05 or more, or includes it while p is below) has ``interval_p_disagree``: the two come from different methods, and for gate and score p is the one to read, since the percentile bootstrap runs narrow over few tasks (the inconclusive note then gives way to this one). A descriptive row @@ -99,6 +99,8 @@ from pathlib import Path from typing import Any +from scipy import stats as sp_stats + from dokimi.context.consistency import Run, order_distance, pass_hat, run_from_events from dokimi.eval.stats import adjust_pvalues, bootstrap_ci, mean_ci @@ -114,9 +116,9 @@ EPS = 1e-9 #: Measures Holm's correction runs over. PRIMARY = ("gate", "score") -#: The level the notes read p and the interval at, and the normal quantiles of ``mde`` (two-sided 5%, 80% power). +#: The level the notes read p and the interval at (two-sided), and the power ``mde`` is sized for. ALPHA = 0.05 -Z_ALPHA, Z_POWER = 1.959964, 0.841621 +POWER = 0.80 #: Descriptive measures compared only when both arms report them for some shared task (an older batch has none). REPORTED = ("process:", "counter:", "first_prompt_tokens") #: Batch stamp keys that must agree between the arms of a comparison; ``tree`` is shown but may differ (another @@ -547,8 +549,10 @@ def paired_summary(pairs: list[dict[str, Any]], measure: str = "gate", seed: int def min_detectable(sd: float, n: int) -> float: - """The smallest mean delta a two-sided test at :data:`ALPHA` finds with 80% power over ``n`` tasks of spread ``sd``.""" - return (Z_ALPHA + Z_POWER) * sd / math.sqrt(n) + """The smallest mean delta a two-sided test at :data:`ALPHA` finds with :data:`POWER` over ``n`` tasks of + spread ``sd``. The quantiles are t at ``n - 1`` degrees of freedom, as the interval of ``mean_ci`` is.""" + df = n - 1 + return (sp_stats.t.ppf(1 - ALPHA / 2, df) + sp_stats.t.ppf(POWER, df)) * sd / math.sqrt(n) def sign_flip_p(deltas: list[float], seed: int = 0, n_resamples: int = N_RESAMPLES) -> float | None: diff --git a/dokimi/tests/test_repo_compare_reading.py b/dokimi/tests/test_repo_compare_reading.py index 7933e98b6..622be90d4 100644 --- a/dokimi/tests/test_repo_compare_reading.py +++ b/dokimi/tests/test_repo_compare_reading.py @@ -106,9 +106,15 @@ def test_task_refuses_an_id_no_cell_has(tmp_path, capsys): def test_min_detectable_effect_arithmetic(): - # The recall row that read as "no gain": SD 0.222 of the per-task deltas over 26 tasks - assert min_detectable(0.222, 26) == pytest.approx(0.122, abs=5e-4) - assert min_detectable(0.222, 26) == pytest.approx(min_detectable(0.444, 104)) # half the spread or 4x the tasks + # The recall row that read as "no gain": SD 0.222 of the per-task deltas over 26 tasks, t quantiles at df 25 + assert min_detectable(0.222, 26) == pytest.approx(0.126946, abs=1e-6) + # half the spread or 4x the tasks: about the same figure, the t quantiles shrink a little with the tasks + assert min_detectable(0.222, 26) == pytest.approx(min_detectable(0.444, 104), rel=0.05) + + +def test_min_detectable_uses_the_t_quantiles_of_the_interval_at_small_n(): + # four tasks: the interval is a t interval at df 3 (t(0.975, 3) = 3.182, t(0.80, 3) = 0.978), not normal ones + assert min_detectable(0.456435, 4) == pytest.approx(0.949594, abs=1e-6) def _score_tasks(b: Path, deltas: list[float], base: float) -> None: @@ -121,10 +127,10 @@ def test_an_interval_across_zero_is_inconclusive_with_its_mde(tmp_path): _score_tasks(tmp_path / "b", [0.5, -0.5, 0.25, -0.25], 0.5) # sample SD 0.456435 s = compare([tmp_path / "b"], "guidance:none") score = _row(s, "score") - assert score["mde"] == pytest.approx(0.639371, abs=1e-6) and score["interval_p_disagree"] is False + assert score["mde"] == pytest.approx(0.949594, abs=1e-6) and score["interval_p_disagree"] is False text = format_compare(s) assert (" score inconclusive: the interval includes 0; with this spread the smallest effect 80% power " - "detects is about ±0.639 (4 tasks)") in text.splitlines() + "detects is about ±0.950 (4 tasks)") in text.splitlines() assert "no effect" not in text _score_tasks(tmp_path / "one", [0.5], 0.5) one = compare([tmp_path / "one"], "guidance:none") From 349d5443c79abbb7a3c71c11b9a6df024a33a499 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 19:30:44 -0700 Subject: [PATCH 04/16] fix(repo): compare --task matches a missing cell by its task, not a prefix A task id may contain '__', so a missing cell of fix__a starts with fix__ and was counted as a cell of fix. The task is now split off the cell name from the right, as the arm, model and rep are the last three parts. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/compare.py | 12 +++++++++++- dokimi/tests/test_repo_compare_reading.py | 15 +++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index bc8c27374..68c0c29db 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -223,10 +223,20 @@ def _select_tasks(cells: list[dict[str, Any]], missing: list[str], unknown = [t for t in dict.fromkeys(tasks) if t not in known] if unknown: raise CompareError(f"no finished cell of the compared batches has --task {', '.join(unknown)}") - kept = [m for m in missing if any(m.partition("/")[2].startswith(f"{t}__") for t in tasks)] # BATCH/__... + kept = [m for m in missing if _cell_task(m.partition("/")[2]) in tasks] # BATCH/ return [c for c in cells if c["task"] in tasks], kept, {"tasks": sorted(set(tasks)), "of": len(known)} +def _cell_task(cell: str) -> str | None: + """The task of a cell name ``______r`` (:func:`~dokimi.repos.runner.cell_name`). + + A task id may itself contain ``__``, so the three parts after it are split off from the right; the arm and + model are slugs, which the split assumes carry no ``__``. ``None`` for a name of another shape. + """ + parts = cell.rsplit("__", 3) + return parts[0] if len(parts) == 4 else None + + def _one_arm_models(treatment: str, base_vals: dict[tuple[str, str], dict[str, Any]], treat_vals: dict[tuple[str, str], dict[str, Any]], models: list[str] | None) -> list[dict[str, Any]]: """The tasks of a model only one arm ran: it has no task to pair, so every task is unpaired.""" diff --git a/dokimi/tests/test_repo_compare_reading.py b/dokimi/tests/test_repo_compare_reading.py index 622be90d4..816701c67 100644 --- a/dokimi/tests/test_repo_compare_reading.py +++ b/dokimi/tests/test_repo_compare_reading.py @@ -94,6 +94,21 @@ def test_cli_task_is_repeatable_and_lands_in_the_json(tmp_path, capsys): assert [u["task"] for u in out["unpaired"]] == ["t4"] and len(out["missing"]) == 1 +def test_task_matches_a_missing_cell_by_its_task_not_by_a_prefix(tmp_path): + # a task id may contain "__": the unfinished cell of fix__a is not a cell of fix + b = tmp_path / "b" + for task in ("fix", "fix__a"): + _cell(b, task, "guidance:none", 1) + _cell(b, task, "guidance:raw", 1) + (b / "fix__a__guidance-raw__m-1__r2").mkdir() + (b / "fix__a__guidance-raw__m-1__r2" / "meta.json").write_text("{}") # started, never finished: missing + s = compare([b], "guidance:none", tasks=["fix"]) + assert s["missing"] == [] and s["task_filter"] == {"tasks": ["fix"], "of": 2} + assert " tasks: 1 of 2 (--task)" in format_compare(s).splitlines() + assert "cells missing" not in format_compare(s) + assert compare([b], "guidance:none", tasks=["fix__a"])["missing"] == ["b/fix__a__guidance-raw__m-1__r2"] + + def test_task_refuses_an_id_no_cell_has(tmp_path, capsys): b = tmp_path / "b" _three_tasks(b) From d5f2d7c37d9e2f2ce42ecff3e3196d97890fa1e5 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 19:32:08 -0700 Subject: [PATCH 05/16] fix(repo): compare's arm and metric errors say when --task is the reason With --task the arm and metric checks run on the selected cells, so an arm that ran only other tasks read as a wrong name. Those errors now add 'among the --task selection' when a selection is set. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/compare.py | 16 +++++++++------- dokimi/tests/test_repo_compare_reading.py | 18 ++++++++++++++++++ 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index 68c0c29db..af7bc5446 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -180,11 +180,13 @@ def compare(batch_dirs: list[Path], baseline: str, treatments: list[str] | None raise CompareError(f"--pass-k must be at least 1, got {pass_k}") cells, missing = load_batches(batch_dirs, metrics, path_stability) cells, missing, task_filter = _select_tasks(cells, missing, tasks) - _check_metrics(metrics, cells) + # the arms and metrics are checked among the selected cells, so a miss may be the selection's doing + scope = " among the --task selection" if task_filter else "" + _check_metrics(metrics, cells, scope) base_sel = parse_selector(baseline) if not any(selector_matches(base_sel, c) for c in cells): - raise CompareError(f"no finished cell matches the baseline {baseline!r}") - treatments = _treatments(cells, baseline, base_sel, treatments) + raise CompareError(f"no finished cell matches the baseline {baseline!r}{scope}") + treatments = _treatments(cells, baseline, base_sel, treatments, scope) measures = ["gate", "score", *([f"pass^{pass_k}"] if pass_k else []), *(["path_distance"] if path_stability else []), "cost_usd", "turns", "first_prompt_tokens", *(f"metric:{m}" for m in metrics)] @@ -350,11 +352,11 @@ def _process_measures(cells: list[dict[str, Any]], selectors: list[str]) -> list return sorted(found, key=process_order) -def _check_metrics(metrics: tuple[str, ...], cells: list[dict[str, Any]]) -> None: +def _check_metrics(metrics: tuple[str, ...], cells: list[dict[str, Any]], scope: str = "") -> None: """A ``--metric`` that no finished cell reports is a typo or not a number: refuse it, not print nothing.""" unknown = [m for m in metrics if not any(m in c["metrics"] for c in cells)] if unknown: - raise CompareError(f"no finished cell reports a numeric or boolean --metric {', '.join(unknown)}") + raise CompareError(f"no finished cell reports a numeric or boolean --metric {', '.join(unknown)}{scope}") def _check_models(models: list[str] | None, rows: list[dict[str, Any]], unpaired: list[dict[str, Any]]) -> None: @@ -484,7 +486,7 @@ def _unpaired(treatment: str, model: str, task: str, measure: str | None, reason def _treatments(cells: list[dict[str, Any]], baseline: str, base_sel: tuple[str | None, str], - treatments: list[str] | None) -> list[str]: + treatments: list[str] | None, scope: str = "") -> list[str]: """The treatment selectors to compare, checked: each matches cells, and none of the baseline's.""" if treatments is not None and not treatments: raise CompareError("no treatment arms named; leave --arm out to compare every other arm") @@ -498,7 +500,7 @@ def _treatments(cells: list[dict[str, Any]], baseline: str, base_sel: tuple[str raise CompareError(f"treatment {text!r} is the baseline") matched = [c for c in cells if selector_matches(sel, c)] if not matched: - raise CompareError(f"no finished cell matches the treatment {text!r}") + raise CompareError(f"no finished cell matches the treatment {text!r}{scope}") if any(selector_matches(base_sel, c) for c in matched): raise CompareError(f"treatment {text!r} also matches cells of the baseline {baseline!r}; " "name its batch (BATCH/ARM)") diff --git a/dokimi/tests/test_repo_compare_reading.py b/dokimi/tests/test_repo_compare_reading.py index 816701c67..b3992c997 100644 --- a/dokimi/tests/test_repo_compare_reading.py +++ b/dokimi/tests/test_repo_compare_reading.py @@ -109,6 +109,24 @@ def test_task_matches_a_missing_cell_by_its_task_not_by_a_prefix(tmp_path): assert compare([b], "guidance:none", tasks=["fix__a"])["missing"] == ["b/fix__a__guidance-raw__m-1__r2"] +def test_an_arm_or_metric_error_under_task_says_the_selection_is_the_reason(tmp_path): + b = tmp_path / "b" + _cell(b, "t1", "guidance:none", 1) + _cell(b, "t1", "guidance:raw", 1) + for arm in ("guidance:none", "guidance:other"): # guidance:other ran only t2, with the metric + _cell(b, "t2", arm, 1, metrics={"tok": 3.0}) + with pytest.raises(CompareError, match=r"no finished cell matches the treatment 'guidance:other' among the " + r"--task selection$"): + compare([b], "guidance:none", treatments=["guidance:other"], tasks=["t1"]) + with pytest.raises(CompareError, match=r"no finished cell matches the baseline 'guidance:other' among the " + r"--task selection$"): + compare([b], "guidance:other", tasks=["t1"]) + with pytest.raises(CompareError, match=r"reports a numeric or boolean --metric tok among the --task selection$"): + compare([b], "guidance:none", metrics=("tok",), tasks=["t1"]) + with pytest.raises(CompareError, match=r"^no finished cell matches the treatment 'guidance:nope'$"): + compare([b], "guidance:none", treatments=["guidance:nope"]) # without --task, no selection to name + + def test_task_refuses_an_id_no_cell_has(tmp_path, capsys): b = tmp_path / "b" _three_tasks(b) From 5fc51988453fde018229c1c843ed0b3022acdaf1 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 19:32:22 -0700 Subject: [PATCH 06/16] docs(context-window): say what first_prompt_tokens records The sentence bolded an orphan 'first'. It now names the first prompt. Signed-off-by: Nikolay Petrov --- dokimi/docs/context-window.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dokimi/docs/context-window.md b/dokimi/docs/context-window.md index 1d093fef6..f85438eca 100644 --- a/dokimi/docs/context-window.md +++ b/dokimi/docs/context-window.md @@ -80,6 +80,6 @@ point `CLAUDE_CONFIG_DIR` at an empty directory to leave it out. The the MCP servers and loaded plugins) shows what start-up offered, so two runs that differ there are not the same condition. -`repo run` records **first** for every cell as `result.json` -`agent.first_prompt_tokens`, and `repo compare` compares it between arms as +`repo run` records the size of each cell's **first prompt**, in tokens, as +`result.json` `agent.first_prompt_tokens`, and `repo compare` compares it between arms as the descriptive measure `first_prompt_tokens`. From e00edbcc9722d3e986ab5554e74c1d84dee77d19 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 19:32:22 -0700 Subject: [PATCH 07/16] docs(repo): what the --task of count counts of is the tasks with a finished cell, the only ids --task can name, not the task count of the batches: a task whose cells all crashed before result.json is not in it. Signed-off-by: Nikolay Petrov --- dokimi/docs/repo-evaluation-contract.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 72bbc43d5..fdc4aad43 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1347,8 +1347,8 @@ python cli.py repo compare repo-runs/replay --baseline guidance:none --metric re uncorrected and outside Holm's family. `--task ID` (repeatable) compares only those tasks: an id no finished cell of the batches has is refused, the tasks left out are not listed as unpaired nor their cells as missing, the `--json` summary records the - selection as `task_filter` (`tasks`, and `of`, the task count of the - batches; `null` without `--task`), and the text header says `tasks: N of M + selection as `task_filter` (`tasks`, and `of`, the number of tasks of the + batches that have a finished cell, the ids `--task` can name; `null` without `--task`), and the text header says `tasks: N of M (--task)`. A gate or score comparison carries `headroom`: `ceiling` (both arms at 1.0 or the task's `score.max` on every paired task), `floor` (both at 0 on every one), `near_ceiling` (the baseline at From 525e9d2cf66705688d228408591ed7a297fc212a Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 19:33:25 -0700 Subject: [PATCH 08/16] docs(compare): rewrap the module docstring after the mde change Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/compare.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index af7bc5446..e3fdd1cb3 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -43,8 +43,9 @@ Three notes keep a row from being misread; none changes a number. A gate or score row of 2+ tasks whose interval includes 0 is inconclusive, not "no effect": the note gives ``mde``, the smallest mean delta a two-sided 5% test finds with 80% power at the row's spread, ``(t_0.975 + t_0.80) * sd / sqrt(n)`` over the -per-task deltas, the t quantiles at ``n - 1`` degrees of freedom (in every row; ``None`` for one task or no spread). A row whose interval and p disagree (the -interval excludes 0 while p is 0.05 or more, or includes it while p is below) has ``interval_p_disagree``: +per-task deltas, the t quantiles at ``n - 1`` degrees of freedom (in every row; ``None`` for one task or no +spread). A row whose interval and p disagree (the interval excludes 0 while p is 0.05 or more, or includes it +while p is below) has ``interval_p_disagree``: the two come from different methods, and for gate and score p is the one to read, since the percentile bootstrap runs narrow over few tasks (the inconclusive note then gives way to this one). A descriptive row with p below 0.05 is named on a line saying its p is uncorrected and outside Holm's family. From eb5fbce4f7ca0b6da9169a6642a53861e4c82980 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Fri, 30 Oct 2026 10:00:45 +0000 Subject: [PATCH 09/16] refactor(repo): move compare's --task selection into compare_tasks.py repos/compare.py was 787 lines, and the review answers that follow (4418945747, 4420755924, 4418945753, 4420755898, 4418945763) take it past the 800-line limit of the growth ratchet. The --task selection (_select_tasks, _cell_task) moves unchanged into repos/compare_tasks.py, where the missing-cell fix of 4420755898 adds its code. An unknown id now raises TaskSelectionError there, which compare() re-raises as the same CompareError with the same message. No behaviour changes. Signed-off-by: Anton Volkov --- dokimi/docs/architecture.md | 3 ++- dokimi/dokimi/repos/compare.py | 31 ++++------------------- dokimi/dokimi/repos/compare_tasks.py | 37 ++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 27 deletions(-) create mode 100644 dokimi/dokimi/repos/compare_tasks.py diff --git a/dokimi/docs/architecture.md b/dokimi/docs/architecture.md index 6dd46dc4f..ebc4f30f8 100644 --- a/dokimi/docs/architecture.md +++ b/dokimi/docs/architecture.md @@ -337,7 +337,8 @@ next release. share, score, cost, turns, chosen `grade.json` metrics, `result.json` `process` counts), with an interval over tasks (bootstrap for gate and score, t for the rest), a sign-flip - permutation p and Holm's correction. + permutation p and Holm's correction. `dokimi/repos/compare_tasks.py` holds + what its `--task` selection keeps. - `dokimi/repos/selfcheck.py` — `repo tasks check`: on each task `grade` must fail the untouched workspace and pass it after `oracle`, and an isolation probe must not see the tasks or the source from the agent's diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index e3fdd1cb3..20ff9df26 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -105,6 +105,7 @@ from dokimi.context.consistency import Run, order_distance, pass_hat, run_from_events from dokimi.eval.stats import adjust_pvalues, bootstrap_ci, mean_ci +from .compare_tasks import TaskSelectionError, select_tasks from .process_counts import as_measure, process_order, process_values from .report import (load_cells, mismatch_warnings, model_matches, model_mismatch, reading_cells, resolved_models, unmeasured_note, uptake_counts) @@ -180,7 +181,10 @@ def compare(batch_dirs: list[Path], baseline: str, treatments: list[str] | None if pass_k is not None and pass_k < 1: raise CompareError(f"--pass-k must be at least 1, got {pass_k}") cells, missing = load_batches(batch_dirs, metrics, path_stability) - cells, missing, task_filter = _select_tasks(cells, missing, tasks) + try: + cells, missing, task_filter = select_tasks(cells, missing, tasks) + except TaskSelectionError as exc: + raise CompareError(str(exc)) from None # the arms and metrics are checked among the selected cells, so a miss may be the selection's doing scope = " among the --task selection" if task_filter else "" _check_metrics(metrics, cells, scope) @@ -215,31 +219,6 @@ def compare(batch_dirs: list[Path], baseline: str, treatments: list[str] | None "comparisons": rows, "unpaired": unpaired} -def _select_tasks(cells: list[dict[str, Any]], missing: list[str], - tasks: list[str] | None) -> tuple[list[dict[str, Any]], list[str], dict[str, Any] | None]: - """The cells and missing cells of ``tasks`` (``--task``), and the selection as the summary records it. - - No ``tasks`` keeps every cell and records ``None``. An id no finished cell of the batches has is refused.""" - if not tasks: - return cells, missing, None - known = {c["task"] for c in cells} - unknown = [t for t in dict.fromkeys(tasks) if t not in known] - if unknown: - raise CompareError(f"no finished cell of the compared batches has --task {', '.join(unknown)}") - kept = [m for m in missing if _cell_task(m.partition("/")[2]) in tasks] # BATCH/ - return [c for c in cells if c["task"] in tasks], kept, {"tasks": sorted(set(tasks)), "of": len(known)} - - -def _cell_task(cell: str) -> str | None: - """The task of a cell name ``______r`` (:func:`~dokimi.repos.runner.cell_name`). - - A task id may itself contain ``__``, so the three parts after it are split off from the right; the arm and - model are slugs, which the split assumes carry no ``__``. ``None`` for a name of another shape. - """ - parts = cell.rsplit("__", 3) - return parts[0] if len(parts) == 4 else None - - def _one_arm_models(treatment: str, base_vals: dict[tuple[str, str], dict[str, Any]], treat_vals: dict[tuple[str, str], dict[str, Any]], models: list[str] | None) -> list[dict[str, Any]]: """The tasks of a model only one arm ran: it has no task to pair, so every task is unpaired.""" diff --git a/dokimi/dokimi/repos/compare_tasks.py b/dokimi/dokimi/repos/compare_tasks.py new file mode 100644 index 000000000..51ea7a417 --- /dev/null +++ b/dokimi/dokimi/repos/compare_tasks.py @@ -0,0 +1,37 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""``repo compare --task``: the cells and missing cells a task selection keeps (:mod:`.compare`).""" + +from __future__ import annotations + +from typing import Any + + +class TaskSelectionError(ValueError): + """A ``--task`` id that no finished cell of the batches has.""" + + +def select_tasks(cells: list[dict[str, Any]], missing: list[str], + tasks: list[str] | None) -> tuple[list[dict[str, Any]], list[str], dict[str, Any] | None]: + """The cells and missing cells of ``tasks`` (``--task``), and the selection as the summary records it. + + No ``tasks`` keeps every cell and records ``None``. An id no finished cell of the batches has is refused.""" + if not tasks: + return cells, missing, None + known = {c["task"] for c in cells} + unknown = [t for t in dict.fromkeys(tasks) if t not in known] + if unknown: + raise TaskSelectionError(f"no finished cell of the compared batches has --task {', '.join(unknown)}") + kept = [m for m in missing if _cell_task(m.partition("/")[2]) in tasks] # BATCH/ + return [c for c in cells if c["task"] in tasks], kept, {"tasks": sorted(set(tasks)), "of": len(known)} + + +def _cell_task(cell: str) -> str | None: + """The task of a cell name ``______r`` (:func:`~dokimi.repos.runner.cell_name`). + + A task id may itself contain ``__``, so the three parts after it are split off from the right; the arm and + model are slugs, which the split assumes carry no ``__``. ``None`` for a name of another shape. + """ + parts = cell.rsplit("__", 3) + return parts[0] if len(parts) == 4 else None From 5eff69dbc20051859e474ac3fab501fe6dc25da0 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Fri, 30 Oct 2026 10:01:10 +0000 Subject: [PATCH 10/16] fix(repo): compare gives no mde where the sign-flip p cannot reach 0.05 Answers review comments 4418945747 and 4420755924. The row's p is an exact sign-flip test, whose smallest p over k changed (non-tied) tasks is 2 / 2**k: at 5 or fewer changed tasks no effect of any size reaches p < 0.05, yet the inconclusive note named an effect that 80% power would detect (gate deltas [1, 1, 0]: p 0.5, mde 1.788; the four-task score row: mde 0.950, p floor 0.125). min_sign_flip_p gives that floor (1 / (N + 1) for the Monte Carlo test above 16 changed tasks). Where it is 0.05 or more, mde is null in every row, and the gate or score note says instead "with 2 changed tasks the sign-flip p cannot go below 0.5: no effect can reach p < 0.05 here". From 6 changed tasks on, mde is computed as before. The module docstring, the contract and the changelog fragment say so, and no longer claim the notes change no number: they change no delta, interval or p. Signed-off-by: Anton Volkov --- ...61029_150000_antonwolfy_compare-reading.md | 6 ++- dokimi/docs/repo-evaluation-contract.md | 8 +++- dokimi/dokimi/repos/compare.py | 41 +++++++++++++++---- dokimi/tests/test_repo_compare_reading.py | 36 ++++++++++++++-- 4 files changed, 74 insertions(+), 17 deletions(-) diff --git a/dokimi/changelog.d/20261029_150000_antonwolfy_compare-reading.md b/dokimi/changelog.d/20261029_150000_antonwolfy_compare-reading.md index 3246881de..cb006882f 100644 --- a/dokimi/changelog.d/20261029_150000_antonwolfy_compare-reading.md +++ b/dokimi/changelog.d/20261029_150000_antonwolfy_compare-reading.md @@ -10,6 +10,8 @@ summary (`task_filter`) and the text header (`tasks: N of M (--task)`) record the selection. - **Reading notes under a `repo compare` comparison** — a gate or score row of 2+ tasks whose interval includes 0 says it is inconclusive and gives `mde`, the smallest effect 80% power detects at that - spread (`mde` is in every `--json` row); a row whose interval and p disagree says so and why + spread (`mde` is in every `--json` row). With 5 or fewer changed tasks the sign-flip p cannot go + below 0.05 (its floor is `2 / 2^k` for k changed tasks), so `mde` is `null` and the note says how + low p can go instead. A row whose interval and p disagree says so and why (`interval_p_disagree`), and for gate and score that p is the one to read; descriptive rows with - p below 0.05 are named as uncorrected and outside Holm's family. No number changes. + p below 0.05 are named as uncorrected and outside Holm's family. No delta, interval or p changes. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index fdc4aad43..e3913cff9 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1332,13 +1332,17 @@ python cli.py repo compare repo-runs/replay --baseline guidance:none --metric re reports it (a batch run before `repo run` recorded it), the row is absent. Holm's correction runs over the gate and score comparisons of one call; the rest are descriptive. Three notes under a comparison keep a row from being - misread, and change no number. A gate or score row of 2+ tasks whose + misread, and change no delta, interval or p. A gate or score row of 2+ tasks whose interval includes 0 is inconclusive, not evidence of no effect: the note gives `mde`, the smallest mean delta a two-sided 5% test finds with 80% power at that spread, `(t(0.975, n-1) + t(0.80, n-1)) * sd / sqrt(n)` over the per-task deltas, the t quantiles at `n-1` degrees of freedom (every row has `mde` in `--json`; `null` for one task or deltas with - no spread, and then no note). A row whose interval and p disagree (the + no spread, and then no note). The sign-flip p over k changed (non-tied) + tasks cannot go below `2 / 2^k`, so with 5 or fewer no effect of any size + reaches p < 0.05: `mde` is `null` there too, and the note says how low p can + go instead (`with 3 changed tasks the sign-flip p cannot go below 0.25: no + effect can reach p < 0.05 here`). A row whose interval and p disagree (the interval excludes 0 while p is 0.05 or more, or includes 0 while p is below 0.05) has `interval_p_disagree` `true` and a note: they come from two methods, and for gate and score p is the one to read, since a percentile diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index 20ff9df26..eac4b8f0b 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -40,11 +40,13 @@ the gate and score comparisons of one invocation; cost, turns and metrics are descriptive and keep their raw p. -Three notes keep a row from being misread; none changes a number. A gate or score row of 2+ tasks whose -interval includes 0 is inconclusive, not "no effect": the note gives ``mde``, the smallest mean delta a -two-sided 5% test finds with 80% power at the row's spread, ``(t_0.975 + t_0.80) * sd / sqrt(n)`` over the -per-task deltas, the t quantiles at ``n - 1`` degrees of freedom (in every row; ``None`` for one task or no -spread). A row whose interval and p disagree (the interval excludes 0 while p is 0.05 or more, or includes it +Three notes keep a row from being misread; none changes the delta, its interval or p. A gate or score row of +2+ tasks whose interval includes 0 is inconclusive, not "no effect": the note gives ``mde``, the smallest mean +delta a two-sided 5% test finds with 80% power at the row's spread, ``(t_0.975 + t_0.80) * sd / sqrt(n)`` over +the per-task deltas, the t quantiles at ``n - 1`` degrees of freedom (in every row; ``None`` for one task or no +spread). The sign-flip p over k changed tasks cannot go below ``2 / 2**k``, so at 5 or fewer no effect of any +size reaches p < 0.05: ``mde`` is ``None`` there too, and the note says how low p can go instead. A row whose +interval and p disagree (the interval excludes 0 while p is 0.05 or more, or includes it while p is below) has ``interval_p_disagree``: the two come from different methods, and for gate and score p is the one to read, since the percentile bootstrap runs narrow over few tasks (the inconclusive note then gives way to this one). A descriptive row @@ -523,6 +525,7 @@ def paired_summary(pairs: list[dict[str, Any]], measure: str = "gate", seed: int lo, hi = (float(ci["lo"]), float(ci["hi"])) if len(deltas) > 1 else (None, None) # mean_ci gives numpy floats p = sign_flip_p(deltas, seed=seed) sd = statistics.stdev(deltas) if len(deltas) > 1 else 0.0 + reachable = min_sign_flip_p(sum(abs(d) > EPS for d in deltas)) < ALPHA # else no effect can be detected return { "n_tasks": len(pairs), "baseline_mean": _round(_mean([p["baseline"] for p in pairs])), @@ -530,7 +533,7 @@ def paired_summary(pairs: list[dict[str, Any]], measure: str = "gate", seed: int "delta": {"mean": ci["mean"], "lo": lo, "hi": hi}, "p": p, "p_holm": None, - "mde": _round(min_detectable(sd, len(deltas))) if sd > EPS else None, + "mde": _round(min_detectable(sd, len(deltas))) if sd > EPS and reachable else None, "interval_p_disagree": lo is not None and hi is not None and p is not None and (lo <= 0 <= hi) == (p < ALPHA), "up": sum(d > EPS for d in deltas), "down": sum(d < -EPS for d in deltas), @@ -547,6 +550,12 @@ def min_detectable(sd: float, n: int) -> float: return (sp_stats.t.ppf(1 - ALPHA / 2, df) + sp_stats.t.ppf(POWER, df)) * sd / math.sqrt(n) +def min_sign_flip_p(changed: int) -> float: + """The smallest p :func:`sign_flip_p` can give over ``changed`` non-zero deltas, every sign the same: only that + pattern and its mirror reach the observed ``|sum|``, ``2 / 2**changed`` exact, ``1 / (N + 1)`` Monte Carlo.""" + return min(1.0, 2 / 2 ** changed) if changed <= EXACT_MAX else 1 / (N_RESAMPLES + 1) + + def sign_flip_p(deltas: list[float], seed: int = 0, n_resamples: int = N_RESAMPLES) -> float | None: """Two-sided p of ``|sum(deltas)|`` when each delta's sign is a fair coin under the null. @@ -614,9 +623,8 @@ def _reading_notes(rows: list[dict[str, Any]]) -> list[str]: out.append(f" {r['measure']}: the interval [{d['lo']:+.3f}, {d['hi']:+.3f}] " f"{'includes' if inside else 'excludes'} 0 but p {r['p']:.4g} is {'<' if inside else '≥'} " f"{ALPHA}; they come from two methods, {how} and a sign-flip test" + ("; read p" if primary else "")) - elif primary and r["mde"] is not None and d["lo"] <= 0 <= d["hi"]: - out.append(f" {r['measure']} inconclusive: the interval includes 0; with this spread the smallest effect " - f"80% power detects is about ±{r['mde']:.3f} ({r['n_tasks']} tasks)") + elif primary and (note := _inconclusive_note(r)): + out.append(note) low = [r["measure"] for r in rows if r["measure"] not in PRIMARY and r["p"] is not None and r["p"] < ALPHA] if low: out.append(f" p < {ALPHA} on {', '.join(low)}: descriptive rows, their p uncorrected and outside Holm's " @@ -624,6 +632,21 @@ def _reading_notes(rows: list[dict[str, Any]]) -> list[str]: return out +def _inconclusive_note(r: dict[str, Any]) -> str | None: + """The note of a gate or score row of 2+ tasks whose interval includes 0: its ``mde``, or why it has none.""" + d = r["delta"] + if d["lo"] is None or not d["lo"] <= 0 <= d["hi"]: + return None + head = f" {r['measure']} inconclusive: the interval includes 0; " + if r["mde"] is not None: + return head + f"with this spread the smallest effect 80% power detects is about ±{r['mde']:.3f} ({r['n_tasks']} tasks)" + changed = r["up"] + r["down"] + if r["p"] is None or (floor := min_sign_flip_p(changed)) < ALPHA: + return None # every delta tied, or no spread + return head + (f"with {changed} changed task{'' if changed == 1 else 's'} the sign-flip p cannot go below " + f"{floor:.4g}: no effect can reach p < {ALPHA} here") + + def _headroom_note(row: dict[str, Any]) -> str: n, gate = row["n_tasks"], row["measure"] == "gate" if row["headroom"] == "ceiling": diff --git a/dokimi/tests/test_repo_compare_reading.py b/dokimi/tests/test_repo_compare_reading.py index b3992c997..f378cf123 100644 --- a/dokimi/tests/test_repo_compare_reading.py +++ b/dokimi/tests/test_repo_compare_reading.py @@ -157,19 +157,47 @@ def _score_tasks(b: Path, deltas: list[float], base: float) -> None: def test_an_interval_across_zero_is_inconclusive_with_its_mde(tmp_path): - _score_tasks(tmp_path / "b", [0.5, -0.5, 0.25, -0.25], 0.5) # sample SD 0.456435 + # six changed tasks: the sign-flip p can go down to 2/64, so 80% power names an effect + _score_tasks(tmp_path / "b", [0.5, -0.5, 0.25, -0.25, 0.5, -0.5], 0.5) # sample SD 0.474342 s = compare([tmp_path / "b"], "guidance:none") score = _row(s, "score") - assert score["mde"] == pytest.approx(0.949594, abs=1e-6) and score["interval_p_disagree"] is False + assert score["mde"] == pytest.approx(min_detectable(0.474342, 6), abs=1e-6) and score["mde"] > 0.6 + assert score["interval_p_disagree"] is False text = format_compare(s) - assert (" score inconclusive: the interval includes 0; with this spread the smallest effect 80% power " - "detects is about ±0.950 (4 tasks)") in text.splitlines() + assert (f" score inconclusive: the interval includes 0; with this spread the smallest effect 80% power " + f"detects is about ±{score['mde']:.3f} (6 tasks)") in text.splitlines() assert "no effect" not in text _score_tasks(tmp_path / "one", [0.5], 0.5) one = compare([tmp_path / "one"], "guidance:none") assert _row(one, "score")["mde"] is None and "inconclusive" not in format_compare(one) +@pytest.mark.parametrize("deltas,changed,floor", [ + ([0.5, -0.5, 0.25, -0.25], 4, "0.125"), # four tasks, sample SD 0.456435: a t power figure would say ±0.950 + ([0.5, -0.5, 0.25, -0.25, 0.5, 0.0], 5, "0.0625"), # six tasks, one tied: five changed +]) +def test_no_mde_where_the_sign_flip_p_cannot_reach_005(tmp_path, deltas, changed, floor): + _score_tasks(tmp_path / "b", deltas, 0.5) + s = compare([tmp_path / "b"], "guidance:none") + score = _row(s, "score") + assert score["mde"] is None and score["delta"]["lo"] < 0 < score["delta"]["hi"] + assert (f" score inconclusive: the interval includes 0; with {changed} changed tasks the sign-flip p cannot " + f"go below {floor}: no effect can reach p < 0.05 here") in format_compare(s).splitlines() + assert "80% power" not in format_compare(s) + + +def test_a_gate_row_of_two_changed_tasks_has_no_mde(tmp_path): + b = tmp_path / "b" + for task, base_ok in (("t1", False), ("t2", False), ("t3", True)): # gate deltas [1, 1, 0] + _cell(b, task, "guidance:none", 1, success=base_ok) + _cell(b, task, "guidance:raw", 1) + s = compare([b], "guidance:none") + gate = _row(s, "gate") + assert gate["p"] == 0.5 and gate["delta"]["lo"] == 0.0 and gate["mde"] is None + assert (" gate inconclusive: the interval includes 0; with 2 changed tasks the sign-flip p cannot go below " + "0.5: no effect can reach p < 0.05 here") in format_compare(s).splitlines() + + def test_a_bootstrap_interval_that_excludes_zero_against_p_025_is_flagged(tmp_path): _score_tasks(tmp_path / "b", [0.0, 0.0, 0.333, 0.167, 0.0, 0.333], 0.0) s = compare([tmp_path / "b"], "guidance:none") From aeea5f1bde4ae2c61df22ecdccd2547888dc01ee Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Fri, 30 Oct 2026 10:09:33 +0000 Subject: [PATCH 11/16] fix(repo): compare flags a gate or score row against its Holm p Answers review comment 4418945753. interval_p_disagree was set in paired_summary, before _holm ran, so a gate or score row compared its interval with the raw p while p_holm is what decides it. With two treatments and gate deltas six +1 and one tie, each row printed +0.857 [+0.571, +1.000], p 0.031, p_holm 0.062 and no note. _holm now sets interval_p_disagree on the gate and score rows from Holm's verdict (correct_family, unrounded against 0.05), and their note quotes p_holm and says to read p_holm. Descriptive rows keep the flag on their raw p. The module docstring, the contract and the changelog fragment say so. Signed-off-by: Anton Volkov --- ...61029_150000_antonwolfy_compare-reading.md | 3 +- dokimi/docs/repo-evaluation-contract.md | 5 ++- dokimi/dokimi/repos/compare.py | 20 +++++---- dokimi/tests/test_repo_compare_reading.py | 42 +++++++++++++++++-- 4 files changed, 56 insertions(+), 14 deletions(-) diff --git a/dokimi/changelog.d/20261029_150000_antonwolfy_compare-reading.md b/dokimi/changelog.d/20261029_150000_antonwolfy_compare-reading.md index cb006882f..a9a4665cc 100644 --- a/dokimi/changelog.d/20261029_150000_antonwolfy_compare-reading.md +++ b/dokimi/changelog.d/20261029_150000_antonwolfy_compare-reading.md @@ -13,5 +13,6 @@ spread (`mde` is in every `--json` row). With 5 or fewer changed tasks the sign-flip p cannot go below 0.05 (its floor is `2 / 2^k` for k changed tasks), so `mde` is `null` and the note says how low p can go instead. A row whose interval and p disagree says so and why - (`interval_p_disagree`), and for gate and score that p is the one to read; descriptive rows with + (`interval_p_disagree`); for gate and score it compares the interval with Holm's `p_holm`, quotes + it and says it is the one to read, and a descriptive row uses its raw p; descriptive rows with p below 0.05 are named as uncorrected and outside Holm's family. No delta, interval or p changes. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index e3913cff9..fc4aecd9a 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1344,8 +1344,9 @@ python cli.py repo compare repo-runs/replay --baseline guidance:none --metric re go instead (`with 3 changed tasks the sign-flip p cannot go below 0.25: no effect can reach p < 0.05 here`). A row whose interval and p disagree (the interval excludes 0 while p is 0.05 or more, or includes 0 while p is below - 0.05) has `interval_p_disagree` `true` and a note: they come from two - methods, and for gate and score p is the one to read, since a percentile + 0.05; for gate and score that p is `p_holm`, the one that decides them, and the + note quotes it) has `interval_p_disagree` `true` and a note: they come from two + methods, and for gate and score `p_holm` is the one to read, since a percentile bootstrap runs narrow over few tasks; the inconclusive note gives way to it. Descriptive rows with p below 0.05 are named on one line: their p is uncorrected and outside Holm's family. `--task ID` (repeatable) compares only diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index eac4b8f0b..c8bff4b44 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -47,8 +47,8 @@ spread). The sign-flip p over k changed tasks cannot go below ``2 / 2**k``, so at 5 or fewer no effect of any size reaches p < 0.05: ``mde`` is ``None`` there too, and the note says how low p can go instead. A row whose interval and p disagree (the interval excludes 0 while p is 0.05 or more, or includes it -while p is below) has ``interval_p_disagree``: -the two come from different methods, and for gate and score p is the one to read, since the percentile +while p is below; for gate and score the p is ``p_holm``, the one that decides them) has ``interval_p_disagree``: +the two come from different methods, and for gate and score ``p_holm`` is the one to read, since the percentile bootstrap runs narrow over few tasks (the inconclusive note then gives way to this one). A descriptive row with p below 0.05 is named on a line saying its p is uncorrected and outside Holm's family. @@ -105,7 +105,7 @@ from scipy import stats as sp_stats from dokimi.context.consistency import Run, order_distance, pass_hat, run_from_events -from dokimi.eval.stats import adjust_pvalues, bootstrap_ci, mean_ci +from dokimi.eval.stats import bootstrap_ci, correct_family, mean_ci from .compare_tasks import TaskSelectionError, select_tasks from .process_counts import as_measure, process_order, process_values @@ -349,10 +349,12 @@ def _check_models(models: list[str] | None, rows: list[dict[str, Any]], unpaired def _holm(rows: list[dict[str, Any]]) -> None: - """``p_holm`` on the gate and score rows, corrected as one family.""" + """``p_holm`` on the gate and score rows, corrected as one family; their ``interval_p_disagree`` reads it.""" primary = [r for r in rows if r["measure"] in PRIMARY] - for row, p in zip(primary, adjust_pvalues([r["p"] for r in primary]), strict=True): + for row, (p, below) in zip(primary, correct_family([r["p"] for r in primary]), strict=True): + d = row["delta"] row["p_holm"] = p + row["interval_p_disagree"] = d["lo"] is not None and below is not None and (d["lo"] <= 0 <= d["hi"]) == below def _compare_model(treatment: str, model: str, base_vals: dict[tuple[str, str], dict[str, Any]], @@ -518,7 +520,8 @@ def paired_summary(pairs: list[dict[str, Any]], measure: str = "gate", seed: int permutation test. The interval is a percentile bootstrap for the bounded gate and score shares and a t interval (:func:`~dokimi.eval.stats.mean_ci`) for the continuous measures, where the bootstrap under-covers at these task counts; one task has no interval (``lo``/``hi`` None), not a zero-width one. - ``mde`` and ``interval_p_disagree`` are the reading aids of the module docstring. + ``mde`` and ``interval_p_disagree`` are the reading aids of the module docstring; the flag reads the raw p + here, and :func:`compare` sets it again on a gate or score row once Holm's p is known. """ deltas = [p["delta"] for p in pairs] ci = (bootstrap_ci if measure in PRIMARY else mean_ci)(deltas, seed=seed) @@ -620,9 +623,10 @@ def _reading_notes(rows: list[dict[str, Any]]) -> list[str]: if r["interval_p_disagree"]: inside = d["lo"] <= 0 <= d["hi"] how = "a percentile bootstrap, too narrow over few tasks," if primary else "a t interval" + name = "p_holm" if primary else "p" # a gate or score row is decided by its Holm p (_holm) out.append(f" {r['measure']}: the interval [{d['lo']:+.3f}, {d['hi']:+.3f}] " - f"{'includes' if inside else 'excludes'} 0 but p {r['p']:.4g} is {'<' if inside else '≥'} " - f"{ALPHA}; they come from two methods, {how} and a sign-flip test" + ("; read p" if primary else "")) + f"{'includes' if inside else 'excludes'} 0 but {name} {r[name]:.4g} is {'<' if inside else '≥'} " + f"{ALPHA}; they come from two methods, {how} and a sign-flip test" + ("; read p_holm" if primary else "")) elif primary and (note := _inconclusive_note(r)): out.append(note) low = [r["measure"] for r in rows if r["measure"] not in PRIMARY and r["p"] is not None and r["p"] < ALPHA] diff --git a/dokimi/tests/test_repo_compare_reading.py b/dokimi/tests/test_repo_compare_reading.py index f378cf123..20d85d4d1 100644 --- a/dokimi/tests/test_repo_compare_reading.py +++ b/dokimi/tests/test_repo_compare_reading.py @@ -205,9 +205,45 @@ def test_a_bootstrap_interval_that_excludes_zero_against_p_025_is_flagged(tmp_pa assert score["delta"]["lo"] > 0 and score["p"] == 0.25 # three non-zero deltas, all positive: p cannot go lower assert score["interval_p_disagree"] is True notes = [line for line in format_compare(s).splitlines() if line.startswith(" score")] - assert notes[-1].startswith(f" score: the interval [{score['delta']['lo']:+.3f}, +0.250] excludes 0 but p 0.25 " - "is ≥ 0.05; they come from two methods, a percentile bootstrap") - assert notes[-1].endswith("; read p") and not any("inconclusive" in line for line in notes) + assert notes[-1].startswith(f" score: the interval [{score['delta']['lo']:+.3f}, +0.250] excludes 0 but " + "p_holm 0.25 is ≥ 0.05; they come from two methods, a percentile bootstrap") + assert notes[-1].endswith("; read p_holm") and not any("inconclusive" in line for line in notes) + + +def _six_of_seven_up(b: Path, treatments: tuple[str, ...]) -> None: + """Gate deltas six +1 and one tie for each treatment: sign-flip p 2/64 for each.""" + for i in range(7): + _cell(b, f"t{i}", "guidance:none", 1, success=i == 6) + for arm in treatments: + _cell(b, f"t{i}", arm, 1) + + +def test_a_gate_interval_against_p_holm_is_flagged(tmp_path): + # two treatments: p 0.031 each, Holm 0.0625; the interval excludes 0, so the two disagree + _six_of_seven_up(tmp_path / "b", ("guidance:raw", "guidance:other")) + s = compare([tmp_path / "b"], "guidance:none") + for arm in ("guidance:raw", "guidance:other"): + gate = _row(s, "gate", treatment=arm) + assert gate["delta"]["lo"] > 0 and gate["p"] == pytest.approx(2 / 64) and gate["p_holm"] == 0.0625 + assert gate["interval_p_disagree"] is True + notes = [line for line in format_compare(s).splitlines() if line.startswith(" gate: the interval")] + assert len(notes) == 2 and all("excludes 0 but p_holm 0.0625 is ≥ 0.05" in n and n.endswith("; read p_holm") + for n in notes) + + +def test_an_interval_across_zero_against_a_raw_p_below_005_is_not_flagged_when_p_holm_is_not(tmp_path, monkeypatch): + # the mirror: an interval that includes 0 against p 0.031, which Holm takes to 0.0625: the two agree + monkeypatch.setattr("dokimi.repos.compare.bootstrap_ci", + lambda deltas, seed=0: {"mean": round(sum(deltas) / len(deltas), 4), "lo": -0.1, "hi": 1.0}) + _six_of_seven_up(tmp_path / "two", ("guidance:raw", "guidance:other")) + two = compare([tmp_path / "two"], "guidance:none") + gate = _row(two, "gate") + assert gate["p"] == pytest.approx(2 / 64) and gate["p_holm"] == 0.0625 and gate["interval_p_disagree"] is False + assert "gate: the interval" not in format_compare(two) + _six_of_seven_up(tmp_path / "one", ("guidance:raw",)) # a family of one: p_holm is p, below 0.05 + one = compare([tmp_path / "one"], "guidance:none") + assert _row(one, "gate")["interval_p_disagree"] is True + assert "includes 0 but p_holm 0.0312 is < 0.05" in format_compare(one) def test_a_descriptive_p_below_005_is_named_as_uncorrected(tmp_path): From d986407febf9bc560a55ceee4d68b421906a8dae Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Fri, 30 Oct 2026 10:10:42 +0000 Subject: [PATCH 12/16] fix(repo): compare --task reads a missing cell's task from its meta.json Answers review comment 4420755898 (and finishes 4418945759). 349d544 took a missing cell's task from its name, split from the right, which assumed the arm and model slugs carry no "__". They can: runner.cell_name keeps the "__" of a model (cell_name("fix", "guidance:raw", "m__v1", 2) is fix__guidance-raw__m__v1__r2, read as task fix__guidance-raw) and arms.slug keeps that of an arm such as skill:lint__v2, so --task fix silently left such an unfinished cell out of the missing list. The task now comes from the cell's meta.json, which repo run writes with the task id when the cell starts, read with the same guarded read as the module's other cell files. A cell only plan.json names (no files) gets the longest task id of its batch's finished cells that its name starts with followed by "__". A cell neither gives a task for is missing only without --task. The name-splitting _cell_task and its comment about slugs are gone. The contract and the changelog fragment say how the task is found. Signed-off-by: Anton Volkov --- ...61029_150000_antonwolfy_compare-reading.md | 3 +- dokimi/docs/repo-evaluation-contract.md | 6 ++- dokimi/dokimi/repos/compare.py | 2 +- dokimi/dokimi/repos/compare_tasks.py | 39 +++++++++++++------ dokimi/tests/test_repo_compare_reading.py | 27 +++++++++++++ 5 files changed, 62 insertions(+), 15 deletions(-) diff --git a/dokimi/changelog.d/20261029_150000_antonwolfy_compare-reading.md b/dokimi/changelog.d/20261029_150000_antonwolfy_compare-reading.md index a9a4665cc..7a12aac4b 100644 --- a/dokimi/changelog.d/20261029_150000_antonwolfy_compare-reading.md +++ b/dokimi/changelog.d/20261029_150000_antonwolfy_compare-reading.md @@ -6,7 +6,8 @@ the task. It is `null` for a trace without that call and for the `text` format. `repo compare` adds a descriptive `first_prompt_tokens` row; a batch written before the field has no such row. - **`repo compare --task ID`** — repeatable: compares only those task ids. An id no finished cell of the - batches has exits 1 and names it; the tasks left out are not listed as unpaired, and the `--json` + batches has exits 1 and names it; the tasks left out are not listed as unpaired, nor their unfinished + cells as missing (a cell's task is the one its `meta.json` records), and the `--json` summary (`task_filter`) and the text header (`tasks: N of M (--task)`) record the selection. - **Reading notes under a `repo compare` comparison** — a gate or score row of 2+ tasks whose interval includes 0 says it is inconclusive and gives `mde`, the smallest effect 80% power detects at that diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index fc4aecd9a..a2f695ea6 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1351,7 +1351,11 @@ python cli.py repo compare repo-runs/replay --baseline guidance:none --metric re it. Descriptive rows with p below 0.05 are named on one line: their p is uncorrected and outside Holm's family. `--task ID` (repeatable) compares only those tasks: an id no finished cell of the batches has is refused, the tasks - left out are not listed as unpaired nor their cells as missing, the `--json` summary records the + left out are not listed as unpaired nor their cells as missing (a missing + cell's task is the one its `meta.json` records, else, for a cell only + `plan.json` names, the longest task id of the batch's finished cells that its + name starts with followed by `__`; a cell of neither is missing only without + `--task`), the `--json` summary records the selection as `task_filter` (`tasks`, and `of`, the number of tasks of the batches that have a finished cell, the ids `--task` can name; `null` without `--task`), and the text header says `tasks: N of M (--task)`. A gate or score comparison carries `headroom`: diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index c8bff4b44..b9320aac3 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -184,7 +184,7 @@ def compare(batch_dirs: list[Path], baseline: str, treatments: list[str] | None raise CompareError(f"--pass-k must be at least 1, got {pass_k}") cells, missing = load_batches(batch_dirs, metrics, path_stability) try: - cells, missing, task_filter = select_tasks(cells, missing, tasks) + cells, missing, task_filter = select_tasks(cells, missing, tasks, batch_dirs) except TaskSelectionError as exc: raise CompareError(str(exc)) from None # the arms and metrics are checked among the selected cells, so a miss may be the selection's doing diff --git a/dokimi/dokimi/repos/compare_tasks.py b/dokimi/dokimi/repos/compare_tasks.py index 51ea7a417..d3d9d1696 100644 --- a/dokimi/dokimi/repos/compare_tasks.py +++ b/dokimi/dokimi/repos/compare_tasks.py @@ -5,6 +5,8 @@ from __future__ import annotations +import json +from pathlib import Path from typing import Any @@ -12,26 +14,39 @@ class TaskSelectionError(ValueError): """A ``--task`` id that no finished cell of the batches has.""" -def select_tasks(cells: list[dict[str, Any]], missing: list[str], - tasks: list[str] | None) -> tuple[list[dict[str, Any]], list[str], dict[str, Any] | None]: +def select_tasks(cells: list[dict[str, Any]], missing: list[str], tasks: list[str] | None, + batch_dirs: list[Path]) -> tuple[list[dict[str, Any]], list[str], dict[str, Any] | None]: """The cells and missing cells of ``tasks`` (``--task``), and the selection as the summary records it. - No ``tasks`` keeps every cell and records ``None``. An id no finished cell of the batches has is refused.""" + No ``tasks`` keeps every cell, a missing cell of no known task too, and records ``None``. An id no finished + cell of the batches has is refused.""" if not tasks: return cells, missing, None known = {c["task"] for c in cells} unknown = [t for t in dict.fromkeys(tasks) if t not in known] if unknown: raise TaskSelectionError(f"no finished cell of the compared batches has --task {', '.join(unknown)}") - kept = [m for m in missing if _cell_task(m.partition("/")[2]) in tasks] # BATCH/ + dirs = {d.resolve().name: d for d in batch_dirs} + ids = {name: {c["task"] for c in cells if c["batch"] == name} for name in dirs} + kept = [] + for entry in missing: # BATCH/ + batch, _, cell = entry.partition("/") + # a cell whose task neither its meta.json nor a known id gives is not in any selection + if _missing_task(dirs[batch], cell, ids[batch]) in tasks: + kept.append(entry) return [c for c in cells if c["task"] in tasks], kept, {"tasks": sorted(set(tasks)), "of": len(known)} -def _cell_task(cell: str) -> str | None: - """The task of a cell name ``______r`` (:func:`~dokimi.repos.runner.cell_name`). - - A task id may itself contain ``__``, so the three parts after it are split off from the right; the arm and - model are slugs, which the split assumes carry no ``__``. ``None`` for a name of another shape. - """ - parts = cell.rsplit("__", 3) - return parts[0] if len(parts) == 4 else None +def _missing_task(batch_dir: Path, cell: str, known: set[str]) -> str | None: + """The task of a missing cell: the id its ``meta.json`` records (``repo run`` writes it when the cell starts), + else, for a cell only ``plan.json`` names, the longest of the batch's finished task ids ``known`` that the + name starts with followed by ``__``. The name alone does not say: :func:`~dokimi.repos.runner.cell_name` + keeps a ``__`` of the task id, the arm and the model. ``None`` when neither gives a task.""" + try: + meta = json.loads((batch_dir / cell / "meta.json").read_text(encoding="utf-8")) + except (OSError, ValueError, RecursionError): + meta = None + task = meta.get("task") if isinstance(meta, dict) else None + if isinstance(task, str): + return task + return max((t for t in known if cell.startswith(f"{t}__")), key=len, default=None) diff --git a/dokimi/tests/test_repo_compare_reading.py b/dokimi/tests/test_repo_compare_reading.py index 20d85d4d1..f3f0b24e8 100644 --- a/dokimi/tests/test_repo_compare_reading.py +++ b/dokimi/tests/test_repo_compare_reading.py @@ -12,6 +12,7 @@ from dokimi.cli import build_parser from dokimi.repos.compare import CompareError, compare, format_compare, min_detectable +from dokimi.repos.runner import cell_name from test_repo_compare import _cell, _row @@ -109,6 +110,32 @@ def test_task_matches_a_missing_cell_by_its_task_not_by_a_prefix(tmp_path): assert compare([b], "guidance:none", tasks=["fix__a"])["missing"] == ["b/fix__a__guidance-raw__m-1__r2"] +def test_task_reads_a_missing_cell_s_task_from_its_meta_or_the_known_ids(tmp_path): + # cell names keep a "__" of the task, the arm and the model: fix__guidance-raw__m__v1__r2 is a cell of fix, + # though a task fix__guidance-raw exists too + b, model = tmp_path / "b", "m__v1" + for task in ("fix", "fix__a", "fix__guidance-raw"): + _cell(b, task, "guidance:none", 1, model=model) + _cell(b, task, "skill:lint__v2", 1, model=model) + started = {cell_name("fix", "guidance:raw", model, 2): "fix", + cell_name("fix__a", "skill:lint__v2", model, 2): "fix__a"} + for cell, task in started.items(): + (b / cell).mkdir() + (b / cell / "meta.json").write_text(json.dumps({"task": task})) # what repo run writes at the start + planned = [cell_name("fix", "skill:lint__v2", model, 3), cell_name("fix__a", "guidance:none", model, 3), + cell_name("gone", "guidance:none", model, 1)] + (b / "_eval").mkdir() + (b / "_eval" / "plan.json").write_text(json.dumps({"cells": [p.parent.name for p in b.glob("*/result.json")] + + [*started, *planned]})) + fix = compare([b], "guidance:none", tasks=["fix"]) + assert fix["missing"] == ["b/fix__guidance-raw__m__v1__r2", "b/fix__skill-lint__v2__m__v1__r3"] + assert compare([b], "guidance:none", tasks=["fix__a"])["missing"] == ["b/fix__a__guidance-none__m__v1__r3", + "b/fix__a__skill-lint__v2__m__v1__r2"] + assert compare([b], "guidance:none", tasks=["fix__guidance-raw"])["missing"] == [] + # a planned cell of a task no finished cell has belongs to no selection, but is missing without one + assert "b/gone__guidance-none__m__v1__r1" in compare([b], "guidance:none")["missing"] + + def test_an_arm_or_metric_error_under_task_says_the_selection_is_the_reason(tmp_path): b = tmp_path / "b" _cell(b, "t1", "guidance:none", 1) From 68b4c8609ae7bf782e803cf50d58eff2eba7179d Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Fri, 30 Oct 2026 10:13:21 +0000 Subject: [PATCH 13/16] fix(repo): compare names the arms --task leaves without a cell Answers the second half of review comment 4418945763. Without --arm, compare takes every other arm that has a finished cell among the --task selection, so an arm that ran none of the selected tasks dropped out of the comparison with no word. task_filter now carries dropped_arms: without --arm, the arms of the batches with no finished cell among the selection ([] with --arm, where such an arm is refused). The text header prints them under the tasks line ("not compared, no finished cell among the --task selection: guidance:raw"). The default arm list is one helper, _other_arms, for both. The module docstring, the contract and the changelog fragment say so; the fragment also says what M of "tasks: N of M" counts (the tasks with a finished cell), which e00edbc said in the contract only (4418945768). To keep repos/compare.py under the 800-line limit, CompareError moves into compare_tasks.py, which now raises it for an unknown --task id directly; compare.py imports it, so dokimi.repos.compare.CompareError is unchanged for its callers. Signed-off-by: Anton Volkov --- ...61029_150000_antonwolfy_compare-reading.md | 5 ++- dokimi/docs/repo-evaluation-contract.md | 7 +++- dokimi/dokimi/repos/compare.py | 38 +++++++++++-------- dokimi/dokimi/repos/compare_tasks.py | 6 +-- dokimi/tests/test_repo_compare_reading.py | 22 +++++++++-- 5 files changed, 53 insertions(+), 25 deletions(-) diff --git a/dokimi/changelog.d/20261029_150000_antonwolfy_compare-reading.md b/dokimi/changelog.d/20261029_150000_antonwolfy_compare-reading.md index 7a12aac4b..d1d50d5ab 100644 --- a/dokimi/changelog.d/20261029_150000_antonwolfy_compare-reading.md +++ b/dokimi/changelog.d/20261029_150000_antonwolfy_compare-reading.md @@ -8,7 +8,10 @@ - **`repo compare --task ID`** — repeatable: compares only those task ids. An id no finished cell of the batches has exits 1 and names it; the tasks left out are not listed as unpaired, nor their unfinished cells as missing (a cell's task is the one its `meta.json` records), and the `--json` - summary (`task_filter`) and the text header (`tasks: N of M (--task)`) record the selection. + summary (`task_filter`) and the text header (`tasks: N of M (--task)`, M the tasks with a finished + cell) record the selection. Without `--arm`, an arm with no finished cell among the selection is + not compared, and both name it (`task_filter.dropped_arms`, `not compared, no finished cell among + the --task selection: `). - **Reading notes under a `repo compare` comparison** — a gate or score row of 2+ tasks whose interval includes 0 says it is inconclusive and gives `mde`, the smallest effect 80% power detects at that spread (`mde` is in every `--json` row). With 5 or fewer changed tasks the sign-flip p cannot go diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index a2f695ea6..16007dca7 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1357,8 +1357,11 @@ python cli.py repo compare repo-runs/replay --baseline guidance:none --metric re name starts with followed by `__`; a cell of neither is missing only without `--task`), the `--json` summary records the selection as `task_filter` (`tasks`, and `of`, the number of tasks of the - batches that have a finished cell, the ids `--task` can name; `null` without `--task`), and the text header says `tasks: N of M - (--task)`. A gate or score comparison carries `headroom`: + batches that have a finished cell, the ids `--task` can name, and + `dropped_arms`: without `--arm`, the arms with no finished cell among the + selection, which drop out of the comparison, `[]` with `--arm`; `null` + without `--task`), and the text header says `tasks: N of M (--task)` and, + under it, `not compared, no finished cell among the --task selection: `. A gate or score comparison carries `headroom`: `ceiling` (both arms at 1.0 or the task's `score.max` on every paired task), `floor` (both at 0 on every one), `near_ceiling` (the baseline at the top on all but at most one of 2+ tasks and the treatment nowhere below diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index b9320aac3..3470933cd 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -53,7 +53,8 @@ with p below 0.05 is named on a line saying its p is uncorrected and outside Holm's family. ``--task`` limits a comparison to some task ids: the others are left out, not listed as unpaired, and the -summary's ``task_filter`` records the selection. +summary's ``task_filter`` records the selection; without ``--arm``, its ``dropped_arms`` names an arm with no +finished cell among it, which drops out of the comparison. A gate or score comparison also says whether its tasks had room to move (``headroom``), since a zero delta on a task set nobody can fail, or nobody can pass, is not evidence of no effect. A task's top is 1.0 for @@ -107,7 +108,7 @@ from dokimi.context.consistency import Run, order_distance, pass_hat, run_from_events from dokimi.eval.stats import bootstrap_ci, correct_family, mean_ci -from .compare_tasks import TaskSelectionError, select_tasks +from .compare_tasks import CompareError, select_tasks from .process_counts import as_measure, process_order, process_values from .report import (load_cells, mismatch_warnings, model_matches, model_mismatch, reading_cells, resolved_models, unmeasured_note, uptake_counts) @@ -130,10 +131,6 @@ STAMP_KEYS = ("eval_dir", "image", "graders") -class CompareError(ValueError): - """The arms or batches named cannot be compared.""" - - def parse_selector(text: str) -> tuple[str | None, str]: """``ARM`` or ``BATCH/ARM`` (batch = the batch directory's name) as ``(batch, arm)``.""" batch, sep, arm = text.rpartition("/") @@ -182,17 +179,17 @@ def compare(batch_dirs: list[Path], baseline: str, treatments: list[str] | None metrics = tuple(dict.fromkeys(metrics)) if pass_k is not None and pass_k < 1: raise CompareError(f"--pass-k must be at least 1, got {pass_k}") - cells, missing = load_batches(batch_dirs, metrics, path_stability) - try: - cells, missing, task_filter = select_tasks(cells, missing, tasks, batch_dirs) - except TaskSelectionError as exc: - raise CompareError(str(exc)) from None + found, missing = load_batches(batch_dirs, metrics, path_stability) + cells, missing, task_filter = select_tasks(found, missing, tasks, batch_dirs) # the arms and metrics are checked among the selected cells, so a miss may be the selection's doing scope = " among the --task selection" if task_filter else "" _check_metrics(metrics, cells, scope) base_sel = parse_selector(baseline) if not any(selector_matches(base_sel, c) for c in cells): raise CompareError(f"no finished cell matches the baseline {baseline!r}{scope}") + if task_filter is not None: # without --arm, an arm that ran none of the selected tasks would drop out unsaid + task_filter["dropped_arms"] = ([] if treatments is not None + else sorted(_other_arms(found, base_sel) - _other_arms(cells, base_sel))) treatments = _treatments(cells, baseline, base_sel, treatments, scope) measures = ["gate", "score", *([f"pass^{pass_k}"] if pass_k else []), *(["path_distance"] if path_stability else []), "cost_usd", "turns", "first_prompt_tokens", @@ -469,15 +466,19 @@ def _unpaired(treatment: str, model: str, task: str, measure: str | None, reason return {"treatment": treatment, "model": model, "task": task, "measure": measure, "reason": reason} +def _other_arms(cells: list[dict[str, Any]], base_sel: tuple[str | None, str]) -> set[str]: + """Every other arm that has cells; the baseline's own arm from another batch is named by its batch.""" + return {f"{c['batch']}/{c['arm']}" if c["arm"] == base_sel[1] else c["arm"] + for c in cells if not selector_matches(base_sel, c)} + + def _treatments(cells: list[dict[str, Any]], baseline: str, base_sel: tuple[str | None, str], treatments: list[str] | None, scope: str = "") -> list[str]: """The treatment selectors to compare, checked: each matches cells, and none of the baseline's.""" if treatments is not None and not treatments: raise CompareError("no treatment arms named; leave --arm out to compare every other arm") if treatments is None: - # every other arm that has cells; the baseline's own arm from another batch is named by its batch - treatments = sorted({f"{c['batch']}/{c['arm']}" if c["arm"] == base_sel[1] else c["arm"] - for c in cells if not selector_matches(base_sel, c)}) + treatments = sorted(_other_arms(cells, base_sel)) for text in treatments: sel = parse_selector(text) if sel == base_sel: @@ -626,7 +627,8 @@ def _reading_notes(rows: list[dict[str, Any]]) -> list[str]: name = "p_holm" if primary else "p" # a gate or score row is decided by its Holm p (_holm) out.append(f" {r['measure']}: the interval [{d['lo']:+.3f}, {d['hi']:+.3f}] " f"{'includes' if inside else 'excludes'} 0 but {name} {r[name]:.4g} is {'<' if inside else '≥'} " - f"{ALPHA}; they come from two methods, {how} and a sign-flip test" + ("; read p_holm" if primary else "")) + f"{ALPHA}; they come from two methods, {how} and a sign-flip test" + + ("; read p_holm" if primary else "")) elif primary and (note := _inconclusive_note(r)): out.append(note) low = [r["measure"] for r in rows if r["measure"] not in PRIMARY and r["p"] is not None and r["p"] < ALPHA] @@ -643,7 +645,8 @@ def _inconclusive_note(r: dict[str, Any]) -> str | None: return None head = f" {r['measure']} inconclusive: the interval includes 0; " if r["mde"] is not None: - return head + f"with this spread the smallest effect 80% power detects is about ±{r['mde']:.3f} ({r['n_tasks']} tasks)" + return head + (f"with this spread the smallest effect 80% power detects is about ±{r['mde']:.3f} " + f"({r['n_tasks']} tasks)") changed = r["up"] + r["down"] if r["p"] is None or (floor := min_sign_flip_p(changed)) < ALPHA: return None # every delta tied, or no spread @@ -667,6 +670,9 @@ def _format_header(summary: dict[str, Any]) -> list[str]: f"{len(summary['batches'])} batch(es), {len(summary['missing'])} missing"] if summary.get("task_filter"): lines.append(f" tasks: {len(summary['task_filter']['tasks'])} of {summary['task_filter']['of']} (--task)") + if summary["task_filter"].get("dropped_arms"): + lines.append(f" not compared, no finished cell among the --task selection: " + f"{', '.join(summary['task_filter']['dropped_arms'])}") for text, batches in summary["sources"].items(): lines.append(f" {text}: {', '.join(batches)}") for name, stamp in summary["stamps"].items(): diff --git a/dokimi/dokimi/repos/compare_tasks.py b/dokimi/dokimi/repos/compare_tasks.py index d3d9d1696..81bd2e712 100644 --- a/dokimi/dokimi/repos/compare_tasks.py +++ b/dokimi/dokimi/repos/compare_tasks.py @@ -10,8 +10,8 @@ from typing import Any -class TaskSelectionError(ValueError): - """A ``--task`` id that no finished cell of the batches has.""" +class CompareError(ValueError): + """The arms or batches named cannot be compared (here, a ``--task`` id; :mod:`.compare` raises the others).""" def select_tasks(cells: list[dict[str, Any]], missing: list[str], tasks: list[str] | None, @@ -25,7 +25,7 @@ def select_tasks(cells: list[dict[str, Any]], missing: list[str], tasks: list[st known = {c["task"] for c in cells} unknown = [t for t in dict.fromkeys(tasks) if t not in known] if unknown: - raise TaskSelectionError(f"no finished cell of the compared batches has --task {', '.join(unknown)}") + raise CompareError(f"no finished cell of the compared batches has --task {', '.join(unknown)}") dirs = {d.resolve().name: d for d in batch_dirs} ids = {name: {c["task"] for c in cells if c["batch"] == name} for name in dirs} kept = [] diff --git a/dokimi/tests/test_repo_compare_reading.py b/dokimi/tests/test_repo_compare_reading.py index f3f0b24e8..99ffa8fbb 100644 --- a/dokimi/tests/test_repo_compare_reading.py +++ b/dokimi/tests/test_repo_compare_reading.py @@ -78,7 +78,7 @@ def test_task_compares_only_the_named_tasks_and_says_so(tmp_path): gate = _row(s, "gate") assert [t["task"] for t in gate["tasks"]] == ["t1", "t2"] and gate["up"] == 1 assert s["unpaired"] == [] and s["missing"] == [] and s["cells"] == 4 - assert s["task_filter"] == {"tasks": ["t1", "t2"], "of": 4} + assert s["task_filter"] == {"tasks": ["t1", "t2"], "of": 4, "dropped_arms": []} assert " tasks: 2 of 4 (--task)" in format_compare(s).splitlines() assert "(--task)" not in format_compare(whole) @@ -90,7 +90,7 @@ def test_cli_task_is_repeatable_and_lands_in_the_json(tmp_path, capsys): "--task", "t3", "--task", "t4", "--json"]) args.func(args) out = json.loads(capsys.readouterr().out) - assert out["task_filter"] == {"tasks": ["t3", "t4"], "of": 4} + assert out["task_filter"] == {"tasks": ["t3", "t4"], "of": 4, "dropped_arms": []} assert [t["task"] for t in _row(out, "gate")["tasks"]] == ["t3"] assert [u["task"] for u in out["unpaired"]] == ["t4"] and len(out["missing"]) == 1 @@ -104,7 +104,7 @@ def test_task_matches_a_missing_cell_by_its_task_not_by_a_prefix(tmp_path): (b / "fix__a__guidance-raw__m-1__r2").mkdir() (b / "fix__a__guidance-raw__m-1__r2" / "meta.json").write_text("{}") # started, never finished: missing s = compare([b], "guidance:none", tasks=["fix"]) - assert s["missing"] == [] and s["task_filter"] == {"tasks": ["fix"], "of": 2} + assert s["missing"] == [] and s["task_filter"] == {"tasks": ["fix"], "of": 2, "dropped_arms": []} assert " tasks: 1 of 2 (--task)" in format_compare(s).splitlines() assert "cells missing" not in format_compare(s) assert compare([b], "guidance:none", tasks=["fix__a"])["missing"] == ["b/fix__a__guidance-raw__m-1__r2"] @@ -154,6 +154,22 @@ def test_an_arm_or_metric_error_under_task_says_the_selection_is_the_reason(tmp_ compare([b], "guidance:none", treatments=["guidance:nope"]) # without --task, no selection to name +def test_task_names_an_arm_that_ran_none_of_the_selected_tasks(tmp_path): + b = tmp_path / "b" + for task in ("t1", "t2", "t3"): + _cell(b, task, "guidance:none", 1) + _cell(b, task, "guidance:other", 1) + for task in ("t1", "t2"): # guidance:raw ran only t1 and t2 + _cell(b, task, "guidance:raw", 1) + s = compare([b], "guidance:none", tasks=["t3"]) + assert s["treatments"] == ["guidance:other"] and s["task_filter"]["dropped_arms"] == ["guidance:raw"] + assert (" not compared, no finished cell among the --task selection: guidance:raw" + in format_compare(s).splitlines()) + named = compare([b], "guidance:none", treatments=["guidance:other"], tasks=["t3"]) # --arm: nothing drops out + assert named["task_filter"]["dropped_arms"] == [] and "not compared" not in format_compare(named) + assert compare([b], "guidance:none", tasks=["t1"])["task_filter"]["dropped_arms"] == [] + + def test_task_refuses_an_id_no_cell_has(tmp_path, capsys): b = tmp_path / "b" _three_tasks(b) From c19f9d20ec57262b58d94671bff4728b098b121c Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Fri, 30 Oct 2026 10:42:44 +0000 Subject: [PATCH 14/16] test(cli): regenerate the CLI snapshot for repo compare --task The merge of main brought #254's flag registry and its snapshot check: docs/reference/cli-interface.json lacked the --task option this branch adds to `repo compare`, so test_the_cli_interface_snapshot_is_current failed. Regenerated with scripts/cli_surface.py dump (`make cli-snapshot`). --task needs no allowlist row: it is already a canonical spelling of the registry's `inputs` concept ("a filter on --tasks", text, repeatable), which is what the compare option is (an append of task ids). #298's check_comment_blocks.py --base origin/main --head HEAD finds no comment block over two lines among the lines this branch adds or edits, so no comment changes. Signed-off-by: Anton Volkov --- dokimi/docs/reference/cli-interface.json | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index 3778ecf3d..1bbc56081 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -10263,6 +10263,22 @@ "position": null, "required": false, "type": null + }, + { + "action": "_AppendAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--task", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null } ], "path": "dokimi repo compare", From 63c112e4468d41730419fa97274364cc4a4b36fe Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Fri, 30 Oct 2026 10:48:45 +0000 Subject: [PATCH 15/16] fix(repo): compare gives no mde where Holm keeps p_holm from 0.05 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review comment 4422703568: mde and the inconclusive note read the raw sign-flip floor, but gate and score rows are decided by p_holm. Two score treatments with six changed tasks each both got an mde and an "80% power detects" note, though Holm over the two rows cannot take either p_holm below 2 * 2/64 = 0.0625. After Holm, each gate or score row gets p_holm_floor: its p_holm with its own p put at its floor (min_sign_flip_p over its changed tasks) and the family's other p as observed, from correct_family. Where that is not below 0.05, mde is null and the inconclusive note says "with Holm over N rows, p_holm cannot go below X here: no effect can reach p_holm < 0.05". A family of one row is the raw rule, as before; descriptive rows keep it. min_detectable, ALPHA and POWER move to the new repos/compare_reach.py with the floor helper, holm_floor, so compare.py stays under 800 lines; compare re-imports them. Tests pin the two-treatment case (both rows null, the Holm note) and one where a row of eight changed tasks keeps its mde beside a six-task row that loses it. The module docstring, contract §9, architecture.md and the changelog fragment follow. Signed-off-by: Anton Volkov --- ...61029_150000_antonwolfy_compare-reading.md | 5 +- dokimi/docs/architecture.md | 4 +- dokimi/docs/repo-evaluation-contract.md | 8 ++- dokimi/dokimi/repos/compare.py | 54 +++++++++---------- dokimi/dokimi/repos/compare_reach.py | 30 +++++++++++ dokimi/tests/test_repo_compare_reading.py | 40 ++++++++++++++ 6 files changed, 111 insertions(+), 30 deletions(-) create mode 100644 dokimi/dokimi/repos/compare_reach.py diff --git a/dokimi/changelog.d/20261029_150000_antonwolfy_compare-reading.md b/dokimi/changelog.d/20261029_150000_antonwolfy_compare-reading.md index d1d50d5ab..609049dc0 100644 --- a/dokimi/changelog.d/20261029_150000_antonwolfy_compare-reading.md +++ b/dokimi/changelog.d/20261029_150000_antonwolfy_compare-reading.md @@ -16,7 +16,10 @@ includes 0 says it is inconclusive and gives `mde`, the smallest effect 80% power detects at that spread (`mde` is in every `--json` row). With 5 or fewer changed tasks the sign-flip p cannot go below 0.05 (its floor is `2 / 2^k` for k changed tasks), so `mde` is `null` and the note says how - low p can go instead. A row whose interval and p disagree says so and why + low p can go instead. The same holds where Holm's correction keeps a gate or score row's + `p_holm_floor` (its `p_holm` with its p at that floor, the other rows of the family as observed) at + 0.05 or above, as with two such rows of six changed tasks each (`p_holm` 0.0625 at best); the note + then says `with Holm over N rows, p_holm cannot go below ...`. A row whose interval and p disagree says so and why (`interval_p_disagree`); for gate and score it compares the interval with Holm's `p_holm`, quotes it and says it is the one to read, and a descriptive row uses its raw p; descriptive rows with p below 0.05 are named as uncorrected and outside Holm's family. No delta, interval or p changes. diff --git a/dokimi/docs/architecture.md b/dokimi/docs/architecture.md index ebc4f30f8..2fcf45bfa 100644 --- a/dokimi/docs/architecture.md +++ b/dokimi/docs/architecture.md @@ -338,7 +338,9 @@ next release. `process` counts), with an interval over tasks (bootstrap for gate and score, t for the rest), a sign-flip permutation p and Holm's correction. `dokimi/repos/compare_tasks.py` holds - what its `--task` selection keeps. + what its `--task` selection keeps; `dokimi/repos/compare_reach.py` how small + an effect a row can find (`mde`, and the lowest `p_holm` Holm's family + leaves it). - `dokimi/repos/selfcheck.py` — `repo tasks check`: on each task `grade` must fail the untouched workspace and pass it after `oracle`, and an isolation probe must not see the tasks or the source from the agent's diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 082f54708..e1f4fd795 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1361,7 +1361,13 @@ python cli.py repo compare repo-runs/replay --baseline guidance:none --metric re tasks cannot go below `2 / 2^k`, so with 5 or fewer no effect of any size reaches p < 0.05: `mde` is `null` there too, and the note says how low p can go instead (`with 3 changed tasks the sign-flip p cannot go below 0.25: no - effect can reach p < 0.05 here`). A row whose interval and p disagree (the + effect can reach p < 0.05 here`). Holm's correction can lift that floor: a + gate or score row's `p_holm_floor` is its `p_holm` with its p put at that + floor and the other p of the family as observed (in every `--json` row; + `null` on a descriptive row or one with every task tied), and where it is not below + 0.05, `mde` is `null` and the note names Holm instead (`with Holm over 2 + rows, p_holm cannot go below 0.0625 here: no effect can reach p_holm < + 0.05`); a family of one row is the raw rule. A row whose interval and p disagree (the interval excludes 0 while p is 0.05 or more, or includes 0 while p is below 0.05; for gate and score that p is `p_holm`, the one that decides them, and the note quotes it) has `interval_p_disagree` `true` and a note: they come from two diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index 3470933cd..fcd68b66e 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -45,7 +45,9 @@ delta a two-sided 5% test finds with 80% power at the row's spread, ``(t_0.975 + t_0.80) * sd / sqrt(n)`` over the per-task deltas, the t quantiles at ``n - 1`` degrees of freedom (in every row; ``None`` for one task or no spread). The sign-flip p over k changed tasks cannot go below ``2 / 2**k``, so at 5 or fewer no effect of any -size reaches p < 0.05: ``mde`` is ``None`` there too, and the note says how low p can go instead. A row whose +size reaches p < 0.05: ``mde`` is ``None`` there too, and the note says how low p can go instead. ``mde`` is ``None`` +too where a gate or score row's ``p_holm_floor`` (its ``p_holm`` with its p at that floor and the family's other +p as observed, :mod:`.compare_reach`) is not below 0.05, and the note names Holm. A row whose interval and p disagree (the interval excludes 0 while p is 0.05 or more, or includes it while p is below; for gate and score the p is ``p_holm``, the one that decides them) has ``interval_p_disagree``: the two come from different methods, and for gate and score ``p_holm`` is the one to read, since the percentile @@ -96,18 +98,16 @@ import itertools import json -import math import random import statistics from collections import defaultdict from pathlib import Path from typing import Any -from scipy import stats as sp_stats - from dokimi.context.consistency import Run, order_distance, pass_hat, run_from_events from dokimi.eval.stats import bootstrap_ci, correct_family, mean_ci +from .compare_reach import ALPHA, holm_floor, min_detectable from .compare_tasks import CompareError, select_tasks from .process_counts import as_measure, process_order, process_values from .report import (load_cells, mismatch_warnings, model_matches, model_mismatch, reading_cells, resolved_models, @@ -121,9 +121,6 @@ EPS = 1e-9 #: Measures Holm's correction runs over. PRIMARY = ("gate", "score") -#: The level the notes read p and the interval at (two-sided), and the power ``mde`` is sized for. -ALPHA = 0.05 -POWER = 0.80 #: Descriptive measures compared only when both arms report them for some shared task (an older batch has none). REPORTED = ("process:", "counter:", "first_prompt_tokens") #: Batch stamp keys that must agree between the arms of a comparison; ``tree`` is shown but may differ (another @@ -346,12 +343,17 @@ def _check_models(models: list[str] | None, rows: list[dict[str, Any]], unpaired def _holm(rows: list[dict[str, Any]]) -> None: - """``p_holm`` on the gate and score rows, corrected as one family; their ``interval_p_disagree`` reads it.""" + """``p_holm`` on the gate and score rows, corrected as one family; their ``interval_p_disagree`` reads it. + ``p_holm_floor`` is a row's ``p_holm`` at its p floor: where it cannot go below :data:`ALPHA`, ``mde`` is None.""" primary = [r for r in rows if r["measure"] in PRIMARY] - for row, (p, below) in zip(primary, correct_family([r["p"] for r in primary]), strict=True): + ps = [r["p"] for r in primary] + for i, (row, (p, below)) in enumerate(zip(primary, correct_family(ps), strict=True)): d = row["delta"] row["p_holm"] = p row["interval_p_disagree"] = d["lo"] is not None and below is not None and (d["lo"] <= 0 <= d["hi"]) == below + if p is not None: + row["p_holm_floor"], reachable = holm_floor(ps, i, min_sign_flip_p(row["up"] + row["down"])) + row["mde"] = row["mde"] if reachable else None def _compare_model(treatment: str, model: str, base_vals: dict[tuple[str, str], dict[str, Any]], @@ -521,8 +523,8 @@ def paired_summary(pairs: list[dict[str, Any]], measure: str = "gate", seed: int permutation test. The interval is a percentile bootstrap for the bounded gate and score shares and a t interval (:func:`~dokimi.eval.stats.mean_ci`) for the continuous measures, where the bootstrap under-covers at these task counts; one task has no interval (``lo``/``hi`` None), not a zero-width one. - ``mde`` and ``interval_p_disagree`` are the reading aids of the module docstring; the flag reads the raw p - here, and :func:`compare` sets it again on a gate or score row once Holm's p is known. + ``mde`` and ``interval_p_disagree`` are the reading aids of the module docstring; both read the raw p here, + and :func:`compare` revisits both on a gate or score row once Holm's p is known. """ deltas = [p["delta"] for p in pairs] ci = (bootstrap_ci if measure in PRIMARY else mean_ci)(deltas, seed=seed) @@ -537,6 +539,7 @@ def paired_summary(pairs: list[dict[str, Any]], measure: str = "gate", seed: int "delta": {"mean": ci["mean"], "lo": lo, "hi": hi}, "p": p, "p_holm": None, + "p_holm_floor": None, "mde": _round(min_detectable(sd, len(deltas))) if sd > EPS and reachable else None, "interval_p_disagree": lo is not None and hi is not None and p is not None and (lo <= 0 <= hi) == (p < ALPHA), "up": sum(d > EPS for d in deltas), @@ -547,13 +550,6 @@ def paired_summary(pairs: list[dict[str, Any]], measure: str = "gate", seed: int } -def min_detectable(sd: float, n: int) -> float: - """The smallest mean delta a two-sided test at :data:`ALPHA` finds with :data:`POWER` over ``n`` tasks of - spread ``sd``. The quantiles are t at ``n - 1`` degrees of freedom, as the interval of ``mean_ci`` is.""" - df = n - 1 - return (sp_stats.t.ppf(1 - ALPHA / 2, df) + sp_stats.t.ppf(POWER, df)) * sd / math.sqrt(n) - - def min_sign_flip_p(changed: int) -> float: """The smallest p :func:`sign_flip_p` can give over ``changed`` non-zero deltas, every sign the same: only that pattern and its mirror reach the observed ``|sum|``, ``2 / 2**changed`` exact, ``1 / (N + 1)`` Monte Carlo.""" @@ -584,6 +580,7 @@ def format_compare(summary: dict[str, Any]) -> str: for row in summary["comparisons"]: groups[(row["treatment"], row["model"])].append(row) width = max([len(r["measure"]) for r in summary["comparisons"]] + [7]) + family = sum(r["p_holm"] is not None for r in summary["comparisons"]) # the rows Holm corrected together for (treatment, model), rows in groups.items(): lines += ["", f"{treatment} vs {summary['baseline']}, model {model}", f" {'measure':<{width}} {'tasks':>5} {'baseline':>9} {'treatment':>9} " @@ -602,7 +599,7 @@ def format_compare(summary: dict[str, Any]) -> str: f"{t['task']} {t['delta']:+.2f}" for t in sorted(moved, key=lambda t: t["delta"]))) if r["headroom"]: lines.append(f" {r['measure']} {_headroom_note(r)}") - lines += _reading_notes(rows) + lines += _reading_notes(rows, family) if summary["unpaired"]: shown = summary["unpaired"][:10] lines.append("unpaired: " + ", ".join( @@ -616,8 +613,8 @@ def format_compare(summary: dict[str, Any]) -> str: return "\n".join(lines) -def _reading_notes(rows: list[dict[str, Any]]) -> list[str]: - """One comparison's notes against a misreading (module docstring); no number changes.""" +def _reading_notes(rows: list[dict[str, Any]], family: int) -> list[str]: + """One comparison's notes against a misreading (module docstring), ``family`` Holm's rows; no number changes.""" out = [] for r in rows: d, primary = r["delta"], r["measure"] in PRIMARY @@ -629,7 +626,7 @@ def _reading_notes(rows: list[dict[str, Any]]) -> list[str]: f"{'includes' if inside else 'excludes'} 0 but {name} {r[name]:.4g} is {'<' if inside else '≥'} " f"{ALPHA}; they come from two methods, {how} and a sign-flip test" + ("; read p_holm" if primary else "")) - elif primary and (note := _inconclusive_note(r)): + elif primary and (note := _inconclusive_note(r, family)): out.append(note) low = [r["measure"] for r in rows if r["measure"] not in PRIMARY and r["p"] is not None and r["p"] < ALPHA] if low: @@ -638,7 +635,7 @@ def _reading_notes(rows: list[dict[str, Any]]) -> list[str]: return out -def _inconclusive_note(r: dict[str, Any]) -> str | None: +def _inconclusive_note(r: dict[str, Any], family: int) -> str | None: """The note of a gate or score row of 2+ tasks whose interval includes 0: its ``mde``, or why it has none.""" d = r["delta"] if d["lo"] is None or not d["lo"] <= 0 <= d["hi"]: @@ -647,11 +644,14 @@ def _inconclusive_note(r: dict[str, Any]) -> str | None: if r["mde"] is not None: return head + (f"with this spread the smallest effect 80% power detects is about ±{r['mde']:.3f} " f"({r['n_tasks']} tasks)") - changed = r["up"] + r["down"] - if r["p"] is None or (floor := min_sign_flip_p(changed)) < ALPHA: + changed, holm = r["up"] + r["down"], r["p_holm_floor"] + if r["p"] is None or (floor := min_sign_flip_p(changed)) < ALPHA and holm < ALPHA: return None # every delta tied, or no spread - return head + (f"with {changed} changed task{'' if changed == 1 else 's'} the sign-flip p cannot go below " - f"{floor:.4g}: no effect can reach p < {ALPHA} here") + if floor >= ALPHA: + return head + (f"with {changed} changed task{'' if changed == 1 else 's'} the sign-flip p cannot go below " + f"{floor:.4g}: no effect can reach p < {ALPHA} here") + return head + (f"with Holm over {family} rows, p_holm cannot go below {holm:.4g} here: no effect can reach " + f"p_holm < {ALPHA}") def _headroom_note(row: dict[str, Any]) -> str: diff --git a/dokimi/dokimi/repos/compare_reach.py b/dokimi/dokimi/repos/compare_reach.py new file mode 100644 index 000000000..fbbba75d2 --- /dev/null +++ b/dokimi/dokimi/repos/compare_reach.py @@ -0,0 +1,30 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""How small an effect a ``repo compare`` row can find: ``mde``, and the lowest ``p_holm`` Holm's family leaves a +gate or score row (:mod:`.compare`).""" + +from __future__ import annotations + +import math + +from scipy import stats as sp_stats + +from dokimi.eval.stats import correct_family + +#: The level the notes read p and the interval at (two-sided), and the power ``mde`` is sized for. +ALPHA = 0.05 +POWER = 0.80 + + +def min_detectable(sd: float, n: int) -> float: + """The smallest mean delta a two-sided test at :data:`ALPHA` finds with :data:`POWER` over ``n`` tasks of + spread ``sd``. The quantiles are t at ``n - 1`` degrees of freedom, as the interval of ``mean_ci`` is.""" + df = n - 1 + return (sp_stats.t.ppf(1 - ALPHA / 2, df) + sp_stats.t.ppf(POWER, df)) * sd / math.sqrt(n) + + +def holm_floor(pvals: list[float | None], i: int, floor: float) -> tuple[float | None, bool | None]: + """Holm's p of row ``i`` of the family ``pvals`` with its p put at ``floor``, the lowest it can go, and the + others as observed; and whether that p is below :data:`ALPHA`. Where it is not, no effect can be.""" + return correct_family([*pvals[:i], floor, *pvals[i + 1:]])[i] diff --git a/dokimi/tests/test_repo_compare_reading.py b/dokimi/tests/test_repo_compare_reading.py index 99ffa8fbb..ea0f5730b 100644 --- a/dokimi/tests/test_repo_compare_reading.py +++ b/dokimi/tests/test_repo_compare_reading.py @@ -6,6 +6,7 @@ from __future__ import annotations import json +import statistics from pathlib import Path import pytest @@ -229,6 +230,45 @@ def test_no_mde_where_the_sign_flip_p_cannot_reach_005(tmp_path, deltas, changed assert "80% power" not in format_compare(s) +SIX_MIXED = [0.5, -0.5, 0.25, -0.25, 0.5, -0.5] # six changed tasks: a sign-flip p floor of 2/64 +HOLM_NOTE = (" score inconclusive: the interval includes 0; with Holm over 2 rows, p_holm cannot go below {} here: " + "no effect can reach p_holm < 0.05") + + +def _two_score_arms(b: Path, raw: list[float], other: list[float]) -> None: + for i, (r, o) in enumerate(zip(raw, other, strict=True)): + _cell(b, f"review_{i}", "guidance:none", 1, score=0.5) + _cell(b, f"review_{i}", "guidance:raw", 1, score=0.5 + r) + _cell(b, f"review_{i}", "guidance:other", 1, score=0.5 + o) + + +def test_no_mde_where_holm_over_the_family_keeps_p_holm_from_005(tmp_path): + # each row alone could reach p 2/64; Holm over two rows doubles the smaller, so p_holm cannot go below 0.0625 + _two_score_arms(tmp_path / "b", SIX_MIXED, SIX_MIXED) + s = compare([tmp_path / "b"], "guidance:none") + for arm in ("guidance:raw", "guidance:other"): + score = _row(s, "score", treatment=arm) + assert score["mde"] is None and score["p_holm_floor"] == 0.0625 + assert score["delta"]["lo"] < 0 < score["delta"]["hi"] + text = format_compare(s) + assert text.splitlines().count(HOLM_NOTE.format("0.0625")) == 2 and "80% power" not in text + + +def test_a_row_holm_still_lets_reach_005_keeps_its_mde(tmp_path): + # eight changed tasks: a floor of 2/256, which Holm over two rows takes to 0.0156; six: 2/64 to 0.0625 + eight = [0.5, -0.5, 0.25, -0.25, 0.5, -0.5, 0.25, -0.25] + _two_score_arms(tmp_path / "b", eight, [*SIX_MIXED, 0.0, 0.0]) + s = compare([tmp_path / "b"], "guidance:none") + raw, other = _row(s, "score"), _row(s, "score", treatment="guidance:other") + assert raw["p_holm_floor"] == 0.0156 and raw["delta"]["lo"] < 0 < raw["delta"]["hi"] + assert raw["mde"] == pytest.approx(min_detectable(statistics.stdev(eight), 8), abs=1e-6) + assert other["p_holm_floor"] == 0.0625 and other["mde"] is None + lines = format_compare(s).splitlines() + assert (f" score inconclusive: the interval includes 0; with this spread the smallest effect 80% power " + f"detects is about ±{raw['mde']:.3f} (8 tasks)") in lines + assert HOLM_NOTE.format("0.0625") in lines + + def test_a_gate_row_of_two_changed_tasks_has_no_mde(tmp_path): b = tmp_path / "b" for task, base_ok in (("t1", False), ("t2", False), ("t3", True)): # gate deltas [1, 1, 0] From fda7c07c681af762e317030c0a80ffd32af2382f Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Fri, 30 Oct 2026 11:08:01 +0000 Subject: [PATCH 16/16] docs(repo): name mde an uncorrected t-based estimate, not Holm power MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review comments 4422911586 and 4422911592: `mde` is computed as (t(0.975, n-1) + t(0.80, n-1)) * sd / sqrt(n), the effect a one-sample t test at the uncorrected 5% level detects with 80% power. It is not the power of the sign-flip test nor of the Holm-corrected decision: with two rows and the other row's p at 1, Holm needs this row's raw p below 0.025, and the figure is still the one-row value. `p_holm_floor` only decides whether `mde` is given. The inconclusive note now reads "an uncorrected t-test estimate puts the effect 80% power detects at this spread at about ±X (N tasks), before Holm and not for the sign-flip test". The `compare.py` module docstring, `compare_reach.min_detectable`, contract section 9, architecture.md and the changelog fragment say the same; the contract says why no Holm-calibrated figure is given (it would depend on the other rows' observed p, which a planning figure cannot know). How `mde` is computed and every JSON key and value are unchanged; the two tests that assert the note text take the new wording. Signed-off-by: Anton Volkov --- ...61029_150000_antonwolfy_compare-reading.md | 5 +-- dokimi/docs/architecture.md | 4 +-- dokimi/docs/repo-evaluation-contract.md | 17 ++++++---- dokimi/dokimi/repos/compare.py | 31 ++++++++++--------- dokimi/dokimi/repos/compare_reach.py | 10 +++--- dokimi/tests/test_repo_compare_reading.py | 11 ++++--- 6 files changed, 45 insertions(+), 33 deletions(-) diff --git a/dokimi/changelog.d/20261029_150000_antonwolfy_compare-reading.md b/dokimi/changelog.d/20261029_150000_antonwolfy_compare-reading.md index 609049dc0..8cb62ce25 100644 --- a/dokimi/changelog.d/20261029_150000_antonwolfy_compare-reading.md +++ b/dokimi/changelog.d/20261029_150000_antonwolfy_compare-reading.md @@ -13,8 +13,9 @@ not compared, and both name it (`task_filter.dropped_arms`, `not compared, no finished cell among the --task selection: `). - **Reading notes under a `repo compare` comparison** — a gate or score row of 2+ tasks whose interval - includes 0 says it is inconclusive and gives `mde`, the smallest effect 80% power detects at that - spread (`mde` is in every `--json` row). With 5 or fewer changed tasks the sign-flip p cannot go + includes 0 says it is inconclusive and gives `mde`, an approximate, uncorrected t-based estimate of + the effect a 5% one-sample t test detects with 80% power at that spread, a planning figure and not + the power of the sign-flip test or of the Holm-corrected decision (`mde` is in every `--json` row). With 5 or fewer changed tasks the sign-flip p cannot go below 0.05 (its floor is `2 / 2^k` for k changed tasks), so `mde` is `null` and the note says how low p can go instead. The same holds where Holm's correction keeps a gate or score row's `p_holm_floor` (its `p_holm` with its p at that floor, the other rows of the family as observed) at diff --git a/dokimi/docs/architecture.md b/dokimi/docs/architecture.md index 2fcf45bfa..6ef5d08bd 100644 --- a/dokimi/docs/architecture.md +++ b/dokimi/docs/architecture.md @@ -339,8 +339,8 @@ next release. over tasks (bootstrap for gate and score, t for the rest), a sign-flip permutation p and Holm's correction. `dokimi/repos/compare_tasks.py` holds what its `--task` selection keeps; `dokimi/repos/compare_reach.py` how small - an effect a row can find (`mde`, and the lowest `p_holm` Holm's family - leaves it). + an effect a row can find (`mde`, an uncorrected t-based estimate, and the + lowest `p_holm` Holm's family leaves it). - `dokimi/repos/selfcheck.py` — `repo tasks check`: on each task `grade` must fail the untouched workspace and pass it after `oracle`, and an isolation probe must not see the tasks or the source from the agent's diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index e1f4fd795..b15eecc3b 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1353,11 +1353,15 @@ python cli.py repo compare repo-runs/replay --baseline guidance:none --metric re rest are descriptive. Three notes under a comparison keep a row from being misread, and change no delta, interval or p. A gate or score row of 2+ tasks whose interval includes 0 is inconclusive, not evidence of no effect: the note - gives `mde`, the smallest mean delta a two-sided 5% test finds with 80% power - at that spread, `(t(0.975, n-1) + t(0.80, n-1)) * sd / sqrt(n)` over the - per-task deltas, the t quantiles at `n-1` degrees of freedom (every row has - `mde` in `--json`; `null` for one task or deltas with - no spread, and then no note). The sign-flip p over k changed (non-tied) + gives `mde`, a planning figure: an approximate, uncorrected t-based estimate, + `(t(0.975, n-1) + t(0.80, n-1)) * sd / sqrt(n)` over the per-task deltas, of + the mean delta a one-sample t test at the uncorrected two-sided 5% level + detects with 80% power at that spread, the t quantiles at `n-1` degrees of + freedom (every row has `mde` in `--json`; `null` for one task or deltas with + no spread, and then no note). It is not the power of the sign-flip test nor + of the Holm-corrected decision; a Holm-calibrated figure would depend on the + other rows' observed p, which a figure for planning the next run's task count + cannot know. The sign-flip p over k changed (non-tied) tasks cannot go below `2 / 2^k`, so with 5 or fewer no effect of any size reaches p < 0.05: `mde` is `null` there too, and the note says how low p can go instead (`with 3 changed tasks the sign-flip p cannot go below 0.25: no @@ -1367,7 +1371,8 @@ python cli.py repo compare repo-runs/replay --baseline guidance:none --metric re `null` on a descriptive row or one with every task tied), and where it is not below 0.05, `mde` is `null` and the note names Holm instead (`with Holm over 2 rows, p_holm cannot go below 0.0625 here: no effect can reach p_holm < - 0.05`); a family of one row is the raw rule. A row whose interval and p disagree (the + 0.05`); a family of one row is the raw rule. `p_holm_floor` decides only + whether `mde` is given, not its size. A row whose interval and p disagree (the interval excludes 0 while p is 0.05 or more, or includes 0 while p is below 0.05; for gate and score that p is `p_holm`, the one that decides them, and the note quotes it) has `interval_p_disagree` `true` and a note: they come from two diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index fcd68b66e..fc15552bd 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -40,19 +40,20 @@ the gate and score comparisons of one invocation; cost, turns and metrics are descriptive and keep their raw p. -Three notes keep a row from being misread; none changes the delta, its interval or p. A gate or score row of -2+ tasks whose interval includes 0 is inconclusive, not "no effect": the note gives ``mde``, the smallest mean -delta a two-sided 5% test finds with 80% power at the row's spread, ``(t_0.975 + t_0.80) * sd / sqrt(n)`` over -the per-task deltas, the t quantiles at ``n - 1`` degrees of freedom (in every row; ``None`` for one task or no -spread). The sign-flip p over k changed tasks cannot go below ``2 / 2**k``, so at 5 or fewer no effect of any -size reaches p < 0.05: ``mde`` is ``None`` there too, and the note says how low p can go instead. ``mde`` is ``None`` -too where a gate or score row's ``p_holm_floor`` (its ``p_holm`` with its p at that floor and the family's other -p as observed, :mod:`.compare_reach`) is not below 0.05, and the note names Holm. A row whose -interval and p disagree (the interval excludes 0 while p is 0.05 or more, or includes it -while p is below; for gate and score the p is ``p_holm``, the one that decides them) has ``interval_p_disagree``: -the two come from different methods, and for gate and score ``p_holm`` is the one to read, since the percentile -bootstrap runs narrow over few tasks (the inconclusive note then gives way to this one). A descriptive row -with p below 0.05 is named on a line saying its p is uncorrected and outside Holm's family. +Three notes keep a row from being misread; none changes the delta, its interval or p. A gate or score row of 2+ tasks +whose interval includes 0 is inconclusive, not "no effect": the note gives ``mde``, a planning figure: an +approximate, uncorrected t-based estimate of the mean delta a two-sided 5% one-sample t test detects with 80% power +at the row's spread, ``(t_0.975 + t_0.80) * sd / sqrt(n)`` over the per-task deltas, t at ``n - 1`` degrees of +freedom (in every row; ``None`` for one task or no spread), not the power of the sign-flip test nor of the +Holm-corrected decision. The sign-flip p over k changed tasks cannot go below ``2 / 2**k``, so at 5 or fewer no +effect of any size reaches p < 0.05: ``mde`` is ``None`` there too, and the note says how low p can go instead. +``mde`` is ``None`` too where a gate or score row's ``p_holm_floor`` (its ``p_holm`` with its p at that floor and the +family's other p as observed, :mod:`.compare_reach`) is not below 0.05, and the note names Holm: that floor decides +whether ``mde`` is given, not its size. A row whose interval and p disagree (the interval excludes 0 while p is 0.05 +or more, or includes it while p is below; for gate and score the p is ``p_holm``, the one that decides them) has +``interval_p_disagree``: the two come from different methods, and for gate and score ``p_holm`` is the one to read, +since the percentile bootstrap runs narrow over few tasks (the inconclusive note then gives way to this one). A +descriptive row with p below 0.05 is named on a line saying its p is uncorrected and outside Holm's family. ``--task`` limits a comparison to some task ids: the others are left out, not listed as unpaired, and the summary's ``task_filter`` records the selection; without ``--arm``, its ``dropped_arms`` names an arm with no @@ -642,8 +643,8 @@ def _inconclusive_note(r: dict[str, Any], family: int) -> str | None: return None head = f" {r['measure']} inconclusive: the interval includes 0; " if r["mde"] is not None: - return head + (f"with this spread the smallest effect 80% power detects is about ±{r['mde']:.3f} " - f"({r['n_tasks']} tasks)") + return head + (f"an uncorrected t-test estimate puts the effect 80% power detects at this spread at about " + f"±{r['mde']:.3f} ({r['n_tasks']} tasks), before Holm and not for the sign-flip test") changed, holm = r["up"] + r["down"], r["p_holm_floor"] if r["p"] is None or (floor := min_sign_flip_p(changed)) < ALPHA and holm < ALPHA: return None # every delta tied, or no spread diff --git a/dokimi/dokimi/repos/compare_reach.py b/dokimi/dokimi/repos/compare_reach.py index fbbba75d2..2f594a026 100644 --- a/dokimi/dokimi/repos/compare_reach.py +++ b/dokimi/dokimi/repos/compare_reach.py @@ -1,8 +1,8 @@ # Copyright (C) 2026 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -"""How small an effect a ``repo compare`` row can find: ``mde``, and the lowest ``p_holm`` Holm's family leaves a -gate or score row (:mod:`.compare`).""" +"""How small an effect a ``repo compare`` row can find: ``mde``, an uncorrected t-based planning estimate, and the +lowest ``p_holm`` Holm's family leaves a gate or score row (:mod:`.compare`).""" from __future__ import annotations @@ -18,8 +18,10 @@ def min_detectable(sd: float, n: int) -> float: - """The smallest mean delta a two-sided test at :data:`ALPHA` finds with :data:`POWER` over ``n`` tasks of - spread ``sd``. The quantiles are t at ``n - 1`` degrees of freedom, as the interval of ``mean_ci`` is.""" + """An approximate, uncorrected t-based estimate of the mean delta a two-sided one-sample t test at :data:`ALPHA` + detects with :data:`POWER` over ``n`` tasks of spread ``sd``, a planning figure: not the power of the sign-flip + test nor of the Holm-corrected decision. The quantiles are t at ``n - 1`` degrees of freedom, as the interval + of ``mean_ci`` is.""" df = n - 1 return (sp_stats.t.ppf(1 - ALPHA / 2, df) + sp_stats.t.ppf(POWER, df)) * sd / math.sqrt(n) diff --git a/dokimi/tests/test_repo_compare_reading.py b/dokimi/tests/test_repo_compare_reading.py index ea0f5730b..d098aa3ea 100644 --- a/dokimi/tests/test_repo_compare_reading.py +++ b/dokimi/tests/test_repo_compare_reading.py @@ -200,6 +200,10 @@ def _score_tasks(b: Path, deltas: list[float], base: float) -> None: _cell(b, f"review_{i}", "guidance:raw", 1, score=base + d) +MDE_NOTE = (" score inconclusive: the interval includes 0; an uncorrected t-test estimate puts the effect 80% power " + "detects at this spread at about ±{:.3f} ({} tasks), before Holm and not for the sign-flip test") + + def test_an_interval_across_zero_is_inconclusive_with_its_mde(tmp_path): # six changed tasks: the sign-flip p can go down to 2/64, so 80% power names an effect _score_tasks(tmp_path / "b", [0.5, -0.5, 0.25, -0.25, 0.5, -0.5], 0.5) # sample SD 0.474342 @@ -208,8 +212,7 @@ def test_an_interval_across_zero_is_inconclusive_with_its_mde(tmp_path): assert score["mde"] == pytest.approx(min_detectable(0.474342, 6), abs=1e-6) and score["mde"] > 0.6 assert score["interval_p_disagree"] is False text = format_compare(s) - assert (f" score inconclusive: the interval includes 0; with this spread the smallest effect 80% power " - f"detects is about ±{score['mde']:.3f} (6 tasks)") in text.splitlines() + assert MDE_NOTE.format(score["mde"], 6) in text.splitlines() assert "no effect" not in text _score_tasks(tmp_path / "one", [0.5], 0.5) one = compare([tmp_path / "one"], "guidance:none") @@ -264,8 +267,8 @@ def test_a_row_holm_still_lets_reach_005_keeps_its_mde(tmp_path): assert raw["mde"] == pytest.approx(min_detectable(statistics.stdev(eight), 8), abs=1e-6) assert other["p_holm_floor"] == 0.0625 and other["mde"] is None lines = format_compare(s).splitlines() - assert (f" score inconclusive: the interval includes 0; with this spread the smallest effect 80% power " - f"detects is about ±{raw['mde']:.3f} (8 tasks)") in lines + # the figure is the one-row t estimate: Holm (raw p < 0.025 here) only decides that it is given + assert MDE_NOTE.format(raw["mde"], 8) in lines assert HOLM_NOTE.format("0.0625") in lines