diff --git a/dokimi/changelog.d/20261029_150000_antonwolfy_compare-reading.md b/dokimi/changelog.d/20261029_150000_antonwolfy_compare-reading.md new file mode 100644 index 000000000..8cb62ce25 --- /dev/null +++ b/dokimi/changelog.d/20261029_150000_antonwolfy_compare-reading.md @@ -0,0 +1,26 @@ +### Added + +- **`repo run` records each cell's start context, and `repo compare` compares it** — `result.json` + `agent.first_prompt_tokens` is what the first main-thread model call was sent (input plus cache + creation plus cache read tokens, one call per message id, as `context trace` reads it): start-up and + the task. It is `null` for a trace without that call and for the `text` format. `repo compare` adds a + descriptive `first_prompt_tokens` row; a batch written before the field has no such row. +- **`repo compare --task ID`** — repeatable: compares only those task ids. An id no finished cell of the + batches has exits 1 and names it; the tasks left out are not listed as unpaired, nor their unfinished + cells as missing (a cell's task is the one its `meta.json` records), and the `--json` + summary (`task_filter`) and the text header (`tasks: N of M (--task)`, M the tasks with a finished + cell) record the selection. Without `--arm`, an arm with no finished cell among the selection is + not compared, and both name it (`task_filter.dropped_arms`, `not compared, no finished cell among + the --task selection: `). +- **Reading notes under a `repo compare` comparison** — a gate or score row of 2+ tasks whose interval + includes 0 says it is inconclusive and gives `mde`, an approximate, uncorrected t-based estimate of + the effect a 5% one-sample t test detects with 80% power at that spread, a planning figure and not + the power of the sign-flip test or of the Holm-corrected decision (`mde` is in every `--json` row). With 5 or fewer changed tasks the sign-flip p cannot go + below 0.05 (its floor is `2 / 2^k` for k changed tasks), so `mde` is `null` and the note says how + low p can go instead. The same holds where Holm's correction keeps a gate or score row's + `p_holm_floor` (its `p_holm` with its p at that floor, the other rows of the family as observed) at + 0.05 or above, as with two such rows of six changed tasks each (`p_holm` 0.0625 at best); the note + then says `with Holm over N rows, p_holm cannot go below ...`. A row whose interval and p disagree says so and why + (`interval_p_disagree`); for gate and score it compares the interval with Holm's `p_holm`, quotes + it and says it is the one to read, and a descriptive row uses its raw p; descriptive rows with + p below 0.05 are named as uncorrected and outside Holm's family. No delta, interval or p changes. diff --git a/dokimi/docs/architecture.md b/dokimi/docs/architecture.md index 6dd46dc4f..6ef5d08bd 100644 --- a/dokimi/docs/architecture.md +++ b/dokimi/docs/architecture.md @@ -337,7 +337,10 @@ next release. share, score, cost, turns, chosen `grade.json` metrics, `result.json` `process` counts), with an interval over tasks (bootstrap for gate and score, t for the rest), a sign-flip - permutation p and Holm's correction. + permutation p and Holm's correction. `dokimi/repos/compare_tasks.py` holds + what its `--task` selection keeps; `dokimi/repos/compare_reach.py` how small + an effect a row can find (`mde`, an uncorrected t-based estimate, and the + lowest `p_holm` Holm's family leaves it). - `dokimi/repos/selfcheck.py` — `repo tasks check`: on each task `grade` must fail the untouched workspace and pass it after `oracle`, and an isolation probe must not see the tasks or the source from the agent's diff --git a/dokimi/docs/context-window.md b/dokimi/docs/context-window.md index 1469a6ecf..f85438eca 100644 --- a/dokimi/docs/context-window.md +++ b/dokimi/docs/context-window.md @@ -79,3 +79,7 @@ point `CLAUDE_CONFIG_DIR` at an empty directory to leave it out. The `surface` field of `--json` (the tool, skill and agent counts, and the names of the MCP servers and loaded plugins) shows what start-up offered, so two runs that differ there are not the same condition. + +`repo run` records the size of each cell's **first prompt**, in tokens, as +`result.json` `agent.first_prompt_tokens`, and `repo compare` compares it between arms as +the descriptive measure `first_prompt_tokens`. diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index 3778ecf3d..1bbc56081 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -10263,6 +10263,22 @@ "position": null, "required": false, "type": null + }, + { + "action": "_AppendAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--task", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null } ], "path": "dokimi repo compare", diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 5a05bba8c..b15eecc3b 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -419,6 +419,12 @@ reads the upstream base (`base_sha`) and the workspace start commit Code's `modelUsage`), `main` when the harness reported the parent's only, and `null` when it reported none. `subagent_calls` counts the parent's and subagents' `Agent`/`Task` calls; `null` when the harness wrote no result record. +`first_prompt_tokens` is what the first main-thread model call was sent +(`input + cache_creation + cache_read` tokens of that call, counted once per +message id, as `context trace` reads it): everything start-up loaded plus the +task, so two arms that differ only in guidance differ there by what start-up +loaded of it. It is `null` when the trace has no main-thread call with +`usage`, and for a trace format that carries none (`text`). `result.json` `process` holds what the framework counts in the events, the parent's and subagents' together, so a grader need not: @@ -1334,15 +1340,58 @@ python cli.py repo compare repo-runs/replay --baseline guidance:none --metric re - `repo compare` reduces cells to one value per (task, arm, model) first: repetitions share a task's difficulty and are not paired across arms, so the task is the unit. Per treatment arm, model and measure (gate share, score, - cost, turns, each `--metric` from `grade.json`, and each `process` count a + cost, turns, `first_prompt_tokens`, each `--metric` from `grade.json`, and each `process` count a compared cell reports as `process:` or `counter:`) it gives the mean per-task delta against the baseline with a 95% interval over tasks (a bootstrap for - gate and score, a t interval for cost, turns, metrics and process counts, `-` for a single + gate and score, a t interval for the descriptive measures, `-` for a single task), the tasks up, down and tied, and a two-sided sign-flip permutation p (exact up to 16 changed tasks; at one cell per task and arm it equals McNemar's exact test). - A `process` count only one arm reports for the shared tasks of a - (treatment, model) is not compared; a warning names it. Holm's correction runs over the gate and score comparisons of one call; the - rest are descriptive. A gate or score comparison carries `headroom`: + A `process` count or `first_prompt_tokens` only one arm reports for the shared tasks of a + (treatment, model) is not compared; a warning names it. When neither arm + reports it (a batch run before `repo run` recorded it), the row is absent. + Holm's correction runs over the gate and score comparisons of one call; the + rest are descriptive. Three notes under a comparison keep a row from being + misread, and change no delta, interval or p. A gate or score row of 2+ tasks whose + interval includes 0 is inconclusive, not evidence of no effect: the note + gives `mde`, a planning figure: an approximate, uncorrected t-based estimate, + `(t(0.975, n-1) + t(0.80, n-1)) * sd / sqrt(n)` over the per-task deltas, of + the mean delta a one-sample t test at the uncorrected two-sided 5% level + detects with 80% power at that spread, the t quantiles at `n-1` degrees of + freedom (every row has `mde` in `--json`; `null` for one task or deltas with + no spread, and then no note). It is not the power of the sign-flip test nor + of the Holm-corrected decision; a Holm-calibrated figure would depend on the + other rows' observed p, which a figure for planning the next run's task count + cannot know. The sign-flip p over k changed (non-tied) + tasks cannot go below `2 / 2^k`, so with 5 or fewer no effect of any size + reaches p < 0.05: `mde` is `null` there too, and the note says how low p can + go instead (`with 3 changed tasks the sign-flip p cannot go below 0.25: no + effect can reach p < 0.05 here`). Holm's correction can lift that floor: a + gate or score row's `p_holm_floor` is its `p_holm` with its p put at that + floor and the other p of the family as observed (in every `--json` row; + `null` on a descriptive row or one with every task tied), and where it is not below + 0.05, `mde` is `null` and the note names Holm instead (`with Holm over 2 + rows, p_holm cannot go below 0.0625 here: no effect can reach p_holm < + 0.05`); a family of one row is the raw rule. `p_holm_floor` decides only + whether `mde` is given, not its size. A row whose interval and p disagree (the + interval excludes 0 while p is 0.05 or more, or includes 0 while p is below + 0.05; for gate and score that p is `p_holm`, the one that decides them, and the + note quotes it) has `interval_p_disagree` `true` and a note: they come from two + methods, and for gate and score `p_holm` is the one to read, since a percentile + bootstrap runs narrow over few tasks; the inconclusive note gives way to + it. Descriptive rows with p below 0.05 are named on one line: their p is + uncorrected and outside Holm's family. `--task ID` (repeatable) compares only + those tasks: an id no finished cell of the batches has is refused, the tasks + left out are not listed as unpaired nor their cells as missing (a missing + cell's task is the one its `meta.json` records, else, for a cell only + `plan.json` names, the longest task id of the batch's finished cells that its + name starts with followed by `__`; a cell of neither is missing only without + `--task`), the `--json` summary records the + selection as `task_filter` (`tasks`, and `of`, the number of tasks of the + batches that have a finished cell, the ids `--task` can name, and + `dropped_arms`: without `--arm`, the arms with no finished cell among the + selection, which drop out of the comparison, `[]` with `--arm`; `null` + without `--task`), and the text header says `tasks: N of M (--task)` and, + under it, `not compared, no finished cell among the --task selection: `. A gate or score comparison carries `headroom`: `ceiling` (both arms at 1.0 or the task's `score.max` on every paired task), `floor` (both at 0 on every one), `near_ceiling` (the baseline at the top on all but at most one of 2+ tasks and the treatment nowhere below diff --git a/dokimi/dokimi/commands/repo_run.py b/dokimi/dokimi/commands/repo_run.py index 585a7a544..f1b8d5604 100644 --- a/dokimi/dokimi/commands/repo_run.py +++ b/dokimi/dokimi/commands/repo_run.py @@ -411,7 +411,7 @@ def cmd_repo_compare(args: argparse.Namespace) -> None: arms = [a for spec in args.arm for a in spec.split(",") if a] if args.arm else None try: summary = compare(dirs, args.baseline, arms, args.model, tuple(args.metric or ()), - pass_k=args.pass_k, path_stability=args.path_stability) + pass_k=args.pass_k, path_stability=args.path_stability, tasks=args.task) except (CompareError, CellFileError) as exc: print(f"✗ {exc}", file=sys.stderr) sys.exit(1) @@ -556,6 +556,8 @@ def register(repo_sub, positive_int, source_help: str, cache_help: str) -> None: help="Treatment arm (ARM or BATCH/ARM); repeatable or comma-separated " "(default: every other arm)") compare_p.add_argument("--model", action="append", default=None, help="Only this model; repeatable") + compare_p.add_argument("--task", action="append", default=None, + help="Only this task id; repeatable (default: every task of the batches)") compare_p.add_argument("--metric", action="append", default=None, help="Also compare this numeric or boolean grade.json metric; repeatable") compare_p.add_argument("--pass-k", type=positive_int, default=None, dest="pass_k", metavar="K", diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index 5307cf0be..fc15552bd 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -12,6 +12,9 @@ - **score**: the mean of ``score.metric`` for score-only tasks (a grader failure counts as 0); - **cost_usd**, **turns**: the per-cell means the harness reported; +- **first_prompt_tokens**: the mean of ``result.json`` ``agent.first_prompt_tokens`` (what the first + main-thread call was sent: start-up plus the task) over the cells that have it. Like a process count, it is + compared only when both arms report it for some shared task: a batch run before it was recorded has none; - **metric:**: the mean of ``grade.json`` ``metrics.`` over the cells that report it (``true`` is 1), for ``--metric``; - **pass^k**: for ``--pass-k K``, the chance that K cells drawn from the @@ -37,6 +40,25 @@ the gate and score comparisons of one invocation; cost, turns and metrics are descriptive and keep their raw p. +Three notes keep a row from being misread; none changes the delta, its interval or p. A gate or score row of 2+ tasks +whose interval includes 0 is inconclusive, not "no effect": the note gives ``mde``, a planning figure: an +approximate, uncorrected t-based estimate of the mean delta a two-sided 5% one-sample t test detects with 80% power +at the row's spread, ``(t_0.975 + t_0.80) * sd / sqrt(n)`` over the per-task deltas, t at ``n - 1`` degrees of +freedom (in every row; ``None`` for one task or no spread), not the power of the sign-flip test nor of the +Holm-corrected decision. The sign-flip p over k changed tasks cannot go below ``2 / 2**k``, so at 5 or fewer no +effect of any size reaches p < 0.05: ``mde`` is ``None`` there too, and the note says how low p can go instead. +``mde`` is ``None`` too where a gate or score row's ``p_holm_floor`` (its ``p_holm`` with its p at that floor and the +family's other p as observed, :mod:`.compare_reach`) is not below 0.05, and the note names Holm: that floor decides +whether ``mde`` is given, not its size. A row whose interval and p disagree (the interval excludes 0 while p is 0.05 +or more, or includes it while p is below; for gate and score the p is ``p_holm``, the one that decides them) has +``interval_p_disagree``: the two come from different methods, and for gate and score ``p_holm`` is the one to read, +since the percentile bootstrap runs narrow over few tasks (the inconclusive note then gives way to this one). A +descriptive row with p below 0.05 is named on a line saying its p is uncorrected and outside Holm's family. + +``--task`` limits a comparison to some task ids: the others are left out, not listed as unpaired, and the +summary's ``task_filter`` records the selection; without ``--arm``, its ``dropped_arms`` names an arm with no +finished cell among it, which drops out of the comparison. + A gate or score comparison also says whether its tasks had room to move (``headroom``), since a zero delta on a task set nobody can fail, or nobody can pass, is not evidence of no effect. A task's top is 1.0 for gate and the task's ``score.max`` for score (recorded in ``result.json``; a batch without it has no known @@ -78,13 +100,16 @@ import itertools import json import random +import statistics from collections import defaultdict from pathlib import Path from typing import Any from dokimi.context.consistency import Run, order_distance, pass_hat, run_from_events -from dokimi.eval.stats import adjust_pvalues, bootstrap_ci, mean_ci +from dokimi.eval.stats import bootstrap_ci, correct_family, mean_ci +from .compare_reach import ALPHA, holm_floor, min_detectable +from .compare_tasks import CompareError, select_tasks from .process_counts import as_measure, process_order, process_values from .report import (load_cells, mismatch_warnings, model_matches, model_mismatch, reading_cells, resolved_models, unmeasured_note, uptake_counts) @@ -97,15 +122,13 @@ EPS = 1e-9 #: Measures Holm's correction runs over. PRIMARY = ("gate", "score") +#: Descriptive measures compared only when both arms report them for some shared task (an older batch has none). +REPORTED = ("process:", "counter:", "first_prompt_tokens") #: Batch stamp keys that must agree between the arms of a comparison; ``tree`` is shown but may differ (another #: revision of the repository is a treatment of its own). STAMP_KEYS = ("eval_dir", "image", "graders") -class CompareError(ValueError): - """The arms or batches named cannot be compared.""" - - def parse_selector(text: str) -> tuple[str | None, str]: """``ARM`` or ``BATCH/ARM`` (batch = the batch directory's name) as ``(batch, arm)``.""" batch, sep, arm = text.rpartition("/") @@ -150,18 +173,25 @@ def batch_stamp(batch_dir: Path) -> dict[str, str]: def compare(batch_dirs: list[Path], baseline: str, treatments: list[str] | None = None, models: list[str] | None = None, metrics: tuple[str, ...] = (), seed: int = 0, - pass_k: int | None = None, path_stability: bool = False) -> dict[str, Any]: + pass_k: int | None = None, path_stability: bool = False, tasks: list[str] | None = None) -> dict[str, Any]: metrics = tuple(dict.fromkeys(metrics)) if pass_k is not None and pass_k < 1: raise CompareError(f"--pass-k must be at least 1, got {pass_k}") - cells, missing = load_batches(batch_dirs, metrics, path_stability) - _check_metrics(metrics, cells) + found, missing = load_batches(batch_dirs, metrics, path_stability) + cells, missing, task_filter = select_tasks(found, missing, tasks, batch_dirs) + # the arms and metrics are checked among the selected cells, so a miss may be the selection's doing + scope = " among the --task selection" if task_filter else "" + _check_metrics(metrics, cells, scope) base_sel = parse_selector(baseline) if not any(selector_matches(base_sel, c) for c in cells): - raise CompareError(f"no finished cell matches the baseline {baseline!r}") - treatments = _treatments(cells, baseline, base_sel, treatments) + raise CompareError(f"no finished cell matches the baseline {baseline!r}{scope}") + if task_filter is not None: # without --arm, an arm that ran none of the selected tasks would drop out unsaid + task_filter["dropped_arms"] = ([] if treatments is not None + else sorted(_other_arms(found, base_sel) - _other_arms(cells, base_sel))) + treatments = _treatments(cells, baseline, base_sel, treatments, scope) measures = ["gate", "score", *([f"pass^{pass_k}"] if pass_k else []), - *(["path_distance"] if path_stability else []), "cost_usd", "turns", *(f"metric:{m}" for m in metrics)] + *(["path_distance"] if path_stability else []), "cost_usd", "turns", "first_prompt_tokens", + *(f"metric:{m}" for m in metrics)] base_vals = _task_values([c for c in cells if selector_matches(base_sel, c)], metrics, pass_k, path_stability) rows: list[dict[str, Any]] = [] unpaired: list[dict[str, Any]] = [] @@ -182,7 +212,7 @@ def compare(batch_dirs: list[Path], baseline: str, treatments: list[str] | None provenance = _provenance(batch_dirs, cells, baseline, treatments, models) provenance["warnings"] += one_arm return {"batches": [str(d) for d in batch_dirs], "baseline": baseline, "treatments": treatments, - "cells": len(cells), "missing": missing, **provenance, + "task_filter": task_filter, "cells": len(cells), "missing": missing, **provenance, "comparisons": rows, "unpaired": unpaired} @@ -299,11 +329,11 @@ def _process_measures(cells: list[dict[str, Any]], selectors: list[str]) -> list return sorted(found, key=process_order) -def _check_metrics(metrics: tuple[str, ...], cells: list[dict[str, Any]]) -> None: +def _check_metrics(metrics: tuple[str, ...], cells: list[dict[str, Any]], scope: str = "") -> None: """A ``--metric`` that no finished cell reports is a typo or not a number: refuse it, not print nothing.""" unknown = [m for m in metrics if not any(m in c["metrics"] for c in cells)] if unknown: - raise CompareError(f"no finished cell reports a numeric or boolean --metric {', '.join(unknown)}") + raise CompareError(f"no finished cell reports a numeric or boolean --metric {', '.join(unknown)}{scope}") def _check_models(models: list[str] | None, rows: list[dict[str, Any]], unpaired: list[dict[str, Any]]) -> None: @@ -314,10 +344,17 @@ def _check_models(models: list[str] | None, rows: list[dict[str, Any]], unpaired def _holm(rows: list[dict[str, Any]]) -> None: - """``p_holm`` on the gate and score rows, corrected as one family.""" + """``p_holm`` on the gate and score rows, corrected as one family; their ``interval_p_disagree`` reads it. + ``p_holm_floor`` is a row's ``p_holm`` at its p floor: where it cannot go below :data:`ALPHA`, ``mde`` is None.""" primary = [r for r in rows if r["measure"] in PRIMARY] - for row, p in zip(primary, adjust_pvalues([r["p"] for r in primary]), strict=True): + ps = [r["p"] for r in primary] + for i, (row, (p, below)) in enumerate(zip(primary, correct_family(ps), strict=True)): + d = row["delta"] row["p_holm"] = p + row["interval_p_disagree"] = d["lo"] is not None and below is not None and (d["lo"] <= 0 <= d["hi"]) == below + if p is not None: + row["p_holm_floor"], reachable = holm_floor(ps, i, min_sign_flip_p(row["up"] + row["down"])) + row["mde"] = row["mde"] if reachable else None def _compare_model(treatment: str, model: str, base_vals: dict[tuple[str, str], dict[str, Any]], @@ -325,7 +362,7 @@ def _compare_model(treatment: str, model: str, base_vals: dict[tuple[str, str], rows: list[dict[str, Any]], unpaired: list[dict[str, Any]]) -> list[str]: """Append one row per measure of one (treatment, model), and the tasks only one arm has. - Returns the process measures only one arm reports for any shared task: they are skipped, not unpaired. + Returns the :data:`REPORTED` measures only one arm reports for any shared task: they are skipped, not unpaired. """ base_tasks = {t for m, t in base_vals if m == model} treat_tasks = {t for m, t in treat_vals if m == model} @@ -339,10 +376,10 @@ def _compare_model(treatment: str, model: str, base_vals: dict[tuple[str, str], unpaired.append(_unpaired(treatment, model, task, None, f"{b} task in baseline, {t} in treatment")) skipped = [] for measure in measures: - if measure.startswith(("process:", "counter:")): + if measure.startswith(REPORTED): has = [any(vals[(model, t)]["values"].get(measure) is not None for t in shared) for vals in (base_vals, treat_vals)] - if not all(has): # an arm that reports no such count at all (an older batch) leaves no task out of a pair + if not all(has): # an arm that reports none of it at all (an older batch) leaves no task out of a pair if any(has): skipped.append(measure) continue @@ -432,22 +469,26 @@ def _unpaired(treatment: str, model: str, task: str, measure: str | None, reason return {"treatment": treatment, "model": model, "task": task, "measure": measure, "reason": reason} +def _other_arms(cells: list[dict[str, Any]], base_sel: tuple[str | None, str]) -> set[str]: + """Every other arm that has cells; the baseline's own arm from another batch is named by its batch.""" + return {f"{c['batch']}/{c['arm']}" if c["arm"] == base_sel[1] else c["arm"] + for c in cells if not selector_matches(base_sel, c)} + + def _treatments(cells: list[dict[str, Any]], baseline: str, base_sel: tuple[str | None, str], - treatments: list[str] | None) -> list[str]: + treatments: list[str] | None, scope: str = "") -> list[str]: """The treatment selectors to compare, checked: each matches cells, and none of the baseline's.""" if treatments is not None and not treatments: raise CompareError("no treatment arms named; leave --arm out to compare every other arm") if treatments is None: - # every other arm that has cells; the baseline's own arm from another batch is named by its batch - treatments = sorted({f"{c['batch']}/{c['arm']}" if c["arm"] == base_sel[1] else c["arm"] - for c in cells if not selector_matches(base_sel, c)}) + treatments = sorted(_other_arms(cells, base_sel)) for text in treatments: sel = parse_selector(text) if sel == base_sel: raise CompareError(f"treatment {text!r} is the baseline") matched = [c for c in cells if selector_matches(sel, c)] if not matched: - raise CompareError(f"no finished cell matches the treatment {text!r}") + raise CompareError(f"no finished cell matches the treatment {text!r}{scope}") if any(selector_matches(base_sel, c) for c in matched): raise CompareError(f"treatment {text!r} also matches cells of the baseline {baseline!r}; " "name its batch (BATCH/ARM)") @@ -483,17 +524,25 @@ def paired_summary(pairs: list[dict[str, Any]], measure: str = "gate", seed: int permutation test. The interval is a percentile bootstrap for the bounded gate and score shares and a t interval (:func:`~dokimi.eval.stats.mean_ci`) for the continuous measures, where the bootstrap under-covers at these task counts; one task has no interval (``lo``/``hi`` None), not a zero-width one. + ``mde`` and ``interval_p_disagree`` are the reading aids of the module docstring; both read the raw p here, + and :func:`compare` revisits both on a gate or score row once Holm's p is known. """ deltas = [p["delta"] for p in pairs] ci = (bootstrap_ci if measure in PRIMARY else mean_ci)(deltas, seed=seed) lo, hi = (float(ci["lo"]), float(ci["hi"])) if len(deltas) > 1 else (None, None) # mean_ci gives numpy floats + p = sign_flip_p(deltas, seed=seed) + sd = statistics.stdev(deltas) if len(deltas) > 1 else 0.0 + reachable = min_sign_flip_p(sum(abs(d) > EPS for d in deltas)) < ALPHA # else no effect can be detected return { "n_tasks": len(pairs), "baseline_mean": _round(_mean([p["baseline"] for p in pairs])), "treatment_mean": _round(_mean([p["treatment"] for p in pairs])), "delta": {"mean": ci["mean"], "lo": lo, "hi": hi}, - "p": sign_flip_p(deltas, seed=seed), + "p": p, "p_holm": None, + "p_holm_floor": None, + "mde": _round(min_detectable(sd, len(deltas))) if sd > EPS and reachable else None, + "interval_p_disagree": lo is not None and hi is not None and p is not None and (lo <= 0 <= hi) == (p < ALPHA), "up": sum(d > EPS for d in deltas), "down": sum(d < -EPS for d in deltas), "tied": sum(abs(d) <= EPS for d in deltas), @@ -502,6 +551,12 @@ def paired_summary(pairs: list[dict[str, Any]], measure: str = "gate", seed: int } +def min_sign_flip_p(changed: int) -> float: + """The smallest p :func:`sign_flip_p` can give over ``changed`` non-zero deltas, every sign the same: only that + pattern and its mirror reach the observed ``|sum|``, ``2 / 2**changed`` exact, ``1 / (N + 1)`` Monte Carlo.""" + return min(1.0, 2 / 2 ** changed) if changed <= EXACT_MAX else 1 / (N_RESAMPLES + 1) + + def sign_flip_p(deltas: list[float], seed: int = 0, n_resamples: int = N_RESAMPLES) -> float | None: """Two-sided p of ``|sum(deltas)|`` when each delta's sign is a fair coin under the null. @@ -526,6 +581,7 @@ def format_compare(summary: dict[str, Any]) -> str: for row in summary["comparisons"]: groups[(row["treatment"], row["model"])].append(row) width = max([len(r["measure"]) for r in summary["comparisons"]] + [7]) + family = sum(r["p_holm"] is not None for r in summary["comparisons"]) # the rows Holm corrected together for (treatment, model), rows in groups.items(): lines += ["", f"{treatment} vs {summary['baseline']}, model {model}", f" {'measure':<{width}} {'tasks':>5} {'baseline':>9} {'treatment':>9} " @@ -544,6 +600,7 @@ def format_compare(summary: dict[str, Any]) -> str: f"{t['task']} {t['delta']:+.2f}" for t in sorted(moved, key=lambda t: t["delta"]))) if r["headroom"]: lines.append(f" {r['measure']} {_headroom_note(r)}") + lines += _reading_notes(rows, family) if summary["unpaired"]: shown = summary["unpaired"][:10] lines.append("unpaired: " + ", ".join( @@ -557,6 +614,47 @@ def format_compare(summary: dict[str, Any]) -> str: return "\n".join(lines) +def _reading_notes(rows: list[dict[str, Any]], family: int) -> list[str]: + """One comparison's notes against a misreading (module docstring), ``family`` Holm's rows; no number changes.""" + out = [] + for r in rows: + d, primary = r["delta"], r["measure"] in PRIMARY + if r["interval_p_disagree"]: + inside = d["lo"] <= 0 <= d["hi"] + how = "a percentile bootstrap, too narrow over few tasks," if primary else "a t interval" + name = "p_holm" if primary else "p" # a gate or score row is decided by its Holm p (_holm) + out.append(f" {r['measure']}: the interval [{d['lo']:+.3f}, {d['hi']:+.3f}] " + f"{'includes' if inside else 'excludes'} 0 but {name} {r[name]:.4g} is {'<' if inside else '≥'} " + f"{ALPHA}; they come from two methods, {how} and a sign-flip test" + + ("; read p_holm" if primary else "")) + elif primary and (note := _inconclusive_note(r, family)): + out.append(note) + low = [r["measure"] for r in rows if r["measure"] not in PRIMARY and r["p"] is not None and r["p"] < ALPHA] + if low: + out.append(f" p < {ALPHA} on {', '.join(low)}: descriptive rows, their p uncorrected and outside Holm's " + "family, which covers gate and score only") + return out + + +def _inconclusive_note(r: dict[str, Any], family: int) -> str | None: + """The note of a gate or score row of 2+ tasks whose interval includes 0: its ``mde``, or why it has none.""" + d = r["delta"] + if d["lo"] is None or not d["lo"] <= 0 <= d["hi"]: + return None + head = f" {r['measure']} inconclusive: the interval includes 0; " + if r["mde"] is not None: + return head + (f"an uncorrected t-test estimate puts the effect 80% power detects at this spread at about " + f"±{r['mde']:.3f} ({r['n_tasks']} tasks), before Holm and not for the sign-flip test") + changed, holm = r["up"] + r["down"], r["p_holm_floor"] + if r["p"] is None or (floor := min_sign_flip_p(changed)) < ALPHA and holm < ALPHA: + return None # every delta tied, or no spread + if floor >= ALPHA: + return head + (f"with {changed} changed task{'' if changed == 1 else 's'} the sign-flip p cannot go below " + f"{floor:.4g}: no effect can reach p < {ALPHA} here") + return head + (f"with Holm over {family} rows, p_holm cannot go below {holm:.4g} here: no effect can reach " + f"p_holm < {ALPHA}") + + def _headroom_note(row: dict[str, Any]) -> str: n, gate = row["n_tasks"], row["measure"] == "gate" if row["headroom"] == "ceiling": @@ -571,6 +669,11 @@ def _headroom_note(row: dict[str, Any]) -> str: def _format_header(summary: dict[str, Any]) -> list[str]: lines = [f"baseline {summary['baseline']}: {summary['cells']} finished cells in " f"{len(summary['batches'])} batch(es), {len(summary['missing'])} missing"] + if summary.get("task_filter"): + lines.append(f" tasks: {len(summary['task_filter']['tasks'])} of {summary['task_filter']['of']} (--task)") + if summary["task_filter"].get("dropped_arms"): + lines.append(f" not compared, no finished cell among the --task selection: " + f"{', '.join(summary['task_filter']['dropped_arms'])}") for text, batches in summary["sources"].items(): lines.append(f" {text}: {', '.join(batches)}") for name, stamp in summary["stamps"].items(): @@ -591,6 +694,7 @@ def _task_values(cells: list[dict[str, Any]], metrics: tuple[str, ...], pass_k: score_task = any(r["score_task"] for r in recs) found = {"cost_usd": [r["agent"].get("cost_usd") for r in recs], "turns": [r["agent"].get("num_turns") for r in recs], + "first_prompt_tokens": [r["agent"].get("first_prompt_tokens") for r in recs], **{f"metric:{name}": [r["metrics"].get(name) for r in recs] for name in metrics}} found.update(_process_lists(recs)) if score_task: diff --git a/dokimi/dokimi/repos/compare_reach.py b/dokimi/dokimi/repos/compare_reach.py new file mode 100644 index 000000000..2f594a026 --- /dev/null +++ b/dokimi/dokimi/repos/compare_reach.py @@ -0,0 +1,32 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""How small an effect a ``repo compare`` row can find: ``mde``, an uncorrected t-based planning estimate, and the +lowest ``p_holm`` Holm's family leaves a gate or score row (:mod:`.compare`).""" + +from __future__ import annotations + +import math + +from scipy import stats as sp_stats + +from dokimi.eval.stats import correct_family + +#: The level the notes read p and the interval at (two-sided), and the power ``mde`` is sized for. +ALPHA = 0.05 +POWER = 0.80 + + +def min_detectable(sd: float, n: int) -> float: + """An approximate, uncorrected t-based estimate of the mean delta a two-sided one-sample t test at :data:`ALPHA` + detects with :data:`POWER` over ``n`` tasks of spread ``sd``, a planning figure: not the power of the sign-flip + test nor of the Holm-corrected decision. The quantiles are t at ``n - 1`` degrees of freedom, as the interval + of ``mean_ci`` is.""" + df = n - 1 + return (sp_stats.t.ppf(1 - ALPHA / 2, df) + sp_stats.t.ppf(POWER, df)) * sd / math.sqrt(n) + + +def holm_floor(pvals: list[float | None], i: int, floor: float) -> tuple[float | None, bool | None]: + """Holm's p of row ``i`` of the family ``pvals`` with its p put at ``floor``, the lowest it can go, and the + others as observed; and whether that p is below :data:`ALPHA`. Where it is not, no effect can be.""" + return correct_family([*pvals[:i], floor, *pvals[i + 1:]])[i] diff --git a/dokimi/dokimi/repos/compare_tasks.py b/dokimi/dokimi/repos/compare_tasks.py new file mode 100644 index 000000000..81bd2e712 --- /dev/null +++ b/dokimi/dokimi/repos/compare_tasks.py @@ -0,0 +1,52 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""``repo compare --task``: the cells and missing cells a task selection keeps (:mod:`.compare`).""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + + +class CompareError(ValueError): + """The arms or batches named cannot be compared (here, a ``--task`` id; :mod:`.compare` raises the others).""" + + +def select_tasks(cells: list[dict[str, Any]], missing: list[str], tasks: list[str] | None, + batch_dirs: list[Path]) -> tuple[list[dict[str, Any]], list[str], dict[str, Any] | None]: + """The cells and missing cells of ``tasks`` (``--task``), and the selection as the summary records it. + + No ``tasks`` keeps every cell, a missing cell of no known task too, and records ``None``. An id no finished + cell of the batches has is refused.""" + if not tasks: + return cells, missing, None + known = {c["task"] for c in cells} + unknown = [t for t in dict.fromkeys(tasks) if t not in known] + if unknown: + raise CompareError(f"no finished cell of the compared batches has --task {', '.join(unknown)}") + dirs = {d.resolve().name: d for d in batch_dirs} + ids = {name: {c["task"] for c in cells if c["batch"] == name} for name in dirs} + kept = [] + for entry in missing: # BATCH/ + batch, _, cell = entry.partition("/") + # a cell whose task neither its meta.json nor a known id gives is not in any selection + if _missing_task(dirs[batch], cell, ids[batch]) in tasks: + kept.append(entry) + return [c for c in cells if c["task"] in tasks], kept, {"tasks": sorted(set(tasks)), "of": len(known)} + + +def _missing_task(batch_dir: Path, cell: str, known: set[str]) -> str | None: + """The task of a missing cell: the id its ``meta.json`` records (``repo run`` writes it when the cell starts), + else, for a cell only ``plan.json`` names, the longest of the batch's finished task ids ``known`` that the + name starts with followed by ``__``. The name alone does not say: :func:`~dokimi.repos.runner.cell_name` + keeps a ``__`` of the task id, the arm and the model. ``None`` when neither gives a task.""" + try: + meta = json.loads((batch_dir / cell / "meta.json").read_text(encoding="utf-8")) + except (OSError, ValueError, RecursionError): + meta = None + task = meta.get("task") if isinstance(meta, dict) else None + if isinstance(task, str): + return task + return max((t for t in known if cell.startswith(f"{t}__")), key=len, default=None) diff --git a/dokimi/dokimi/repos/events.py b/dokimi/dokimi/repos/events.py index f6c59f0fd..e1a15649a 100644 --- a/dokimi/dokimi/repos/events.py +++ b/dokimi/dokimi/repos/events.py @@ -20,7 +20,9 @@ Token counts cover the same scope as ``cost_usd``: summed over ``modelUsage`` (parent, subagents and helper models) when the result has it, else the parent's -``usage``; ``usage_scope`` says which (``tree`` or ``main``). +``usage``; ``usage_scope`` says which (``tree`` or ``main``). ``first_prompt_tokens`` is +the prompt of the first main-thread model call, everything start-up loaded plus the +task, read as :mod:`dokimi.context.trace` reads it (one call per message id). """ from __future__ import annotations @@ -32,6 +34,8 @@ from pathlib import PurePosixPath, PureWindowsPath from typing import Any +from dokimi.context.trace import parse_trace + TRACE_FORMATS = ("claude-stream-json", "text") OUTPUT_LIMIT = 8192 OUTPUT_KEEP = 4096 @@ -57,6 +61,7 @@ class Transcript: cache_read_tokens: int | None = None cache_write_tokens: int | None = None usage_scope: str | None = None # "tree": tokens include subagents and helper models; "main": parent only + first_prompt_tokens: int | None = None # the first main-thread call's prompt; None for a format without usage subagent_calls: int | None = None # counted once the harness's result record is seen is_error: bool = False harness_version: str | None = None @@ -113,6 +118,8 @@ def _claude_stream_json(raw: str, workdir: str | None) -> Transcript: _apply(out, rec, t, by_id, workdir) except (AttributeError, TypeError) as exc: # valid JSON of an unexpected shape out.errors.append(f"line {n}: unexpected shape ({exc})") + main = parse_trace(raw).main_calls() + out.first_prompt_tokens = main[0]["prompt_tokens"] if main else None return out diff --git a/dokimi/dokimi/repos/runner.py b/dokimi/dokimi/repos/runner.py index 42f18926f..a76ea7feb 100644 --- a/dokimi/dokimi/repos/runner.py +++ b/dokimi/dokimi/repos/runner.py @@ -277,7 +277,7 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, # A killed harness reports no cost; the batch budget counts the cell's cap instead of $0. "charged_usd": caps["budget_usd"], "num_turns": None, "is_error": None, "result_subtype": None, "input_tokens": None, "output_tokens": None, "cache_read_tokens": None, - "cache_write_tokens": None, "usage_scope": None, "subagent_calls": None, + "cache_write_tokens": None, "usage_scope": None, "first_prompt_tokens": None, "subagent_calls": None, "plugins": [], "over_budget": False, "caps": caps, "network": network, "trace_errors": []} process: dict[str, Any] | None = None # null when the trace could not be read added = meta.get("arm_patch") or {} @@ -302,6 +302,7 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, result_subtype=transcript.result_subtype, input_tokens=transcript.input_tokens, output_tokens=transcript.output_tokens, cache_read_tokens=transcript.cache_read_tokens, cache_write_tokens=transcript.cache_write_tokens, usage_scope=transcript.usage_scope, + first_prompt_tokens=transcript.first_prompt_tokens, subagent_calls=transcript.subagent_calls, plugins=transcript.plugins, over_budget=cost is not None and cost > caps["budget_usd"], trace_errors=transcript.errors[:5]) if events is not None: # after the cost is recorded: a metric that fails must not cost the cell its record diff --git a/dokimi/tests/repo_eval_fixture.py b/dokimi/tests/repo_eval_fixture.py index 8682e3e49..5f0e7038d 100644 --- a/dokimi/tests/repo_eval_fixture.py +++ b/dokimi/tests/repo_eval_fixture.py @@ -128,7 +128,8 @@ "usage": {"input_tokens": 5, "output_tokens": 2, "cache_read_input_tokens": 100, "cache_creation_input_tokens": 40}} if os.environ.get("FAKE_USAGE"): # a subagent (Agent tool) and the token counts of the whole tree - lines.append({"type": "assistant", "message": {"content": [ + lines.append({"type": "assistant", "message": {"id": "msg_1", "usage": { + "input_tokens": 3, "cache_creation_input_tokens": 1200, "cache_read_input_tokens": 15000}, "content": [ {"type": "tool_use", "id": "t2", "name": "Agent", "input": {"prompt": "look around"}}]}}) result.update(usage={"input_tokens": 7, "output_tokens": 2}, modelUsage={"m-main": {"inputTokens": 7, "outputTokens": 2, "cacheReadInputTokens": 100}, diff --git a/dokimi/tests/test_repo_compare_reading.py b/dokimi/tests/test_repo_compare_reading.py new file mode 100644 index 000000000..d098aa3ea --- /dev/null +++ b/dokimi/tests/test_repo_compare_reading.py @@ -0,0 +1,349 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""`repo compare`: the start context of each arm, `--task`, and the notes that keep a row from being misread.""" + +from __future__ import annotations + +import json +import statistics +from pathlib import Path + +import pytest + +from dokimi.cli import build_parser +from dokimi.repos.compare import CompareError, compare, format_compare, min_detectable +from dokimi.repos.runner import cell_name + +from test_repo_compare import _cell, _row + + +def _first(batch: Path, task: str, arm: str, rep: int, tokens: int | None) -> None: + """A finished gate cell whose ``agent`` records ``first_prompt_tokens`` (left out for ``None``: an older batch).""" + _cell(batch, task, arm, rep) + path = batch / f"{task}__{arm.replace(':', '-')}__m-1__r{rep}" / "result.json" + rec = json.loads(path.read_text()) + if tokens is not None: + rec["agent"]["first_prompt_tokens"] = tokens + path.write_text(json.dumps(rec)) + + +def test_first_prompt_tokens_is_compared_per_task_over_the_cells_that_have_it(tmp_path): + b = tmp_path / "b" + for task, base, treat in (("t1", (16_310, 16_320), (19_370, 19_390)), ("t2", (16_300, None), (19_400, 19_380))): + for rep, (x, y) in enumerate(zip(base, treat, strict=True), 1): + _first(b, task, "guidance:none", rep, x) + _first(b, task, "guidance:raw", rep, y) + row = _row(compare([b], "guidance:none"), "first_prompt_tokens") + by_task = {t["task"]: t for t in row["tasks"]} + assert by_task["t1"]["baseline"] == 16_315 and by_task["t1"]["treatment"] == 19_380 + assert by_task["t2"]["baseline"] == 16_300 and by_task["t2"]["n_baseline"] == 1 # the cell without it is left out + assert row["delta"]["mean"] == pytest.approx((3_065 + 3_090) / 2) and row["p_holm"] is None + + +def test_first_prompt_tokens_of_an_older_batch_is_no_row_not_zero(tmp_path): + old, mixed, reverse = tmp_path / "old", tmp_path / "mixed", tmp_path / "reverse" + for rep in (1, 2): + _first(old, "t1", "guidance:none", rep, None) + _first(old, "t1", "guidance:raw", rep, None) + _first(mixed, "t1", "guidance:none", rep, None) + _first(mixed, "t1", "guidance:raw", rep, 19_380) + _first(reverse, "t1", "guidance:none", rep, 16_315) + _first(reverse, "t1", "guidance:raw", rep, None) + s = compare([old], "guidance:none") + assert "first_prompt_tokens" not in {r["measure"] for r in s["comparisons"]} + assert not s["unpaired"] and not s["warnings"] + for batch in (mixed, reverse): # the arm without it is the baseline, then the treatment + s = compare([batch], "guidance:none") + assert "first_prompt_tokens" not in {r["measure"] for r in s["comparisons"]} and not s["unpaired"] + assert s["warnings"] == ["guidance:raw m-1: only one arm reports first_prompt_tokens for any shared task; " + "not compared"] + + +def _three_tasks(b: Path) -> None: + for task, base_ok in (("t1", False), ("t2", True), ("t3", False)): + _cell(b, task, "guidance:none", 1, success=base_ok) + _cell(b, task, "guidance:raw", 1) + _cell(b, "t4", "guidance:none", 1) # only one arm ran it: unpaired, unless --task leaves it out + (b / "t4__guidance-raw__m-1__r1").mkdir() + (b / "t4__guidance-raw__m-1__r1" / "meta.json").write_text("{}") # started, never finished: missing + + +def test_task_compares_only_the_named_tasks_and_says_so(tmp_path): + b = tmp_path / "b" + _three_tasks(b) + whole = compare([b], "guidance:none") + assert whole["task_filter"] is None and _row(whole, "gate")["n_tasks"] == 3 + assert [u["task"] for u in whole["unpaired"]] == ["t4"] and len(whole["missing"]) == 1 + s = compare([b], "guidance:none", tasks=["t1", "t2"]) + gate = _row(s, "gate") + assert [t["task"] for t in gate["tasks"]] == ["t1", "t2"] and gate["up"] == 1 + assert s["unpaired"] == [] and s["missing"] == [] and s["cells"] == 4 + assert s["task_filter"] == {"tasks": ["t1", "t2"], "of": 4, "dropped_arms": []} + assert " tasks: 2 of 4 (--task)" in format_compare(s).splitlines() + assert "(--task)" not in format_compare(whole) + + +def test_cli_task_is_repeatable_and_lands_in_the_json(tmp_path, capsys): + b = tmp_path / "b" + _three_tasks(b) + args = build_parser().parse_args(["repo", "compare", str(b), "--baseline", "guidance:none", + "--task", "t3", "--task", "t4", "--json"]) + args.func(args) + out = json.loads(capsys.readouterr().out) + assert out["task_filter"] == {"tasks": ["t3", "t4"], "of": 4, "dropped_arms": []} + assert [t["task"] for t in _row(out, "gate")["tasks"]] == ["t3"] + assert [u["task"] for u in out["unpaired"]] == ["t4"] and len(out["missing"]) == 1 + + +def test_task_matches_a_missing_cell_by_its_task_not_by_a_prefix(tmp_path): + # a task id may contain "__": the unfinished cell of fix__a is not a cell of fix + b = tmp_path / "b" + for task in ("fix", "fix__a"): + _cell(b, task, "guidance:none", 1) + _cell(b, task, "guidance:raw", 1) + (b / "fix__a__guidance-raw__m-1__r2").mkdir() + (b / "fix__a__guidance-raw__m-1__r2" / "meta.json").write_text("{}") # started, never finished: missing + s = compare([b], "guidance:none", tasks=["fix"]) + assert s["missing"] == [] and s["task_filter"] == {"tasks": ["fix"], "of": 2, "dropped_arms": []} + assert " tasks: 1 of 2 (--task)" in format_compare(s).splitlines() + assert "cells missing" not in format_compare(s) + assert compare([b], "guidance:none", tasks=["fix__a"])["missing"] == ["b/fix__a__guidance-raw__m-1__r2"] + + +def test_task_reads_a_missing_cell_s_task_from_its_meta_or_the_known_ids(tmp_path): + # cell names keep a "__" of the task, the arm and the model: fix__guidance-raw__m__v1__r2 is a cell of fix, + # though a task fix__guidance-raw exists too + b, model = tmp_path / "b", "m__v1" + for task in ("fix", "fix__a", "fix__guidance-raw"): + _cell(b, task, "guidance:none", 1, model=model) + _cell(b, task, "skill:lint__v2", 1, model=model) + started = {cell_name("fix", "guidance:raw", model, 2): "fix", + cell_name("fix__a", "skill:lint__v2", model, 2): "fix__a"} + for cell, task in started.items(): + (b / cell).mkdir() + (b / cell / "meta.json").write_text(json.dumps({"task": task})) # what repo run writes at the start + planned = [cell_name("fix", "skill:lint__v2", model, 3), cell_name("fix__a", "guidance:none", model, 3), + cell_name("gone", "guidance:none", model, 1)] + (b / "_eval").mkdir() + (b / "_eval" / "plan.json").write_text(json.dumps({"cells": [p.parent.name for p in b.glob("*/result.json")] + + [*started, *planned]})) + fix = compare([b], "guidance:none", tasks=["fix"]) + assert fix["missing"] == ["b/fix__guidance-raw__m__v1__r2", "b/fix__skill-lint__v2__m__v1__r3"] + assert compare([b], "guidance:none", tasks=["fix__a"])["missing"] == ["b/fix__a__guidance-none__m__v1__r3", + "b/fix__a__skill-lint__v2__m__v1__r2"] + assert compare([b], "guidance:none", tasks=["fix__guidance-raw"])["missing"] == [] + # a planned cell of a task no finished cell has belongs to no selection, but is missing without one + assert "b/gone__guidance-none__m__v1__r1" in compare([b], "guidance:none")["missing"] + + +def test_an_arm_or_metric_error_under_task_says_the_selection_is_the_reason(tmp_path): + b = tmp_path / "b" + _cell(b, "t1", "guidance:none", 1) + _cell(b, "t1", "guidance:raw", 1) + for arm in ("guidance:none", "guidance:other"): # guidance:other ran only t2, with the metric + _cell(b, "t2", arm, 1, metrics={"tok": 3.0}) + with pytest.raises(CompareError, match=r"no finished cell matches the treatment 'guidance:other' among the " + r"--task selection$"): + compare([b], "guidance:none", treatments=["guidance:other"], tasks=["t1"]) + with pytest.raises(CompareError, match=r"no finished cell matches the baseline 'guidance:other' among the " + r"--task selection$"): + compare([b], "guidance:other", tasks=["t1"]) + with pytest.raises(CompareError, match=r"reports a numeric or boolean --metric tok among the --task selection$"): + compare([b], "guidance:none", metrics=("tok",), tasks=["t1"]) + with pytest.raises(CompareError, match=r"^no finished cell matches the treatment 'guidance:nope'$"): + compare([b], "guidance:none", treatments=["guidance:nope"]) # without --task, no selection to name + + +def test_task_names_an_arm_that_ran_none_of_the_selected_tasks(tmp_path): + b = tmp_path / "b" + for task in ("t1", "t2", "t3"): + _cell(b, task, "guidance:none", 1) + _cell(b, task, "guidance:other", 1) + for task in ("t1", "t2"): # guidance:raw ran only t1 and t2 + _cell(b, task, "guidance:raw", 1) + s = compare([b], "guidance:none", tasks=["t3"]) + assert s["treatments"] == ["guidance:other"] and s["task_filter"]["dropped_arms"] == ["guidance:raw"] + assert (" not compared, no finished cell among the --task selection: guidance:raw" + in format_compare(s).splitlines()) + named = compare([b], "guidance:none", treatments=["guidance:other"], tasks=["t3"]) # --arm: nothing drops out + assert named["task_filter"]["dropped_arms"] == [] and "not compared" not in format_compare(named) + assert compare([b], "guidance:none", tasks=["t1"])["task_filter"]["dropped_arms"] == [] + + +def test_task_refuses_an_id_no_cell_has(tmp_path, capsys): + b = tmp_path / "b" + _three_tasks(b) + with pytest.raises(CompareError, match=r"has --task t9, fix_x$"): + compare([b], "guidance:none", tasks=["t1", "t9", "fix_x"]) + args = build_parser().parse_args(["repo", "compare", str(b), "--baseline", "guidance:none", "--task", "t9"]) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 1 and "no finished cell of the compared batches has --task t9" in capsys.readouterr().err + + +def test_min_detectable_effect_arithmetic(): + # The recall row that read as "no gain": SD 0.222 of the per-task deltas over 26 tasks, t quantiles at df 25 + assert min_detectable(0.222, 26) == pytest.approx(0.126946, abs=1e-6) + # half the spread or 4x the tasks: about the same figure, the t quantiles shrink a little with the tasks + assert min_detectable(0.222, 26) == pytest.approx(min_detectable(0.444, 104), rel=0.05) + + +def test_min_detectable_uses_the_t_quantiles_of_the_interval_at_small_n(): + # four tasks: the interval is a t interval at df 3 (t(0.975, 3) = 3.182, t(0.80, 3) = 0.978), not normal ones + assert min_detectable(0.456435, 4) == pytest.approx(0.949594, abs=1e-6) + + +def _score_tasks(b: Path, deltas: list[float], base: float) -> None: + for i, d in enumerate(deltas): + _cell(b, f"review_{i}", "guidance:none", 1, score=base) + _cell(b, f"review_{i}", "guidance:raw", 1, score=base + d) + + +MDE_NOTE = (" score inconclusive: the interval includes 0; an uncorrected t-test estimate puts the effect 80% power " + "detects at this spread at about ±{:.3f} ({} tasks), before Holm and not for the sign-flip test") + + +def test_an_interval_across_zero_is_inconclusive_with_its_mde(tmp_path): + # six changed tasks: the sign-flip p can go down to 2/64, so 80% power names an effect + _score_tasks(tmp_path / "b", [0.5, -0.5, 0.25, -0.25, 0.5, -0.5], 0.5) # sample SD 0.474342 + s = compare([tmp_path / "b"], "guidance:none") + score = _row(s, "score") + assert score["mde"] == pytest.approx(min_detectable(0.474342, 6), abs=1e-6) and score["mde"] > 0.6 + assert score["interval_p_disagree"] is False + text = format_compare(s) + assert MDE_NOTE.format(score["mde"], 6) in text.splitlines() + assert "no effect" not in text + _score_tasks(tmp_path / "one", [0.5], 0.5) + one = compare([tmp_path / "one"], "guidance:none") + assert _row(one, "score")["mde"] is None and "inconclusive" not in format_compare(one) + + +@pytest.mark.parametrize("deltas,changed,floor", [ + ([0.5, -0.5, 0.25, -0.25], 4, "0.125"), # four tasks, sample SD 0.456435: a t power figure would say ±0.950 + ([0.5, -0.5, 0.25, -0.25, 0.5, 0.0], 5, "0.0625"), # six tasks, one tied: five changed +]) +def test_no_mde_where_the_sign_flip_p_cannot_reach_005(tmp_path, deltas, changed, floor): + _score_tasks(tmp_path / "b", deltas, 0.5) + s = compare([tmp_path / "b"], "guidance:none") + score = _row(s, "score") + assert score["mde"] is None and score["delta"]["lo"] < 0 < score["delta"]["hi"] + assert (f" score inconclusive: the interval includes 0; with {changed} changed tasks the sign-flip p cannot " + f"go below {floor}: no effect can reach p < 0.05 here") in format_compare(s).splitlines() + assert "80% power" not in format_compare(s) + + +SIX_MIXED = [0.5, -0.5, 0.25, -0.25, 0.5, -0.5] # six changed tasks: a sign-flip p floor of 2/64 +HOLM_NOTE = (" score inconclusive: the interval includes 0; with Holm over 2 rows, p_holm cannot go below {} here: " + "no effect can reach p_holm < 0.05") + + +def _two_score_arms(b: Path, raw: list[float], other: list[float]) -> None: + for i, (r, o) in enumerate(zip(raw, other, strict=True)): + _cell(b, f"review_{i}", "guidance:none", 1, score=0.5) + _cell(b, f"review_{i}", "guidance:raw", 1, score=0.5 + r) + _cell(b, f"review_{i}", "guidance:other", 1, score=0.5 + o) + + +def test_no_mde_where_holm_over_the_family_keeps_p_holm_from_005(tmp_path): + # each row alone could reach p 2/64; Holm over two rows doubles the smaller, so p_holm cannot go below 0.0625 + _two_score_arms(tmp_path / "b", SIX_MIXED, SIX_MIXED) + s = compare([tmp_path / "b"], "guidance:none") + for arm in ("guidance:raw", "guidance:other"): + score = _row(s, "score", treatment=arm) + assert score["mde"] is None and score["p_holm_floor"] == 0.0625 + assert score["delta"]["lo"] < 0 < score["delta"]["hi"] + text = format_compare(s) + assert text.splitlines().count(HOLM_NOTE.format("0.0625")) == 2 and "80% power" not in text + + +def test_a_row_holm_still_lets_reach_005_keeps_its_mde(tmp_path): + # eight changed tasks: a floor of 2/256, which Holm over two rows takes to 0.0156; six: 2/64 to 0.0625 + eight = [0.5, -0.5, 0.25, -0.25, 0.5, -0.5, 0.25, -0.25] + _two_score_arms(tmp_path / "b", eight, [*SIX_MIXED, 0.0, 0.0]) + s = compare([tmp_path / "b"], "guidance:none") + raw, other = _row(s, "score"), _row(s, "score", treatment="guidance:other") + assert raw["p_holm_floor"] == 0.0156 and raw["delta"]["lo"] < 0 < raw["delta"]["hi"] + assert raw["mde"] == pytest.approx(min_detectable(statistics.stdev(eight), 8), abs=1e-6) + assert other["p_holm_floor"] == 0.0625 and other["mde"] is None + lines = format_compare(s).splitlines() + # the figure is the one-row t estimate: Holm (raw p < 0.025 here) only decides that it is given + assert MDE_NOTE.format(raw["mde"], 8) in lines + assert HOLM_NOTE.format("0.0625") in lines + + +def test_a_gate_row_of_two_changed_tasks_has_no_mde(tmp_path): + b = tmp_path / "b" + for task, base_ok in (("t1", False), ("t2", False), ("t3", True)): # gate deltas [1, 1, 0] + _cell(b, task, "guidance:none", 1, success=base_ok) + _cell(b, task, "guidance:raw", 1) + s = compare([b], "guidance:none") + gate = _row(s, "gate") + assert gate["p"] == 0.5 and gate["delta"]["lo"] == 0.0 and gate["mde"] is None + assert (" gate inconclusive: the interval includes 0; with 2 changed tasks the sign-flip p cannot go below " + "0.5: no effect can reach p < 0.05 here") in format_compare(s).splitlines() + + +def test_a_bootstrap_interval_that_excludes_zero_against_p_025_is_flagged(tmp_path): + _score_tasks(tmp_path / "b", [0.0, 0.0, 0.333, 0.167, 0.0, 0.333], 0.0) + s = compare([tmp_path / "b"], "guidance:none") + score = _row(s, "score") + assert score["delta"]["lo"] > 0 and score["p"] == 0.25 # three non-zero deltas, all positive: p cannot go lower + assert score["interval_p_disagree"] is True + notes = [line for line in format_compare(s).splitlines() if line.startswith(" score")] + assert notes[-1].startswith(f" score: the interval [{score['delta']['lo']:+.3f}, +0.250] excludes 0 but " + "p_holm 0.25 is ≥ 0.05; they come from two methods, a percentile bootstrap") + assert notes[-1].endswith("; read p_holm") and not any("inconclusive" in line for line in notes) + + +def _six_of_seven_up(b: Path, treatments: tuple[str, ...]) -> None: + """Gate deltas six +1 and one tie for each treatment: sign-flip p 2/64 for each.""" + for i in range(7): + _cell(b, f"t{i}", "guidance:none", 1, success=i == 6) + for arm in treatments: + _cell(b, f"t{i}", arm, 1) + + +def test_a_gate_interval_against_p_holm_is_flagged(tmp_path): + # two treatments: p 0.031 each, Holm 0.0625; the interval excludes 0, so the two disagree + _six_of_seven_up(tmp_path / "b", ("guidance:raw", "guidance:other")) + s = compare([tmp_path / "b"], "guidance:none") + for arm in ("guidance:raw", "guidance:other"): + gate = _row(s, "gate", treatment=arm) + assert gate["delta"]["lo"] > 0 and gate["p"] == pytest.approx(2 / 64) and gate["p_holm"] == 0.0625 + assert gate["interval_p_disagree"] is True + notes = [line for line in format_compare(s).splitlines() if line.startswith(" gate: the interval")] + assert len(notes) == 2 and all("excludes 0 but p_holm 0.0625 is ≥ 0.05" in n and n.endswith("; read p_holm") + for n in notes) + + +def test_an_interval_across_zero_against_a_raw_p_below_005_is_not_flagged_when_p_holm_is_not(tmp_path, monkeypatch): + # the mirror: an interval that includes 0 against p 0.031, which Holm takes to 0.0625: the two agree + monkeypatch.setattr("dokimi.repos.compare.bootstrap_ci", + lambda deltas, seed=0: {"mean": round(sum(deltas) / len(deltas), 4), "lo": -0.1, "hi": 1.0}) + _six_of_seven_up(tmp_path / "two", ("guidance:raw", "guidance:other")) + two = compare([tmp_path / "two"], "guidance:none") + gate = _row(two, "gate") + assert gate["p"] == pytest.approx(2 / 64) and gate["p_holm"] == 0.0625 and gate["interval_p_disagree"] is False + assert "gate: the interval" not in format_compare(two) + _six_of_seven_up(tmp_path / "one", ("guidance:raw",)) # a family of one: p_holm is p, below 0.05 + one = compare([tmp_path / "one"], "guidance:none") + assert _row(one, "gate")["interval_p_disagree"] is True + assert "includes 0 but p_holm 0.0312 is < 0.05" in format_compare(one) + + +def test_a_descriptive_p_below_005_is_named_as_uncorrected(tmp_path): + b = tmp_path / "b" + for i, turns in enumerate([11, 11, 11, 11, 11, 30]): # every task up, one far up: sign-flip p 2/64 + _cell(b, f"fix_{i}", "guidance:none", 1, turns=10) + _cell(b, f"fix_{i}", "guidance:raw", 1, turns=turns) + s = compare([b], "guidance:none") + turns = _row(s, "turns") + assert turns["p"] == pytest.approx(2 / 64) and turns["delta"]["lo"] < 0 < turns["delta"]["hi"] + assert turns["interval_p_disagree"] is True and _row(s, "gate")["interval_p_disagree"] is False + lines = format_compare(s).splitlines() + disagree = [line for line in lines if line.startswith(" turns: the interval")] + assert len(disagree) == 1 and "includes 0 but p 0.03125 is < 0.05" in disagree[0] + assert "a t interval and a sign-flip test" in disagree[0] and "read p" not in disagree[0] + assert (" p < 0.05 on turns: descriptive rows, their p uncorrected and outside Holm's family, which covers " + "gate and score only") in lines diff --git a/dokimi/tests/test_repo_reach.py b/dokimi/tests/test_repo_reach.py index 8c09fab39..bea0759fa 100644 --- a/dokimi/tests/test_repo_reach.py +++ b/dokimi/tests/test_repo_reach.py @@ -450,6 +450,25 @@ def test_tokens_include_task_tool_and_cache_fields(): assert t.usage_scope == "main" and t.subagent_calls == 1 +def test_first_prompt_tokens_is_the_first_main_thread_call_counted_once(): + def msg(mid, usage, parent=None): + return {"type": "assistant", "parent_tool_use_id": parent, + "message": {"id": mid, "usage": usage, "content": [{"type": "text", "text": "x"}]}} + lines = [ # a subagent's call comes first; the main call's three content blocks share one message id, + # and its first and last blocks' usage is sparse: the call's prompt is the merge, not the first or last line's + msg("s1", {"input_tokens": 9, "cache_read_input_tokens": 50_000}, parent="t1"), + msg("m1", {"input_tokens": 4, "cache_read_input_tokens": 16_315}), + msg("m1", {"input_tokens": 4, "cache_creation_input_tokens": 3_061, "cache_read_input_tokens": 16_315, + "output_tokens": 80}), + msg("m1", {"input_tokens": 4, "cache_read_input_tokens": 16_315, "output_tokens": 80}), + msg("m2", {"input_tokens": 1, "cache_read_input_tokens": 19_500}), + ] + assert convert("claude-stream-json", "\n".join(json.dumps(x) for x in lines)).first_prompt_tokens == 19_380 + no_usage = {"type": "assistant", "message": {"content": [{"type": "text", "text": "hi"}]}} + assert convert("claude-stream-json", json.dumps(no_usage)).first_prompt_tokens is None + assert convert("text", "hi").first_prompt_tokens is None + + @pytest.mark.parametrize("line", [ {"type": "assistant", "message": "hi"}, {"type": "result", "result": "x", "usage": "n/a"}, diff --git a/dokimi/tests/test_repo_run.py b/dokimi/tests/test_repo_run.py index bc1eb25ad..10e3cd147 100644 --- a/dokimi/tests/test_repo_run.py +++ b/dokimi/tests/test_repo_run.py @@ -401,7 +401,8 @@ def test_agent_timeout_is_killed_and_still_graded(setup, monkeypatch, capsys): rec = json.loads((setup["out"] / "b1/fix_add__guidance-raw__m-1__r1/result.json").read_text()) assert rec["agent"]["timed_out"] is True and rec["agent"]["rc"] is None and rec["agent"]["wall_s"] < 30 assert rec["grade"]["status"] == "graded" and rec["grade"]["success"] is False # the bug is still there - unmeasured = {k: rec["agent"][k] for k in ("input_tokens", "cache_read_tokens", "usage_scope", "subagent_calls")} + unmeasured = {k: rec["agent"][k] for k in ("input_tokens", "cache_read_tokens", "usage_scope", "subagent_calls", + "first_prompt_tokens")} assert unmeasured == {k: None for k in unmeasured} @@ -703,6 +704,7 @@ def test_result_json_carries_the_token_fields_of_the_whole_tree(setup, monkeypat assert (agent["input_tokens"], agent["output_tokens"], agent["cache_read_tokens"], agent["cache_write_tokens"]) == (37, 11, 100, 5) assert agent["usage_scope"] == "tree" and agent["subagent_calls"] == 1 + assert agent["first_prompt_tokens"] == 3 + 1200 + 15000 # the first main-thread call, not the run's total def test_a_host_error_after_the_agent_is_a_charged_grader_failure(setup, monkeypatch, capsys): @@ -729,7 +731,7 @@ def boom(*a, **kw): rec = _only_cell(setup) assert rec["grade"]["status"] == "grader_failure" and "unreadable trace" in rec["grade"]["reasons"][0] assert rec["agent"]["cost_usd"] is None and rec["agent"]["charged_usd"] == 0.5 - assert all(rec["agent"][k] is None for k in ("input_tokens", "usage_scope", "subagent_calls")) + assert all(rec["agent"][k] is None for k in ("input_tokens", "usage_scope", "subagent_calls", "first_prompt_tokens")) assert "1 ran" in (out := capsys.readouterr().out) and "$0.5000" in out