diff --git a/dokimi/changelog.d/20261026_130000_napetrov_report-eval-generate-agree.md b/dokimi/changelog.d/20261026_130000_napetrov_report-eval-generate-agree.md index cb5ece550..40ad1606f 100644 --- a/dokimi/changelog.d/20261026_130000_napetrov_report-eval-generate-agree.md +++ b/dokimi/changelog.d/20261026_130000_napetrov_report-eval-generate-agree.md @@ -11,10 +11,7 @@ says how many questions are unpaired (`paired_questions` and `unpaired_questions` in the JSON, and a note plus the Count cell in the Markdown). To see the new numbers, run `report generate` on an existing eval file again. - The top and bottom performer tables of `report generate` rank the complete pairs too. The - dashboard and the `report model-compare` run summary still average each arm over its own - scored questions, so for a file with an unpaired question their context and baseline means - differ from the ones `report generate` prints. + The top and bottom performer tables of `report generate` rank the complete pairs too. ### Fixed diff --git a/dokimi/changelog.d/20261029_120000_antonwolfy_paired-arm-means.md b/dokimi/changelog.d/20261029_120000_antonwolfy_paired-arm-means.md new file mode 100644 index 000000000..c04bd31d0 --- /dev/null +++ b/dokimi/changelog.d/20261029_120000_antonwolfy_paired-arm-means.md @@ -0,0 +1,14 @@ +### Fixed + +- **The dashboard and `report model-compare` average each arm over the complete pairs, as + `report generate` does** — the dashboard's per-library and per-source context and baseline + means, the context and baseline columns of the `report model-compare` Overall Summary and + its ranking, and the context column of its per-difficulty rows, used to average each arm over + every question scored in it, so the means could cover different questions than the delta + beside them. They now + cover only the questions scored in both arms, and the dashboard's delta is the difference of + those means rather than the mean of the stored per-question `delta`. For a run with a question + scored in one arm only, the arm means change, and on the dashboard the status band that + follows the context mean can change too. A run that scored one arm only still shows that + arm's mean; a run, source group or difficulty row with both arms scored but never on the same + question shows no means. diff --git a/dokimi/docs/benchmarking-and-comparison.md b/dokimi/docs/benchmarking-and-comparison.md index f1f27ab38..4d771b83e 100644 --- a/dokimi/docs/benchmarking-and-comparison.md +++ b/dokimi/docs/benchmarking-and-comparison.md @@ -568,7 +568,9 @@ The intended input is one judged arms JSON file per model for regular questions and/or golden questions. The report extracts the `baseline_arm` score as the baseline and the treatment arm as the context arm, then computes: -- overall context-arm, baseline, and delta per run, +- overall context-arm, baseline, and delta per run, all three over the questions + scored in both arms (a run that scored one arm only shows that arm's mean); the + context mean and delta of each difficulty row follow the same rule, - statistical significance of context-arm minus baseline (paired t-test, Wilcoxon signed-rank, Cohen's d_z), corrected across the compared runs \u2014 every run tests the same "did the treatment help?" question, so they form one diff --git a/dokimi/docs/quickstart.md b/dokimi/docs/quickstart.md index c1015317d..f95bc0317 100644 --- a/dokimi/docs/quickstart.md +++ b/dokimi/docs/quickstart.md @@ -173,7 +173,9 @@ python cli.py dashboard generate ``` Aggregates per-library results from `eval/*.json` into `DASHBOARD.md` / -`dashboard.json` for cross-library comparison. +`dashboard.json` for cross-library comparison. Like `report generate`, it averages both +arms and the delta, per library and per question source, over the questions scored in +both arms; a run that scored one arm only shows that arm's mean. ## Full run (single command) diff --git a/dokimi/dokimi/dashboard/aggregator.py b/dokimi/dokimi/dashboard/aggregator.py index 656b9d27e..9a1187293 100644 --- a/dokimi/dokimi/dashboard/aggregator.py +++ b/dokimi/dokimi/dashboard/aggregator.py @@ -149,9 +149,6 @@ def _as_float(value: Any) -> Optional[float]: return None questions = [] - with_scores = [] - without_scores = [] - deltas = [] for ev in recs: q_id = ev.question_id @@ -165,13 +162,6 @@ def _as_float(value: Any) -> Optional[float]: without_score = _as_float(ev.score(BASELINE)) delta = _as_float(ev.delta) - if with_score is not None: - with_scores.append(with_score) - if without_score is not None: - without_scores.append(without_score) - if delta is not None: - deltas.append(delta) - q_source = ev.raw.get("question_source", "persona") grounded = bool((with_eval or {}).get("grounded") or (without_eval or {}).get("grounded")) @@ -190,6 +180,7 @@ def _as_float(value: Any) -> Optional[float]: )) by_source = _compute_by_source(questions) + means = _arm_means(questions) return ProductSnapshot( product=product, @@ -197,9 +188,9 @@ def _as_float(value: Any) -> Optional[float]: evaluated_at=data.get("evaluated_at", ""), judge_model=data.get("judge_model", "unknown"), total_questions=len(questions), - avg_with_docs=round(sum(with_scores) / len(with_scores), 1) if with_scores else None, - avg_without_docs=round(sum(without_scores) / len(without_scores), 1) if without_scores else None, - avg_delta=round(sum(deltas) / len(deltas), 1) if deltas else None, + avg_with_docs=means["avg_with_docs"], + avg_without_docs=means["avg_without_docs"], + avg_delta=means["avg_delta"], questions=sorted(questions, key=lambda q: (q.with_docs_score or 0)), # worst first source_file=str(path), by_source=by_source, @@ -215,15 +206,35 @@ def _compute_by_source(questions: "List[QuestionResult]") -> "Dict[str, Dict]": result = {} for source, qs in groups.items(): - with_scores = [q.with_docs_score for q in qs if q.with_docs_score is not None] - without_scores = [q.without_docs_score for q in qs if q.without_docs_score is not None] - deltas = [q.delta for q in qs if q.delta is not None] grounded_count = sum(1 for q in qs if q.grounded) result[source] = { "count": len(qs), "grounded": grounded_count, - "avg_with_docs": round(sum(with_scores) / len(with_scores), 1) if with_scores else None, - "avg_without_docs": round(sum(without_scores) / len(without_scores), 1) if without_scores else None, - "avg_delta": round(sum(deltas) / len(deltas), 1) if deltas else None, + **_arm_means(qs), } return result + + +def _arm_means(questions: "List[QuestionResult]") -> "Dict[str, Optional[float]]": + """``avg_with_docs``, ``avg_without_docs`` and ``avg_delta`` over the complete pairs. + + The questions scored in both arms, the ones :func:`dokimi.eval.stats.paired_summary` + averages for ``report generate``, so the delta is the difference of the two means. + With no pair there is no delta: an arm keeps its mean over its own scores when the + other arm scored nothing (a baseline-only run), and otherwise neither mean is given, + so two means over different questions never stand side by side. + """ + with_s = [q.with_docs_score for q in questions if q.with_docs_score is not None] + without_s = [q.without_docs_score for q in questions if q.without_docs_score is not None] + pairs = [(q.with_docs_score, q.without_docs_score) for q in questions + if q.with_docs_score is not None and q.without_docs_score is not None] + if pairs: + with_s, without_s = [w for w, _ in pairs], [wo for _, wo in pairs] + elif with_s and without_s: + with_s = without_s = [] + + def _mean(vals: "List[float]") -> Optional[float]: + return round(sum(vals) / len(vals), 1) if vals else None + + return {"avg_with_docs": _mean(with_s), "avg_without_docs": _mean(without_s), + "avg_delta": _mean([w - wo for w, wo in pairs])} diff --git a/dokimi/dokimi/eval/stats.py b/dokimi/dokimi/eval/stats.py index fc0e22035..e6b169f5b 100644 --- a/dokimi/dokimi/eval/stats.py +++ b/dokimi/dokimi/eval/stats.py @@ -21,8 +21,8 @@ Per-question paired score vectors for two arms, complete cases only. paired_summary(evaluations) Both arms and their delta over the complete pairs, with the unpaired count. - Called by the eval report, the report generator and the pipeline summary. The - dashboard and the model-compare summary still average each arm separately. + Called by the eval report, the report generator and the pipeline summary; the + dashboard and the model-compare run summary average both arms over the same pairs. mean_ci(values, ...) Student-t CI for the mean, coherent with ``p_ttest``; bootstrap fallback. point_biserial(flags, totals) diff --git a/dokimi/dokimi/report/model_compare.py b/dokimi/dokimi/report/model_compare.py index ae1ac6629..90159883e 100644 --- a/dokimi/dokimi/report/model_compare.py +++ b/dokimi/dokimi/report/model_compare.py @@ -932,13 +932,13 @@ def _resource_usage_lines(runs: list[tuple[str, dict]], pooled: bool) -> list[st def _run_summary(scores: list[dict]) -> dict: """One run's arm means and paired-delta counts over the common questions.""" - with_docs_scores = [s["with_docs"] for s in scores if s["with_docs"] is not None] - without_docs_scores = [s["without_docs"] for s in scores if s["without_docs"] is not None] - # Average the per-question deltas rather than differencing the two arm - # means. The arm means are filtered independently, so when any question - # is missing a score they cover different question subsets and their - # difference is not a paired delta. - paired_deltas = [s["delta"] for s in scores if s.get("delta") is not None] + # Arm means and delta over the pairs (`eval.stats.paired_summary`); one-arm runs keep theirs. + pairs = [s for s in scores if s.get("delta") is not None] + with_docs_scores = [s["with_docs"] for s in pairs or scores if s["with_docs"] is not None] + without_docs_scores = [s["without_docs"] for s in pairs or scores if s["without_docs"] is not None] + if not pairs and with_docs_scores and without_docs_scores: # both arms, never on one question + with_docs_scores = without_docs_scores = [] + paired_deltas = [s["delta"] for s in pairs] # Per-run, distinct from the section-level has_treatment: a single run may # lack a treatment arm while others in the comparison have one. paired = bool(with_docs_scores) and bool(paired_deltas) @@ -1064,14 +1064,12 @@ def _difficulty_cells(sc: list[dict], has_treatment: bool) -> str: """One run's cells of a difficulty row: context mean and delta, or the baseline.""" if not sc: return " — | — |" if has_treatment else " — |" - with_scores = [s["with_docs"] for s in sc if s["with_docs"] is not None] - if with_scores: - # Same pairing rule as the overall summary: average the per-question - # deltas, never the difference of two means filtered over different - # question subsets. - diff_deltas = [s["delta"] for s in sc if s.get("delta") is not None] - delta_cell = _fmt_delta(round(sum(diff_deltas) / len(diff_deltas), 1)) if diff_deltas else "—" - return f" {_avg(with_scores)} | {delta_cell} |" + if any(s["with_docs"] is not None for s in sc): + # Same pairs and one-arm rule as the overall summary: the context mean + # covers the questions the delta does. + st = _run_summary(sc) + delta_cell = "—" if st["delta_avg"] is None else _fmt_delta(st["delta_avg"]) + return f" {_num_or_dash(st['with_avg'])} | {delta_cell} |" # No treatment score in this bucket: report the baseline. without_scores = [s["without_docs"] for s in sc if s["without_docs"] is not None] wo = _avg(without_scores) if without_scores else None diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index b0823eec7..7ad83253b 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -4,7 +4,7 @@ "dokimi/harnesses/benchflow.py": {"lines": 869}, "dokimi/harnesses/docker_solver.py": {"lines": 899, "rationale": "pre-existing when the gate was introduced"}, "dokimi/llm.py": {"lines": 1118, "rationale": "grew on main in #180 (replay a stored retry) after #93 seeded this baseline; split it in its own PR"}, - "dokimi/report/model_compare.py": {"lines": 1209, "rationale": "grew on main in #73 (judge-validity --context stored) before the ratchet landed; split it in its own PR"}, + "dokimi/report/model_compare.py": {"lines": 1207, "rationale": "grew on main in #73 (judge-validity --context stored) before the ratchet landed; split it in its own PR"}, "tests/test_arm_runner.py": {"lines": 1176, "rationale": "pre-existing when the gate was introduced"}, "tests/test_benchflow_harness.py": {"lines": 1716, "rationale": "pre-existing when the gate was introduced"}, "tests/test_container_agent.py": {"lines": 1251, "rationale": "pre-existing when the gate was introduced"}, @@ -30,7 +30,6 @@ "dokimi/commands/report.py::cmd_report_model_compare": {"complexity": 12}, "dokimi/commands/tools.py::cmd_tools_preference": {"complexity": 11}, "dokimi/context/trace.py::_apply": {"complexity": 18}, - "dokimi/dashboard/aggregator.py::ResultsAggregator._load_snapshot": {"complexity": 13}, "dokimi/eval/agent_runner.py::_summarize_tool_usage": {"complexity": 11}, "dokimi/eval/agent_runner.py::run_agent_loop": {"complexity": 14}, "dokimi/eval/arm_runner.py::ArmRunner.judge": {"complexity": 11}, diff --git a/dokimi/tests/fixtures/model_compare_sections.json b/dokimi/tests/fixtures/model_compare_sections.json index 8c953e340..ac3bb12d7 100644 --- a/dokimi/tests/fixtures/model_compare_sections.json +++ b/dokimi/tests/fixtures/model_compare_sections.json @@ -3,18 +3,18 @@ "baseline_only_no_scipy": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | m (p) | same (p) | — (baseline only) |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n> **⚠️ Report Limitation:** These runs contain only **baseline arm** (no treatment comparison).\n> \n> **Missing from this report:**\n> - Context-arm vs Baseline delta analysis\n> - Statistical significance tests\n> - Head-to-head winner comparisons\n> \n> **Available in this report:**\n> - Absolute baseline quality scores per model\n> - Resource usage metrics (cost, tokens, latency)\n> - Model ranking by absolute baseline quality\n> \n> **To generate full comparison with delta analysis, add a treatment arm:**\n> ```bash\n> dokimi arms run \\\n> --product oneTBB --questions questions/oneTBB.json \\\n> --arms \"baseline,docs\" \\\n> --judge \\\n> --judge-model claude-sonnet-5-5\n> ```\n> \n> Treatment arm options: `docs` (Context7 documentation), `skill:` (local skill), `agent` (agentic doc retrieval).\n> \n> See: `docs/benchmarking-and-comparison.md#treatment-arm-comparison`\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | Context arm | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 0 (9 unscored) | — | 56.2 | — | — | — |\n\n## By Difficulty (common questions only)\n| Difficulty | N | a baseline |\n|---|---:|| :---: |\n| easy | 3 | 60.0 |\n| beginner | 1 | 55.0 |\n| intermediate | 1 | 30.0 |\n| medium | 1 | 60.0 |\n| advanced | 1 | 65.0 |\n| hard | 1 | 40.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n\n> **⚠️ Skipped:** Head-to-head requires treatment arm scores.\n> Single-arm runs only have baseline scores.\n\n## Context-Arm Benefit — Delta Comparison\n\n> **⚠️ Skipped:** Delta comparison requires treatment arm.\n> Single-arm runs cannot compute context benefit.\n\n## Model Ranking\n\n### By absolute quality (baseline)\n| Rank | Run | Baseline avg |\n|---:|---|---:|\n| 1 | **a** | 56.2 |\n\n> **Note:** Treatment utilization ranking requires treatment arm.\n\n", "baseline_zero": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | m (p) | same (p) | — (baseline only) |\n\nCommon questions evaluated: **3**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n> **⚠️ Report Limitation:** These runs contain only **baseline arm** (no treatment comparison).\n> \n> **Missing from this report:**\n> - Context-arm vs Baseline delta analysis\n> - Statistical significance tests\n> - Head-to-head winner comparisons\n> \n> **Available in this report:**\n> - Absolute baseline quality scores per model\n> - Resource usage metrics (cost, tokens, latency)\n> - Model ranking by absolute baseline quality\n> \n> **To generate full comparison with delta analysis, add a treatment arm:**\n> ```bash\n> dokimi arms run \\\n> --product oneTBB --questions questions/oneTBB.json \\\n> --arms \"baseline,docs\" \\\n> --judge \\\n> --judge-model claude-sonnet-5-5\n> ```\n> \n> Treatment arm options: `docs` (Context7 documentation), `skill:` (local skill), `agent` (agentic doc retrieval).\n> \n> See: `docs/benchmarking-and-comparison.md#treatment-arm-comparison`\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | Context arm | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 3 | 0 (3 unscored) | — | 30.0 | — | — | — |\n\n## Statistical Significance of Delta (α = 0.05)\n\n> **⚠️ Skipped:** Statistical tests require treatment arm (e.g., docs, skills).\n> Add `--arms \"baseline,docs\"` to enable delta analysis.\n\n\n## By Difficulty (common questions only)\n| Difficulty | N | a baseline |\n|---|---:|| :---: |\n| easy | 1 | 0.0 |\n| medium | 1 | 40.0 |\n| hard | 1 | 50.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n\n> **⚠️ Skipped:** Head-to-head requires treatment arm scores.\n> Single-arm runs only have baseline scores.\n\n## Context-Arm Benefit — Delta Comparison\n\n> **⚠️ Skipped:** Delta comparison requires treatment arm.\n> Single-arm runs cannot compute context benefit.\n\n## Model Ranking\n\n### By absolute quality (baseline)\n| Rank | Run | Baseline avg |\n|---:|---|---:|\n| 1 | **a** | 30.0 |\n\n> **Note:** Treatment utilization ranking requires treatment arm.\n\n", "empty": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n\nCommon questions evaluated: **0**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n> **⚠️ Report Limitation:** These runs contain only **baseline arm** (no treatment comparison).\n> \n> **Missing from this report:**\n> - Context-arm vs Baseline delta analysis\n> - Statistical significance tests\n> - Head-to-head winner comparisons\n> \n> **Available in this report:**\n> - Absolute baseline quality scores per model\n> - Resource usage metrics (cost, tokens, latency)\n> - Model ranking by absolute baseline quality\n> \n> **To generate full comparison with delta analysis, add a treatment arm:**\n> ```bash\n> dokimi arms run \\\n> --product oneTBB --questions questions/oneTBB.json \\\n> --arms \"baseline,docs\" \\\n> --judge \\\n> --judge-model claude-sonnet-5-5\n> ```\n> \n> Treatment arm options: `docs` (Context7 documentation), `skill:` (local skill), `agent` (agentic doc retrieval).\n> \n> See: `docs/benchmarking-and-comparison.md#treatment-arm-comparison`\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | Context arm | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n\n## Statistical Significance of Delta (α = 0.05)\n\n> **⚠️ Skipped:** Statistical tests require treatment arm (e.g., docs, skills).\n> Add `--arms \"baseline,docs\"` to enable delta analysis.\n\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n\n> **⚠️ Skipped:** Head-to-head requires treatment arm scores.\n> Single-arm runs only have baseline scores.\n\n## Context-Arm Benefit — Delta Comparison\n\n> **⚠️ Skipped:** Delta comparison requires treatment arm.\n> Single-arm runs cannot compute context benefit.\n\n## Model Ranking\n\n### By absolute quality (baseline)\n| Rank | Run | Baseline avg |\n|---:|---|---:|\n\n> **Note:** Treatment utilization ranking requires treatment arm.\n\n", - "mixed": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | alpha (p) | same (p) | `with-context` |\n| **b** | beta (p) | same (p) | — (baseline only) |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 7 (2 unscored) | 68.8 | 56.2 | **+17.1** | 6 | 1 |\n| **b** | 9 | 0 (9 unscored) | — | 56.2 | — | — | — |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **a** | +17.1 | 0.0247 | 0.0247 | 0.0469 | +1.1 | large | OK Yes |\n\n## By Difficulty (common questions only)\n| Difficulty | N | a context | a Δ | b context | b Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 3 | 63.3 | +5.0 | — | — |\n| beginner | 1 | 75.0 | +20.0 | — | — |\n| intermediate | 1 | 45.0 | +15.0 | — | — |\n| medium | 1 | 70.0 | +10.0 | — | — |\n| advanced | 1 | 90.0 | +25.0 | — | — |\n| hard | 1 | 80.0 | +40.0 | — | 56.2 (baseline) |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **a** | 8 | 89% |\n| **b** | 0 | 0% |\n| tie | 0 | 0% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **a** | **+17.1** | 6 (86%) | 1 (14%) |\n| **b** | — | — | — |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **a** | 68.8 |\n| 2 | **b** | — |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **a** | +17.1 |\n| 2 | **b** | N/A |\n", - "no_scipy": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | alpha (p) | judge-j (jp) | `with-context` |\n| **b** | beta (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Resource Usage (baseline arm)\n| Run | Total Cost | Prompt tok | Completion tok | Total tok | Avg latency/Q | Cache hit |\n|---|---:|---:|---:|---:|---:|---:|\n| **a** | $0.1235 (3/8) | 12,345 | 678 | 13,023 | 1.23s | 25.0% |\n| **b** | $1.5000 | — | — | — | — | — |\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 7 (2 unscored) | 68.8 | 56.2 | **+17.1** | 6 | 1 |\n| **b** | 9 | 9 | 62.2 | 52.2 | **+10.0** | 9 | 0 |\n\n## By Difficulty (common questions only)\n| Difficulty | N | a context | a Δ | b context | b Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 3 | 63.3 | +5.0 | 56.7 | +10.0 |\n| beginner | 1 | 75.0 | +20.0 | 65.0 | +10.0 |\n| intermediate | 1 | 45.0 | +15.0 | 40.0 | +10.0 |\n| medium | 1 | 70.0 | +10.0 | 70.0 | +10.0 |\n| advanced | 1 | 90.0 | +25.0 | 75.0 | +10.0 |\n| hard | 1 | 80.0 | +40.0 | 50.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **a** | 6 | 67% |\n| **b** | 2 | 22% |\n| tie | 1 | 11% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **a** | **+17.1** | 6 (86%) | 1 (14%) |\n| **b** | **+10.0** | 9 (100%) | 0 (0%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **a** | 68.8 |\n| 2 | **b** | 62.2 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **a** | +17.1 |\n| 2 | **b** | +10.0 |\n", - "pinned_arm": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | m (p) | same (p) | `docs` |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | docs | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 7 (2 unscored) | 68.8 | 56.2 | **+17.1** | 6 | 1 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **a** | +17.1 | 0.0247 | 0.0247 | 0.0469 | +1.1 | large | OK Yes |\n\n## By Difficulty (common questions only)\n| Difficulty | N | a context | a Δ |\n|---|---:| :---: | :---: |\n| easy | 9 | 68.8 | +17.1 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **a** | 8 | 89% |\n| tie | 0 | 0% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `docs`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **a** | **+17.1** | 6 (86%) | 1 (14%) |\n\n## Model Ranking\n\n### By absolute quality (docs)\n| Rank | Run | docs avg |\n|---:|---|---:|\n| 1 | **a** | 68.8 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **a** | +17.1 |\n", + "mixed": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | alpha (p) | same (p) | `with-context` |\n| **b** | beta (p) | same (p) | — (baseline only) |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 7 (2 unscored) | 70.0 | 52.9 | **+17.1** | 6 | 1 |\n| **b** | 9 | 0 (9 unscored) | — | 56.2 | — | — | — |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **a** | +17.1 | 0.0247 | 0.0247 | 0.0469 | +1.1 | large | OK Yes |\n\n## By Difficulty (common questions only)\n| Difficulty | N | a context | a Δ | b context | b Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 3 | 65.0 | +5.0 | — | — |\n| beginner | 1 | 75.0 | +20.0 | — | — |\n| intermediate | 1 | 45.0 | +15.0 | — | — |\n| medium | 1 | 70.0 | +10.0 | — | — |\n| advanced | 1 | 90.0 | +25.0 | — | — |\n| hard | 1 | 80.0 | +40.0 | — | 56.2 (baseline) |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **a** | 8 | 89% |\n| **b** | 0 | 0% |\n| tie | 0 | 0% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **a** | **+17.1** | 6 (86%) | 1 (14%) |\n| **b** | — | — | — |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **a** | 70.0 |\n| 2 | **b** | — |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **a** | +17.1 |\n| 2 | **b** | N/A |\n", + "no_scipy": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | alpha (p) | judge-j (jp) | `with-context` |\n| **b** | beta (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Resource Usage (baseline arm)\n| Run | Total Cost | Prompt tok | Completion tok | Total tok | Avg latency/Q | Cache hit |\n|---|---:|---:|---:|---:|---:|---:|\n| **a** | $0.1235 (3/8) | 12,345 | 678 | 13,023 | 1.23s | 25.0% |\n| **b** | $1.5000 | — | — | — | — | — |\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 7 (2 unscored) | 70.0 | 52.9 | **+17.1** | 6 | 1 |\n| **b** | 9 | 9 | 62.2 | 52.2 | **+10.0** | 9 | 0 |\n\n## By Difficulty (common questions only)\n| Difficulty | N | a context | a Δ | b context | b Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 3 | 65.0 | +5.0 | 56.7 | +10.0 |\n| beginner | 1 | 75.0 | +20.0 | 65.0 | +10.0 |\n| intermediate | 1 | 45.0 | +15.0 | 40.0 | +10.0 |\n| medium | 1 | 70.0 | +10.0 | 70.0 | +10.0 |\n| advanced | 1 | 90.0 | +25.0 | 75.0 | +10.0 |\n| hard | 1 | 80.0 | +40.0 | 50.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **a** | 6 | 67% |\n| **b** | 2 | 22% |\n| tie | 1 | 11% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **a** | **+17.1** | 6 (86%) | 1 (14%) |\n| **b** | **+10.0** | 9 (100%) | 0 (0%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **a** | 70.0 |\n| 2 | **b** | 62.2 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **a** | +17.1 |\n| 2 | **b** | +10.0 |\n", + "pinned_arm": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | m (p) | same (p) | `docs` |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | docs | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 7 (2 unscored) | 70.0 | 52.9 | **+17.1** | 6 | 1 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **a** | +17.1 | 0.0247 | 0.0247 | 0.0469 | +1.1 | large | OK Yes |\n\n## By Difficulty (common questions only)\n| Difficulty | N | a context | a Δ |\n|---|---:| :---: | :---: |\n| easy | 9 | 70.0 | +17.1 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **a** | 8 | 89% |\n| tie | 0 | 0% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `docs`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **a** | **+17.1** | 6 (86%) | 1 (14%) |\n\n## Model Ranking\n\n### By absolute quality (docs)\n| Rank | Run | docs avg |\n|---:|---|---:|\n| 1 | **a** | 70.0 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **a** | +17.1 |\n", "pooled": "## Replicate Pooling\n\nRuns sharing a configuration were averaged per question, so each row below is one configuration rather than one sample of it. Each pooled score is a mean of k runs, so its spread and confidence intervals are narrower than a single run's: that reflects the k runs spent, not a sharper measurement.\n\n- **m[n=2]'** pools 2 replicates: m_run1, m_run2; 2 question(s) dropped for not being scored in every replicate\n\n## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **m[n=2]** | solo (p) | same (p) | `with-context` |\n| **m[n=2]'** | rep (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **7**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Resource Usage (baseline arm)\n| Run | Total Cost | Prompt tok | Completion tok | Total tok | Avg latency/Q | Cache hit |\n|---|---:|---:|---:|---:|---:|---:|\n| **m[n=2]** | — | — | — | — | — | — |\n| **m[n=2]'** | $0.2469 (6/16) | 24,690 | 1,356 | 26,046 | 1.23s | 25.0% |\n\n_A pooled `[n=k]` row shows the total cost, calls and tokens of its k runs and their mean latency._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **m[n=2]** | 7 | 7 | 62.9 | 52.9 | **+10.0** | 7 | 0 |\n| **m[n=2]'** | 7 | 7 | 66.4 | 52.9 | **+13.6** | 6 | 0 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **m[n=2]** | +10.0 | N/A | N/A | N/A | N/A | unknown | N/A |\n| **m[n=2]'** | +13.6 | 0.0033 | 0.0033 | 0.0312 | +1.8 | large | OK Yes |\n\n## By Difficulty (common questions only)\n| Difficulty | N | m[n=2] context | m[n=2] Δ | m[n=2]' context | m[n=2]' Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 2 | 70.0 | +10.0 | 67.5 | +7.5 |\n| beginner | 1 | 65.0 | +10.0 | 70.0 | +15.0 |\n| intermediate | 1 | 40.0 | +10.0 | 42.5 | +12.5 |\n| medium | 1 | 70.0 | +10.0 | 70.0 | +10.0 |\n| advanced | 1 | 75.0 | +10.0 | 82.5 | +17.5 |\n| hard | 1 | 50.0 | +10.0 | 65.0 | +25.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **m[n=2]** | 1 | 14% |\n| **m[n=2]'** | 5 | 71% |\n| tie | 1 | 14% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **m[n=2]** | **+10.0** | 7 (100%) | 0 (0%) |\n| **m[n=2]'** | **+13.6** | 6 (86%) | 0 (0%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **m[n=2]'** | 66.4 |\n| 2 | **m[n=2]** | 62.9 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **m[n=2]'** | +13.6 |\n| 2 | **m[n=2]** | +10.0 |\n", - "pooled_singletons": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **x** | x (p) | same (p) | `with-context` |\n| **y** | y (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **x** | 9 | 7 (2 unscored) | 68.8 | 56.2 | **+17.1** | 6 | 1 |\n| **y** | 9 | 9 | 62.2 | 52.2 | **+10.0** | 9 | 0 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **x** | +17.1 | 0.0247 | 0.0247 | 0.0469 | +1.1 | large | OK Yes |\n| **y** | +10.0 | N/A | N/A | N/A | N/A | unknown | N/A |\n\n## By Difficulty (common questions only)\n| Difficulty | N | x context | x Δ | y context | y Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 3 | 63.3 | +5.0 | 56.7 | +10.0 |\n| beginner | 1 | 75.0 | +20.0 | 65.0 | +10.0 |\n| intermediate | 1 | 45.0 | +15.0 | 40.0 | +10.0 |\n| medium | 1 | 70.0 | +10.0 | 70.0 | +10.0 |\n| advanced | 1 | 90.0 | +25.0 | 75.0 | +10.0 |\n| hard | 1 | 80.0 | +40.0 | 50.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **x** | 6 | 67% |\n| **y** | 2 | 22% |\n| tie | 1 | 11% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **x** | **+17.1** | 6 (86%) | 1 (14%) |\n| **y** | **+10.0** | 9 (100%) | 0 (0%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **x** | 68.8 |\n| 2 | **y** | 62.2 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **x** | +17.1 |\n| 2 | **y** | +10.0 |\n", - "pooled_singletons_cost": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **x** | x (p) | same (p) | `with-context` |\n| **y** | y (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Resource Usage (baseline arm)\n| Run | Total Cost | Prompt tok | Completion tok | Total tok | Avg latency/Q | Cache hit |\n|---|---:|---:|---:|---:|---:|---:|\n| **x** | $0.1235 (3/8) | 12,345 | 678 | 13,023 | 1.23s | 25.0% |\n| **y** | $0.5000 | — | — | — | — | — |\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **x** | 9 | 7 (2 unscored) | 68.8 | 56.2 | **+17.1** | 6 | 1 |\n| **y** | 9 | 9 | 62.2 | 52.2 | **+10.0** | 9 | 0 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **x** | +17.1 | 0.0247 | 0.0247 | 0.0469 | +1.1 | large | OK Yes |\n| **y** | +10.0 | N/A | N/A | N/A | N/A | unknown | N/A |\n\n## By Difficulty (common questions only)\n| Difficulty | N | x context | x Δ | y context | y Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 3 | 63.3 | +5.0 | 56.7 | +10.0 |\n| beginner | 1 | 75.0 | +20.0 | 65.0 | +10.0 |\n| intermediate | 1 | 45.0 | +15.0 | 40.0 | +10.0 |\n| medium | 1 | 70.0 | +10.0 | 70.0 | +10.0 |\n| advanced | 1 | 90.0 | +25.0 | 75.0 | +10.0 |\n| hard | 1 | 80.0 | +40.0 | 50.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **x** | 6 | 67% |\n| **y** | 2 | 22% |\n| tie | 1 | 11% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **x** | **+17.1** | 6 (86%) | 1 (14%) |\n| **y** | **+10.0** | 9 (100%) | 0 (0%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **x** | 68.8 |\n| 2 | **y** | 62.2 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **x** | +17.1 |\n| 2 | **y** | +10.0 |\n", - "treatment": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | alpha (p) | judge-j (jp) | `with-context` |\n| **b** | beta (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Resource Usage (baseline arm)\n| Run | Total Cost | Prompt tok | Completion tok | Total tok | Avg latency/Q | Cache hit |\n|---|---:|---:|---:|---:|---:|---:|\n| **a** | $0.1235 (3/8) | 12,345 | 678 | 13,023 | 1.23s | 25.0% |\n| **b** | $1.5000 | — | — | — | — | — |\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 7 (2 unscored) | 68.8 | 56.2 | **+17.1** | 6 | 1 |\n| **b** | 9 | 9 | 62.2 | 52.2 | **+10.0** | 9 | 0 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **a** | +17.1 | 0.0247 | 0.0247 | 0.0469 | +1.1 | large | OK Yes |\n| **b** | +10.0 | N/A | N/A | N/A | N/A | unknown | N/A |\n\n## By Difficulty (common questions only)\n| Difficulty | N | a context | a Δ | b context | b Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 3 | 63.3 | +5.0 | 56.7 | +10.0 |\n| beginner | 1 | 75.0 | +20.0 | 65.0 | +10.0 |\n| intermediate | 1 | 45.0 | +15.0 | 40.0 | +10.0 |\n| medium | 1 | 70.0 | +10.0 | 70.0 | +10.0 |\n| advanced | 1 | 90.0 | +25.0 | 75.0 | +10.0 |\n| hard | 1 | 80.0 | +40.0 | 50.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **a** | 6 | 67% |\n| **b** | 2 | 22% |\n| tie | 1 | 11% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **a** | **+17.1** | 6 (86%) | 1 (14%) |\n| **b** | **+10.0** | 9 (100%) | 0 (0%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **a** | 68.8 |\n| 2 | **b** | 62.2 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **a** | +17.1 |\n| 2 | **b** | +10.0 |\n", - "treatment_bh": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | alpha (p) | judge-j (jp) | `with-context` |\n| **b** | beta (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Resource Usage (baseline arm)\n| Run | Total Cost | Prompt tok | Completion tok | Total tok | Avg latency/Q | Cache hit |\n|---|---:|---:|---:|---:|---:|---:|\n| **a** | $0.1235 (3/8) | 12,345 | 678 | 13,023 | 1.23s | 25.0% |\n| **b** | $1.5000 | — | — | — | — | — |\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 7 (2 unscored) | 68.8 | 56.2 | **+17.1** | 6 | 1 |\n| **b** | 9 | 9 | 62.2 | 52.2 | **+10.0** | 9 | 0 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Benjamini-Hochberg); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **a** | +17.1 | 0.0247 | 0.0247 | 0.0469 | +1.1 | large | OK Yes |\n| **b** | +10.0 | N/A | N/A | N/A | N/A | unknown | N/A |\n\n## By Difficulty (common questions only)\n| Difficulty | N | a context | a Δ | b context | b Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 3 | 63.3 | +5.0 | 56.7 | +10.0 |\n| beginner | 1 | 75.0 | +20.0 | 65.0 | +10.0 |\n| intermediate | 1 | 45.0 | +15.0 | 40.0 | +10.0 |\n| medium | 1 | 70.0 | +10.0 | 70.0 | +10.0 |\n| advanced | 1 | 90.0 | +25.0 | 75.0 | +10.0 |\n| hard | 1 | 80.0 | +40.0 | 50.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **a** | 6 | 67% |\n| **b** | 2 | 22% |\n| tie | 1 | 11% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **a** | **+17.1** | 6 (86%) | 1 (14%) |\n| **b** | **+10.0** | 9 (100%) | 0 (0%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **a** | 68.8 |\n| 2 | **b** | 62.2 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **a** | +17.1 |\n| 2 | **b** | +10.0 |\n", - "two_arms": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | alpha (p) | same (p) | `docs` |\n| **b** | beta (p) | same (p) | `skill:x` |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | Context arm | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 7 (2 unscored) | 68.8 | 56.2 | **+17.1** | 6 | 1 |\n| **b** | 9 | 9 | 62.2 | 52.2 | **+10.0** | 9 | 0 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **a** | +17.1 | 0.0247 | 0.0247 | 0.0469 | +1.1 | large | OK Yes |\n| **b** | +10.0 | N/A | N/A | N/A | N/A | unknown | N/A |\n\n## By Difficulty (common questions only)\n| Difficulty | N | a context | a Δ | b context | b Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 3 | 63.3 | +5.0 | 56.7 | +10.0 |\n| beginner | 1 | 75.0 | +20.0 | 65.0 | +10.0 |\n| intermediate | 1 | 45.0 | +15.0 | 40.0 | +10.0 |\n| medium | 1 | 70.0 | +10.0 | 70.0 | +10.0 |\n| advanced | 1 | 90.0 | +25.0 | 75.0 | +10.0 |\n| hard | 1 | 80.0 | +40.0 | 50.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **a** | 6 | 67% |\n| **b** | 2 | 22% |\n| tie | 1 | 11% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the non-baseline treatment arm?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **a** | **+17.1** | 6 (86%) | 1 (14%) |\n| **b** | **+10.0** | 9 (100%) | 0 (0%) |\n\n## Model Ranking\n\n### By absolute quality (Context arm)\n| Rank | Run | Context arm avg |\n|---:|---|---:|\n| 1 | **a** | 68.8 |\n| 2 | **b** | 62.2 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **a** | +17.1 |\n| 2 | **b** | +10.0 |\n", - "unscored_and_arms": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **u** | m (p) | same (p) | `docs`, `skill:x` |\n| **v** | m (p) | same (p) | `docs` |\n\nCommon questions evaluated: **8**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Resource Usage (baseline arm)\n| Run | Total Cost | Prompt tok | Completion tok | Total tok | Avg latency/Q | Cache hit |\n|---|---:|---:|---:|---:|---:|---:|\n| **u** | $2.0000 (4/5) | — | — | — | — | — |\n| **v** | $3.0000 | — | — | — | — | — |\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | Context arm | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **u** | 8 | 7 (1 unscored) | 64.3 | 56.2 | **+11.4** | 7 | 0 |\n| **v** | 8 | 7 (1 unscored) | 64.3 | 56.2 | **+11.4** | 7 | 0 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 2 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **u** | +11.4 | 0.0016 | 0.0032 | 0.0156 | +2.1 | large | OK Yes |\n| **v** | +11.4 | 0.0016 | 0.0032 | 0.0156 | +2.1 | large | OK Yes |\n\n## By Difficulty (common questions only)\n| Difficulty | N | u context | u Δ | v context | v Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 2 | 70.0 | +10.0 | 70.0 | +10.0 |\n| beginner | 1 | 70.0 | +15.0 | 70.0 | +15.0 |\n| intermediate | 1 | 50.0 | +20.0 | 50.0 | +20.0 |\n| medium | 1 | 75.0 | +15.0 | 75.0 | +15.0 |\n| advanced | 1 | 70.0 | +5.0 | 70.0 | +5.0 |\n| hard | 1 | 45.0 | +5.0 | 45.0 | +5.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **u** | 0 | 0% |\n| **v** | 0 | 0% |\n| tie | 7 | 88% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the non-baseline treatment arm?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **u** | **+11.4** | 7 (100%) | 0 (0%) |\n| **v** | **+11.4** | 7 (100%) | 0 (0%) |\n\n## Model Ranking\n\n### By absolute quality (Context arm)\n| Rank | Run | Context arm avg |\n|---:|---|---:|\n| 1 | **u** | 64.3 |\n| 2 | **v** | 64.3 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **u** | +11.4 |\n| 2 | **v** | +11.4 |\n", + "pooled_singletons": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **x** | x (p) | same (p) | `with-context` |\n| **y** | y (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **x** | 9 | 7 (2 unscored) | 70.0 | 52.9 | **+17.1** | 6 | 1 |\n| **y** | 9 | 9 | 62.2 | 52.2 | **+10.0** | 9 | 0 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **x** | +17.1 | 0.0247 | 0.0247 | 0.0469 | +1.1 | large | OK Yes |\n| **y** | +10.0 | N/A | N/A | N/A | N/A | unknown | N/A |\n\n## By Difficulty (common questions only)\n| Difficulty | N | x context | x Δ | y context | y Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 3 | 65.0 | +5.0 | 56.7 | +10.0 |\n| beginner | 1 | 75.0 | +20.0 | 65.0 | +10.0 |\n| intermediate | 1 | 45.0 | +15.0 | 40.0 | +10.0 |\n| medium | 1 | 70.0 | +10.0 | 70.0 | +10.0 |\n| advanced | 1 | 90.0 | +25.0 | 75.0 | +10.0 |\n| hard | 1 | 80.0 | +40.0 | 50.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **x** | 6 | 67% |\n| **y** | 2 | 22% |\n| tie | 1 | 11% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **x** | **+17.1** | 6 (86%) | 1 (14%) |\n| **y** | **+10.0** | 9 (100%) | 0 (0%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **x** | 70.0 |\n| 2 | **y** | 62.2 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **x** | +17.1 |\n| 2 | **y** | +10.0 |\n", + "pooled_singletons_cost": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **x** | x (p) | same (p) | `with-context` |\n| **y** | y (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Resource Usage (baseline arm)\n| Run | Total Cost | Prompt tok | Completion tok | Total tok | Avg latency/Q | Cache hit |\n|---|---:|---:|---:|---:|---:|---:|\n| **x** | $0.1235 (3/8) | 12,345 | 678 | 13,023 | 1.23s | 25.0% |\n| **y** | $0.5000 | — | — | — | — | — |\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **x** | 9 | 7 (2 unscored) | 70.0 | 52.9 | **+17.1** | 6 | 1 |\n| **y** | 9 | 9 | 62.2 | 52.2 | **+10.0** | 9 | 0 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **x** | +17.1 | 0.0247 | 0.0247 | 0.0469 | +1.1 | large | OK Yes |\n| **y** | +10.0 | N/A | N/A | N/A | N/A | unknown | N/A |\n\n## By Difficulty (common questions only)\n| Difficulty | N | x context | x Δ | y context | y Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 3 | 65.0 | +5.0 | 56.7 | +10.0 |\n| beginner | 1 | 75.0 | +20.0 | 65.0 | +10.0 |\n| intermediate | 1 | 45.0 | +15.0 | 40.0 | +10.0 |\n| medium | 1 | 70.0 | +10.0 | 70.0 | +10.0 |\n| advanced | 1 | 90.0 | +25.0 | 75.0 | +10.0 |\n| hard | 1 | 80.0 | +40.0 | 50.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **x** | 6 | 67% |\n| **y** | 2 | 22% |\n| tie | 1 | 11% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **x** | **+17.1** | 6 (86%) | 1 (14%) |\n| **y** | **+10.0** | 9 (100%) | 0 (0%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **x** | 70.0 |\n| 2 | **y** | 62.2 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **x** | +17.1 |\n| 2 | **y** | +10.0 |\n", + "treatment": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | alpha (p) | judge-j (jp) | `with-context` |\n| **b** | beta (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Resource Usage (baseline arm)\n| Run | Total Cost | Prompt tok | Completion tok | Total tok | Avg latency/Q | Cache hit |\n|---|---:|---:|---:|---:|---:|---:|\n| **a** | $0.1235 (3/8) | 12,345 | 678 | 13,023 | 1.23s | 25.0% |\n| **b** | $1.5000 | — | — | — | — | — |\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 7 (2 unscored) | 70.0 | 52.9 | **+17.1** | 6 | 1 |\n| **b** | 9 | 9 | 62.2 | 52.2 | **+10.0** | 9 | 0 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **a** | +17.1 | 0.0247 | 0.0247 | 0.0469 | +1.1 | large | OK Yes |\n| **b** | +10.0 | N/A | N/A | N/A | N/A | unknown | N/A |\n\n## By Difficulty (common questions only)\n| Difficulty | N | a context | a Δ | b context | b Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 3 | 65.0 | +5.0 | 56.7 | +10.0 |\n| beginner | 1 | 75.0 | +20.0 | 65.0 | +10.0 |\n| intermediate | 1 | 45.0 | +15.0 | 40.0 | +10.0 |\n| medium | 1 | 70.0 | +10.0 | 70.0 | +10.0 |\n| advanced | 1 | 90.0 | +25.0 | 75.0 | +10.0 |\n| hard | 1 | 80.0 | +40.0 | 50.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **a** | 6 | 67% |\n| **b** | 2 | 22% |\n| tie | 1 | 11% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **a** | **+17.1** | 6 (86%) | 1 (14%) |\n| **b** | **+10.0** | 9 (100%) | 0 (0%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **a** | 70.0 |\n| 2 | **b** | 62.2 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **a** | +17.1 |\n| 2 | **b** | +10.0 |\n", + "treatment_bh": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | alpha (p) | judge-j (jp) | `with-context` |\n| **b** | beta (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Resource Usage (baseline arm)\n| Run | Total Cost | Prompt tok | Completion tok | Total tok | Avg latency/Q | Cache hit |\n|---|---:|---:|---:|---:|---:|---:|\n| **a** | $0.1235 (3/8) | 12,345 | 678 | 13,023 | 1.23s | 25.0% |\n| **b** | $1.5000 | — | — | — | — | — |\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 7 (2 unscored) | 70.0 | 52.9 | **+17.1** | 6 | 1 |\n| **b** | 9 | 9 | 62.2 | 52.2 | **+10.0** | 9 | 0 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Benjamini-Hochberg); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **a** | +17.1 | 0.0247 | 0.0247 | 0.0469 | +1.1 | large | OK Yes |\n| **b** | +10.0 | N/A | N/A | N/A | N/A | unknown | N/A |\n\n## By Difficulty (common questions only)\n| Difficulty | N | a context | a Δ | b context | b Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 3 | 65.0 | +5.0 | 56.7 | +10.0 |\n| beginner | 1 | 75.0 | +20.0 | 65.0 | +10.0 |\n| intermediate | 1 | 45.0 | +15.0 | 40.0 | +10.0 |\n| medium | 1 | 70.0 | +10.0 | 70.0 | +10.0 |\n| advanced | 1 | 90.0 | +25.0 | 75.0 | +10.0 |\n| hard | 1 | 80.0 | +40.0 | 50.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **a** | 6 | 67% |\n| **b** | 2 | 22% |\n| tie | 1 | 11% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **a** | **+17.1** | 6 (86%) | 1 (14%) |\n| **b** | **+10.0** | 9 (100%) | 0 (0%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **a** | 70.0 |\n| 2 | **b** | 62.2 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **a** | +17.1 |\n| 2 | **b** | +10.0 |\n", + "two_arms": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | alpha (p) | same (p) | `docs` |\n| **b** | beta (p) | same (p) | `skill:x` |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | Context arm | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 7 (2 unscored) | 70.0 | 52.9 | **+17.1** | 6 | 1 |\n| **b** | 9 | 9 | 62.2 | 52.2 | **+10.0** | 9 | 0 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **a** | +17.1 | 0.0247 | 0.0247 | 0.0469 | +1.1 | large | OK Yes |\n| **b** | +10.0 | N/A | N/A | N/A | N/A | unknown | N/A |\n\n## By Difficulty (common questions only)\n| Difficulty | N | a context | a Δ | b context | b Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 3 | 65.0 | +5.0 | 56.7 | +10.0 |\n| beginner | 1 | 75.0 | +20.0 | 65.0 | +10.0 |\n| intermediate | 1 | 45.0 | +15.0 | 40.0 | +10.0 |\n| medium | 1 | 70.0 | +10.0 | 70.0 | +10.0 |\n| advanced | 1 | 90.0 | +25.0 | 75.0 | +10.0 |\n| hard | 1 | 80.0 | +40.0 | 50.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **a** | 6 | 67% |\n| **b** | 2 | 22% |\n| tie | 1 | 11% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the non-baseline treatment arm?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **a** | **+17.1** | 6 (86%) | 1 (14%) |\n| **b** | **+10.0** | 9 (100%) | 0 (0%) |\n\n## Model Ranking\n\n### By absolute quality (Context arm)\n| Rank | Run | Context arm avg |\n|---:|---|---:|\n| 1 | **a** | 70.0 |\n| 2 | **b** | 62.2 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **a** | +17.1 |\n| 2 | **b** | +10.0 |\n", + "unscored_and_arms": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **u** | m (p) | same (p) | `docs`, `skill:x` |\n| **v** | m (p) | same (p) | `docs` |\n\nCommon questions evaluated: **8**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Resource Usage (baseline arm)\n| Run | Total Cost | Prompt tok | Completion tok | Total tok | Avg latency/Q | Cache hit |\n|---|---:|---:|---:|---:|---:|---:|\n| **u** | $2.0000 (4/5) | — | — | — | — | — |\n| **v** | $3.0000 | — | — | — | — | — |\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | Context arm | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **u** | 8 | 7 (1 unscored) | 64.3 | 52.9 | **+11.4** | 7 | 0 |\n| **v** | 8 | 7 (1 unscored) | 64.3 | 52.9 | **+11.4** | 7 | 0 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 2 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **u** | +11.4 | 0.0016 | 0.0032 | 0.0156 | +2.1 | large | OK Yes |\n| **v** | +11.4 | 0.0016 | 0.0032 | 0.0156 | +2.1 | large | OK Yes |\n\n## By Difficulty (common questions only)\n| Difficulty | N | u context | u Δ | v context | v Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 2 | 70.0 | +10.0 | 70.0 | +10.0 |\n| beginner | 1 | 70.0 | +15.0 | 70.0 | +15.0 |\n| intermediate | 1 | 50.0 | +20.0 | 50.0 | +20.0 |\n| medium | 1 | 75.0 | +15.0 | 75.0 | +15.0 |\n| advanced | 1 | 70.0 | +5.0 | 70.0 | +5.0 |\n| hard | 1 | 45.0 | +5.0 | 45.0 | +5.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **u** | 0 | 0% |\n| **v** | 0 | 0% |\n| tie | 7 | 88% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the non-baseline treatment arm?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **u** | **+11.4** | 7 (100%) | 0 (0%) |\n| **v** | **+11.4** | 7 (100%) | 0 (0%) |\n\n## Model Ranking\n\n### By absolute quality (Context arm)\n| Rank | Run | Context arm avg |\n|---:|---|---:|\n| 1 | **u** | 64.3 |\n| 2 | **v** | 64.3 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **u** | +11.4 |\n| 2 | **v** | +11.4 |\n", "verdict_holm_no": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **h1** | h1 (p) | same (p) | `with-context` |\n| **h2** | h2 (p) | same (p) | `with-context` |\n| **h3** | h3 (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **8**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **h1** | 8 | 8 | 55.9 | 50.0 | **+5.9** | 6 | 1 |\n| **h2** | 8 | 8 | 55.9 | 50.0 | **+5.9** | 6 | 1 |\n| **h3** | 8 | 8 | 55.9 | 50.0 | **+5.9** | 6 | 1 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 3 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **h1** | +5.9 | 0.024 | 0.0721 | 0.0469 | +1.0 | large | No |\n| **h2** | +5.9 | 0.024 | 0.0721 | 0.0469 | +1.0 | large | No |\n| **h3** | +5.9 | 0.024 | 0.0721 | 0.0469 | +1.0 | large | No |\n\n## By Difficulty (common questions only)\n| Difficulty | N | h1 context | h1 Δ | h2 context | h2 Δ | h3 context | h3 Δ |\n|---|---:| :---: | :---: | :---: | :---: | :---: | :---: |\n| easy | 2 | 52.5 | +2.5 | 52.5 | +2.5 | 52.5 | +2.5 |\n| beginner | 1 | 59.0 | +9.0 | 59.0 | +9.0 | 59.0 | +9.0 |\n| intermediate | 1 | 61.0 | +11.0 | 61.0 | +11.0 | 61.0 | +11.0 |\n| medium | 1 | 50.0 | 0.0 | 50.0 | 0.0 | 50.0 | 0.0 |\n| advanced | 1 | 52.0 | +2.0 | 52.0 | +2.0 | 52.0 | +2.0 |\n| hard | 1 | 64.0 | +14.0 | 64.0 | +14.0 | 64.0 | +14.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **h1** | 0 | 0% |\n| **h2** | 0 | 0% |\n| **h3** | 0 | 0% |\n| tie | 8 | 100% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **h1** | **+5.9** | 6 (75%) | 1 (12%) |\n| **h2** | **+5.9** | 6 (75%) | 1 (12%) |\n| **h3** | **+5.9** | 6 (75%) | 1 (12%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **h1** | 55.9 |\n| 2 | **h2** | 55.9 |\n| 3 | **h3** | 55.9 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **h1** | +5.9 |\n| 2 | **h2** | +5.9 |\n| 3 | **h3** | +5.9 |\n", "verdict_wilcoxon_no": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **w** | m (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **6**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **w** | 6 | 6 | 55.2 | 50.0 | **+5.2** | 5 | 1 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **w** | +5.2 | 0.0384 | 0.0384 | 0.0625 | +1.1 | large | No |\n\n## By Difficulty (common questions only)\n| Difficulty | N | w context | w Δ |\n|---|---:| :---: | :---: |\n| easy | 2 | 52.5 | +2.5 |\n| beginner | 1 | 55.0 | +5.0 |\n| medium | 1 | 57.0 | +7.0 |\n| advanced | 1 | 54.0 | +4.0 |\n| hard | 1 | 60.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **w** | 6 | 100% |\n| tie | 0 | 0% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **w** | **+5.2** | 5 (83%) | 1 (17%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **w** | 55.2 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **w** | +5.2 |\n", "zero_scores": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **z** | m (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **8**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **z** | 8 | 8 | 33.1 | 24.4 | **+8.8** | 6 | 1 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **z** | +8.8 | 0.247 | 0.247 | 0.1719 | +0.4 | small | No |\n\n## By Difficulty (common questions only)\n| Difficulty | N | z context | z Δ |\n|---|---:| :---: | :---: |\n| easy | 2 | 47.5 | +27.5 |\n| beginner | 1 | 0.0 | 0.0 |\n| intermediate | 1 | 50.0 | +5.0 |\n| medium | 1 | 0.0 | -30.0 |\n| advanced | 1 | 35.0 | +15.0 |\n| hard | 1 | 60.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **z** | 8 | 100% |\n| tie | 0 | 0% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **z** | **+8.8** | 6 (75%) | 1 (12%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **z** | 33.1 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **z** | +8.8 |\n", - "zero_values": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **z** | zero (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **6**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Resource Usage (baseline arm)\n| Run | Total Cost | Prompt tok | Completion tok | Total tok | Avg latency/Q | Cache hit |\n|---|---:|---:|---:|---:|---:|---:|\n| **z** | $0.0000 (0/5) | 0 | 0 | 0 | 0.00s | 0.0% |\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **z** | 6 | 5 (1 unscored) | 45.0 | 45.8 | **-10.0** | 0 | 5 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 0 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n\n## By Difficulty (common questions only)\n| Difficulty | N | z context | z Δ |\n|---|---:| :---: | :---: |\n| easy | 2 | 50.0 | -10.0 |\n| beginner | 1 | 45.0 | -10.0 |\n| medium | 1 | 50.0 | -10.0 |\n| advanced | 1 | — | 0.0 (baseline) |\n| hard | 1 | 30.0 | -10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **z** | 5 | 83% |\n| tie | 0 | 0% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **z** | **-10.0** | 0 (0%) | 5 (100%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **z** | 45.0 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **z** | -10.0 |\n" + "zero_values": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **z** | zero (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **6**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Resource Usage (baseline arm)\n| Run | Total Cost | Prompt tok | Completion tok | Total tok | Avg latency/Q | Cache hit |\n|---|---:|---:|---:|---:|---:|---:|\n| **z** | $0.0000 (0/5) | 0 | 0 | 0 | 0.00s | 0.0% |\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **z** | 6 | 5 (1 unscored) | 45.0 | 55.0 | **-10.0** | 0 | 5 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 0 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n\n## By Difficulty (common questions only)\n| Difficulty | N | z context | z Δ |\n|---|---:| :---: | :---: |\n| easy | 2 | 50.0 | -10.0 |\n| beginner | 1 | 45.0 | -10.0 |\n| medium | 1 | 50.0 | -10.0 |\n| advanced | 1 | — | 0.0 (baseline) |\n| hard | 1 | 30.0 | -10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **z** | 5 | 83% |\n| tie | 0 | 0% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **z** | **-10.0** | 0 (0%) | 5 (100%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **z** | 45.0 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **z** | -10.0 |\n" } diff --git a/dokimi/tests/test_dashboard.py b/dokimi/tests/test_dashboard.py index 5ddcbcb33..1aa3982f8 100644 --- a/dokimi/tests/test_dashboard.py +++ b/dokimi/tests/test_dashboard.py @@ -13,6 +13,8 @@ ResultsAggregator, ) from dokimi.dashboard.markdown_renderer import render_dashboard, save_dashboard_markdown, save_dashboard_json +from dokimi.eval.stats import paired_summary +from dokimi.records import records # ── Fixtures ────────────────────────────────────────────────────────────────── @@ -65,6 +67,46 @@ def test_aggregator_finds_eval_files(tmp_path): assert p.avg_delta == 20.0 +# Two pairs, a question scored in the context arm only, one in the baseline arm only. +_ONE_ARM_EVALS = [ + _ev("q1", "Q1?", 80, 60), _ev("q2", "Q2?", 40, 40), + _ev("q3", "Q3?", 100, None), _ev("q4", "Q4?", None, 30), +] + + +def test_aggregator_averages_both_arms_over_the_complete_pairs(tmp_path): + """The snapshot's means are `report generate`'s, which leaves one-arm questions out.""" + _make_eval_file(tmp_path, "oneTBB", _ONE_ARM_EVALS) + p = ResultsAggregator(tmp_path).aggregate().products[0] + s = paired_summary(records(_ONE_ARM_EVALS, kind="evaluations")) + assert (p.avg_with_docs, p.avg_without_docs, p.avg_delta) == (60.0, 50.0, 10.0) + assert (p.avg_with_docs, p.avg_without_docs, p.avg_delta) == ( + round(s.with_mean, 1), round(s.without_mean, 1), round(s.delta_mean, 1)) + assert p.total_questions == 4 + + +def test_by_source_averages_over_the_pairs_and_a_group_without_one_has_no_means(tmp_path): + """Each source row is over its own pairs; a group scored in one arm per question has none.""" + chunk = [{**_ev("c1", "C1?", 90, None), "question_source": "chunk"}, + {**_ev("c2", "C2?", None, 20), "question_source": "chunk"}] + _make_eval_file(tmp_path, "oneTBB", _ONE_ARM_EVALS + chunk) + p = ResultsAggregator(tmp_path).aggregate().products[0] + assert p.by_source["persona"] == {"count": 4, "grounded": 0, "avg_with_docs": 60.0, + "avg_without_docs": 50.0, "avg_delta": 10.0} + assert p.by_source["chunk"] == {"count": 2, "grounded": 0, "avg_with_docs": None, + "avg_without_docs": None, "avg_delta": None} + assert (p.avg_with_docs, p.avg_without_docs, p.avg_delta) == (60.0, 50.0, 10.0) + + +def test_baseline_only_run_keeps_its_baseline_mean(tmp_path): + """No pair because no context arm ran: the baseline mean is still the score shown.""" + _make_eval_file(tmp_path, "oneTBB", [_ev("q1", "Q1?", None, 60), _ev("q2", "Q2?", None, 40)]) + p = ResultsAggregator(tmp_path).aggregate().products[0] + assert (p.avg_with_docs, p.avg_without_docs, p.avg_delta) == (None, 50.0, None) + assert p.doc_score == 50.0 + assert p.status == "fair" + + def test_aggregator_multiple_products(tmp_path): _make_eval_file(tmp_path, "oneTBB", [_ev("q1", "Q1?", 80, 60)]) _make_eval_file(tmp_path, "oneMKL", [_ev("q1", "Q1?", 65, 45)]) diff --git a/dokimi/tests/test_model_compare.py b/dokimi/tests/test_model_compare.py index 9a776a424..3212f845e 100644 --- a/dokimi/tests/test_model_compare.py +++ b/dokimi/tests/test_model_compare.py @@ -510,6 +510,49 @@ def test_delta_avg_is_mean_of_paired_deltas(): assert "1 (1 unscored)" in lines +def _summary_row(run: dict) -> str: + lines = generate_section_report([("a", run)], ["a"], "t") + start = lines.index("## Overall Summary") + return next(ln for ln in lines[start:] if ln.startswith("| **a** |")) + + +def test_run_summary_averages_both_arms_over_the_pairs(): + """The arm means are `report generate`'s on the same data: one-arm questions left out.""" + # Q3 is scored in the context arm only, Q4 in the baseline only. Each arm over its + # own scores gives 73.3 and 43.3; over the pairs Q1 and Q2, 60.0 and 50.0. + run = _eval_run([_scored("Q1", 60, 80), _scored("Q2", 40, 40), + _scored("Q3", None, 100), _scored("Q4", 30, None)]) + assert _summary_row(run) == "| **a** | 4 | 2 (2 unscored) | 60.0 | 50.0 | **+10.0** | 1 | 0 |" + + +def test_run_summary_gives_no_means_for_two_arms_without_a_pair(): + """Both arms scored, never on one question: two means over different questions are not shown.""" + run = _eval_run([_scored("Q1", None, 80), _scored("Q2", 60, None)]) + assert _summary_row(run) == "| **a** | 2 | 0 (2 unscored) | — | — | — | — | — |" + + +def _difficulty_row(run: dict, difficulty: str) -> str: + lines = generate_section_report([("a", run)], ["a"], "t") + return next(ln for ln in lines if ln.startswith(f"| {difficulty} |")) + + +def test_difficulty_row_averages_the_context_arm_over_the_pairs(): + """The context mean of a difficulty row covers the questions its delta does.""" + # Q3 is scored in the context arm only: over every context score the mean is 73.3. + run = _eval_run([_scored("Q1", 60, 80), _scored("Q2", 40, 40), _scored("Q3", None, 100)]) + assert _difficulty_row(run, "easy") == "| easy | 3 | 60.0 | +10.0 |" + + +def test_difficulty_row_without_a_pair_shows_no_context_mean(): + """Both arms scored in a bucket, never on one question: no mean; context arm only: its mean.""" + run = _eval_run([_scored("Q1", 60, 80), + {**_scored("Q2", None, 80), "difficulty": "hard"}, + {**_scored("Q3", 60, None), "difficulty": "hard"}, + {**_scored("Q4", None, 70), "difficulty": "medium"}]) + assert _difficulty_row(run, "hard") == "| hard | 2 | — | — |" + assert _difficulty_row(run, "medium") == "| medium | 1 | 70.0 | — |" + + def test_extract_scores_refuses_a_record_without_question_id(): """It used to drop the record; a run with an unidentifiable question is malformed.""" run = _eval_run([ diff --git a/dokimi/tests/test_records_guard.py b/dokimi/tests/test_records_guard.py index 03c8efff8..ddb9205d0 100644 --- a/dokimi/tests/test_records_guard.py +++ b/dokimi/tests/test_records_guard.py @@ -40,7 +40,7 @@ "every judge's call"), "dokimi.report.generator": (6, "reads the statistics dict it computed from the evaluations, not a record"), "dokimi.report.model_compare": ( - 17, "reads the per-question score rows extract_scores built, which use the binary score names"), + 16, "reads the per-question score rows extract_scores built, which use the binary score names"), }