fix(dashboard, report): average both arms over the complete pairs

dokimi/changelog.d/20261026_130000_napetrov_report-eval-generate-agree.md

@@ -11,10 +11,7 @@
   says how many questions are unpaired (`paired_questions` and `unpaired_questions` in the JSON,
   and a note plus the Count cell in the Markdown). To see the new numbers, run `report generate`
   on an existing eval file again.
-  The top and bottom performer tables of `report generate` rank the complete pairs too. The
-  dashboard and the `report model-compare` run summary still average each arm over its own
-  scored questions, so for a file with an unpaired question their context and baseline means
-  differ from the ones `report generate` prints.
+  The top and bottom performer tables of `report generate` rank the complete pairs too.
 
 ### Fixed
 

dokimi/changelog.d/20261029_120000_antonwolfy_paired-arm-means.md

@@ -0,0 +1,14 @@
+### Fixed
+
+- **The dashboard and `report model-compare` average each arm over the complete pairs, as
+  `report generate` does** — the dashboard's per-library and per-source context and baseline
+  means, the context and baseline columns of the `report model-compare` Overall Summary and
+  its ranking, and the context column of its per-difficulty rows, used to average each arm over
+  every question scored in it, so the means could cover different questions than the delta
+  beside them. They now
+  cover only the questions scored in both arms, and the dashboard's delta is the difference of
+  those means rather than the mean of the stored per-question `delta`. For a run with a question
+  scored in one arm only, the arm means change, and on the dashboard the status band that
+  follows the context mean can change too. A run that scored one arm only still shows that
+  arm's mean; a run, source group or difficulty row with both arms scored but never on the same
+  question shows no means.

dokimi/docs/benchmarking-and-comparison.md

@@ -568,7 +568,9 @@ The intended input is one judged arms JSON file per model for regular questions
 and/or golden questions. The report extracts the `baseline_arm` score as the
 baseline and the treatment arm as the context arm, then computes:
 
-- overall context-arm, baseline, and delta per run,
+- overall context-arm, baseline, and delta per run, all three over the questions
+  scored in both arms (a run that scored one arm only shows that arm's mean); the
+  context mean and delta of each difficulty row follow the same rule,
 - statistical significance of context-arm minus baseline (paired t-test,
   Wilcoxon signed-rank, Cohen's d_z), corrected across the compared runs \u2014
   every run tests the same "did the treatment help?" question, so they form one

dokimi/docs/quickstart.md

@@ -173,7 +173,9 @@ python cli.py dashboard generate
 ```
 
 Aggregates per-library results from `eval/*.json` into `DASHBOARD.md` /
-`dashboard.json` for cross-library comparison.
+`dashboard.json` for cross-library comparison. Like `report generate`, it averages both
+arms and the delta, per library and per question source, over the questions scored in
+both arms; a run that scored one arm only shows that arm's mean.
 
 ## Full run (single command)
 

dokimi/dokimi/dashboard/aggregator.py

@@ -149,9 +149,6 @@ def _as_float(value: Any) -> Optional[float]:
                 return None
 
         questions = []
-        with_scores = []
-        without_scores = []
-        deltas = []
 
         for ev in recs:
             q_id = ev.question_id
@@ -165,13 +162,6 @@ def _as_float(value: Any) -> Optional[float]:
             without_score = _as_float(ev.score(BASELINE))
             delta = _as_float(ev.delta)
 
-            if with_score is not None:
-                with_scores.append(with_score)
-            if without_score is not None:
-                without_scores.append(without_score)
-            if delta is not None:
-                deltas.append(delta)
-
             q_source = ev.raw.get("question_source", "persona")
             grounded = bool((with_eval or {}).get("grounded") or (without_eval or {}).get("grounded"))
 
@@ -190,16 +180,17 @@ def _as_float(value: Any) -> Optional[float]:
             ))
 
         by_source = _compute_by_source(questions)
+        means = _arm_means(questions)
 
         return ProductSnapshot(
             product=product,
             library_key=library_key,
             evaluated_at=data.get("evaluated_at", ""),
             judge_model=data.get("judge_model", "unknown"),
             total_questions=len(questions),
-            avg_with_docs=round(sum(with_scores) / len(with_scores), 1) if with_scores else None,
-            avg_without_docs=round(sum(without_scores) / len(without_scores), 1) if without_scores else None,
-            avg_delta=round(sum(deltas) / len(deltas), 1) if deltas else None,
+            avg_with_docs=means["avg_with_docs"],
+            avg_without_docs=means["avg_without_docs"],
+            avg_delta=means["avg_delta"],
             questions=sorted(questions, key=lambda q: (q.with_docs_score or 0)),  # worst first
             source_file=str(path),
             by_source=by_source,
@@ -215,15 +206,35 @@ def _compute_by_source(questions: "List[QuestionResult]") -> "Dict[str, Dict]":
 
     result = {}
     for source, qs in groups.items():
-        with_scores = [q.with_docs_score for q in qs if q.with_docs_score is not None]
-        without_scores = [q.without_docs_score for q in qs if q.without_docs_score is not None]
-        deltas = [q.delta for q in qs if q.delta is not None]
         grounded_count = sum(1 for q in qs if q.grounded)
         result[source] = {
             "count": len(qs),
             "grounded": grounded_count,
-            "avg_with_docs": round(sum(with_scores) / len(with_scores), 1) if with_scores else None,
-            "avg_without_docs": round(sum(without_scores) / len(without_scores), 1) if without_scores else None,
-            "avg_delta": round(sum(deltas) / len(deltas), 1) if deltas else None,
+            **_arm_means(qs),
         }
     return result
+
+
+def _arm_means(questions: "List[QuestionResult]") -> "Dict[str, Optional[float]]":
+    """``avg_with_docs``, ``avg_without_docs`` and ``avg_delta`` over the complete pairs.
+
+    The questions scored in both arms, the ones :func:`dokimi.eval.stats.paired_summary`
+    averages for ``report generate``, so the delta is the difference of the two means.
+    With no pair there is no delta: an arm keeps its mean over its own scores when the
+    other arm scored nothing (a baseline-only run), and otherwise neither mean is given,
+    so two means over different questions never stand side by side.
+    """
+    with_s = [q.with_docs_score for q in questions if q.with_docs_score is not None]
+    without_s = [q.without_docs_score for q in questions if q.without_docs_score is not None]
+    pairs = [(q.with_docs_score, q.without_docs_score) for q in questions
+             if q.with_docs_score is not None and q.without_docs_score is not None]
+    if pairs:
+        with_s, without_s = [w for w, _ in pairs], [wo for _, wo in pairs]
+    elif with_s and without_s:
+        with_s = without_s = []
+
+    def _mean(vals: "List[float]") -> Optional[float]:
+        return round(sum(vals) / len(vals), 1) if vals else None
+
+    return {"avg_with_docs": _mean(with_s), "avg_without_docs": _mean(without_s),
+            "avg_delta": _mean([w - wo for w, wo in pairs])}

dokimi/dokimi/eval/stats.py

@@ -21,8 +21,8 @@
     Per-question paired score vectors for two arms, complete cases only.
 paired_summary(evaluations)
     Both arms and their delta over the complete pairs, with the unpaired count.
-    Called by the eval report, the report generator and the pipeline summary. The
-    dashboard and the model-compare summary still average each arm separately.
+    Called by the eval report, the report generator and the pipeline summary; the
+    dashboard and the model-compare run summary average both arms over the same pairs.
 mean_ci(values, ...)
     Student-t CI for the mean, coherent with ``p_ttest``; bootstrap fallback.
 point_biserial(flags, totals)

dokimi/dokimi/report/model_compare.py

@@ -932,13 +932,13 @@ def _resource_usage_lines(runs: list[tuple[str, dict]], pooled: bool) -> list[st
 
 def _run_summary(scores: list[dict]) -> dict:
     """One run's arm means and paired-delta counts over the common questions."""
-    with_docs_scores = [s["with_docs"] for s in scores if s["with_docs"] is not None]
-    without_docs_scores = [s["without_docs"] for s in scores if s["without_docs"] is not None]
-    # Average the per-question deltas rather than differencing the two arm
-    # means. The arm means are filtered independently, so when any question
-    # is missing a score they cover different question subsets and their
-    # difference is not a paired delta.
-    paired_deltas = [s["delta"] for s in scores if s.get("delta") is not None]
+    # Arm means and delta over the pairs (`eval.stats.paired_summary`); one-arm runs keep theirs.
+    pairs = [s for s in scores if s.get("delta") is not None]
+    with_docs_scores = [s["with_docs"] for s in pairs or scores if s["with_docs"] is not None]
+    without_docs_scores = [s["without_docs"] for s in pairs or scores if s["without_docs"] is not None]
+    if not pairs and with_docs_scores and without_docs_scores:  # both arms, never on one question
+        with_docs_scores = without_docs_scores = []
+    paired_deltas = [s["delta"] for s in pairs]
     # Per-run, distinct from the section-level has_treatment: a single run may
     # lack a treatment arm while others in the comparison have one.
     paired = bool(with_docs_scores) and bool(paired_deltas)
@@ -1064,14 +1064,12 @@ def _difficulty_cells(sc: list[dict], has_treatment: bool) -> str:
     """One run's cells of a difficulty row: context mean and delta, or the baseline."""
     if not sc:
         return " — | — |" if has_treatment else " — |"
-    with_scores = [s["with_docs"] for s in sc if s["with_docs"] is not None]
-    if with_scores:
-        # Same pairing rule as the overall summary: average the per-question
-        # deltas, never the difference of two means filtered over different
-        # question subsets.
-        diff_deltas = [s["delta"] for s in sc if s.get("delta") is not None]
-        delta_cell = _fmt_delta(round(sum(diff_deltas) / len(diff_deltas), 1)) if diff_deltas else "—"
-        return f" {_avg(with_scores)} | {delta_cell} |"
+    if any(s["with_docs"] is not None for s in sc):
+        # Same pairs and one-arm rule as the overall summary: the context mean
+        # covers the questions the delta does.
+        st = _run_summary(sc)
+        delta_cell = "—" if st["delta_avg"] is None else _fmt_delta(st["delta_avg"])
+        return f" {_num_or_dash(st['with_avg'])} | {delta_cell} |"
     # No treatment score in this bucket: report the baseline.
     without_scores = [s["without_docs"] for s in sc if s["without_docs"] is not None]
     wo = _avg(without_scores) if without_scores else None

dokimi/scripts/growth_baseline.json

@@ -4,7 +4,7 @@
     "dokimi/harnesses/benchflow.py": {"lines": 869},
     "dokimi/harnesses/docker_solver.py": {"lines": 899, "rationale": "pre-existing when the gate was introduced"},
     "dokimi/llm.py": {"lines": 1118, "rationale": "grew on main in #180 (replay a stored retry) after #93 seeded this baseline; split it in its own PR"},
-    "dokimi/report/model_compare.py": {"lines": 1209, "rationale": "grew on main in #73 (judge-validity --context stored) before the ratchet landed; split it in its own PR"},
+    "dokimi/report/model_compare.py": {"lines": 1207, "rationale": "grew on main in #73 (judge-validity --context stored) before the ratchet landed; split it in its own PR"},
     "tests/test_arm_runner.py": {"lines": 1176, "rationale": "pre-existing when the gate was introduced"},
     "tests/test_benchflow_harness.py": {"lines": 1716, "rationale": "pre-existing when the gate was introduced"},
     "tests/test_container_agent.py": {"lines": 1251, "rationale": "pre-existing when the gate was introduced"},
@@ -30,7 +30,6 @@
     "dokimi/commands/report.py::cmd_report_model_compare": {"complexity": 12},
     "dokimi/commands/tools.py::cmd_tools_preference": {"complexity": 11},
     "dokimi/context/trace.py::_apply": {"complexity": 18},
-    "dokimi/dashboard/aggregator.py::ResultsAggregator._load_snapshot": {"complexity": 13},
     "dokimi/eval/agent_runner.py::_summarize_tool_usage": {"complexity": 11},
     "dokimi/eval/agent_runner.py::run_agent_loop": {"complexity": 14},
     "dokimi/eval/arm_runner.py::ArmRunner.judge": {"complexity": 11},