From 6b52c66f75495eee71ed64da5f5ff52005715d16 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 28 Oct 2026 18:25:47 -0700 Subject: [PATCH 01/24] fix(eval): an errored answer is counted as an error, not as unscored An answer whose call failed was tallied with the answers no check could score, so the summary's unscored count mixed provider failures with grading gaps. It is now counted as errors, per condition in eval reference and per arm in arms summaries. Errors leave n, the pass rate and the mean out, as unscored answers do; an arms cell whose answer errored or whose judge raised is no longer a silent None. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/eval/arm_runner.py | 13 ++++------ dokimi/dokimi/eval/arm_summary.py | 19 +++++++++++++- dokimi/dokimi/eval/reference_run.py | 14 +++++++--- dokimi/dokimi/schemas/arms.v1.json | 4 +++ dokimi/tests/test_arm_cell_errors.py | 37 +++++++++++++++++++++++++++ dokimi/tests/test_reference_checks.py | 14 +++++----- 6 files changed, 83 insertions(+), 18 deletions(-) create mode 100644 dokimi/tests/test_arm_cell_errors.py diff --git a/dokimi/dokimi/eval/arm_runner.py b/dokimi/dokimi/eval/arm_runner.py index 4b6045055..b1f75773c 100644 --- a/dokimi/dokimi/eval/arm_runner.py +++ b/dokimi/dokimi/eval/arm_runner.py @@ -25,7 +25,7 @@ from dokimi.eval import stats as arm_stats from dokimi.eval.interactions import interaction_significance from dokimi.eval.arm_summary import ( - cost_summary, summarize_agentic_usage, summarize_skill_selection, + cell_values, cost_summary, summarize_agentic_usage, summarize_skill_selection, ) from dokimi.eval.cells import Cell from dokimi.eval.question_text import question_text_of, require_question_texts @@ -615,7 +615,8 @@ def _score(rec: Record) -> Dict[str, Any]: for arm_name in arm_names(rec): answer = rec.answer(arm_name) if answer is None or not answer.text: - scores[arm_name] = None + # An answer that errored is an error on the cell, not a missing score. + scores[arm_name] = {"error": answer.error} if answer is not None and answer.error else None continue context, source = self.judge_context.resolve_with_source( rec.question_id, arm_name, dict(answer.raw) @@ -752,17 +753,13 @@ def build_output( def _summarize( self, evaluations: List[Dict[str, Any]], baseline_arm: str ) -> Dict[str, Any]: - per_arm: Dict[str, List[float]] = {name: [] for name in self.arm_names} - for ev in as_records(evaluations, source="evaluations", kind="evaluations"): - for arm_name in ev.scores: - if (v := ev.score(arm_name)) is not None: - per_arm.setdefault(arm_name, []).append(float(v)) + per_arm, errors = cell_values(evaluations, self.arm_names) summary: Dict[str, Any] = {"per_arm": {}} base_avg = None for arm_name, vals in per_arm.items(): avg = round(sum(vals) / len(vals), 1) if vals else None - summary["per_arm"][arm_name] = {"avg_aggregate": avg, "n": len(vals)} + summary["per_arm"][arm_name] = {"avg_aggregate": avg, "n": len(vals), "errors": errors.get(arm_name, 0)} if arm_name == baseline_arm: base_avg = avg if base_avg is not None: diff --git a/dokimi/dokimi/eval/arm_summary.py b/dokimi/dokimi/eval/arm_summary.py index cf109ae53..4a960ceb4 100644 --- a/dokimi/dokimi/eval/arm_summary.py +++ b/dokimi/dokimi/eval/arm_summary.py @@ -13,7 +13,7 @@ from __future__ import annotations -from typing import Any, Dict, Iterable, Mapping +from typing import Any, Dict, Iterable, List, Mapping, Sequence, Tuple from dokimi.records import RecordLike from dokimi.records import records as as_records @@ -264,3 +264,20 @@ def cost_summary(records: Iterable[RecordLike]) -> Dict[str, Any]: if prompt else None), } return out + + +def cell_values(evaluations: Iterable[Any], arm_names: Sequence[str]) -> Tuple[Dict[str, List[float]], Dict[str, int]]: + """Per arm, the aggregates that count, and how many cells errored instead. + + An errored cell (the answer call failed, or the judge raised) has no aggregate, so it + is left out of the mean and its ``n``; the errors are counted apart, not as scores. + """ + values: Dict[str, List[float]] = {name: [] for name in arm_names} + errors: Dict[str, int] = {name: 0 for name in arm_names} + for ev in as_records(evaluations, source="evaluations", kind="evaluations"): + for arm_name, cell in ev.scores.items(): + if (v := ev.score(arm_name)) is not None: + values.setdefault(arm_name, []).append(float(v)) + elif cell is not None and cell.error: + errors[arm_name] = errors.get(arm_name, 0) + 1 + return values, errors diff --git a/dokimi/dokimi/eval/reference_run.py b/dokimi/dokimi/eval/reference_run.py index 97dbc473e..ac0ab1c10 100644 --- a/dokimi/dokimi/eval/reference_run.py +++ b/dokimi/dokimi/eval/reference_run.py @@ -29,17 +29,24 @@ @dataclass class ConditionTally: + """Counts for one condition. An errored answer (the answer call failed) is ``errors``, apart from + ``unscored`` (no check could score it, e.g. no judge): neither enters ``n``, the pass rate or the mean.""" + n: int = 0 passed: int = 0 unscored: int = 0 + errors: int = 0 judge_unscored: int = 0 check_unscored: int = 0 score_sum: float = 0.0 - def add(self, g: Grade) -> None: + def add(self, g: Grade, *, errored: bool = False) -> None: score = g.score if score is None: - self.unscored += 1 + if errored: + self.errors += 1 + else: + self.unscored += 1 return self.n += 1 self.score_sum += score @@ -52,6 +59,7 @@ def as_dict(self) -> dict[str, Any]: return { "n": self.n, "unscored": self.unscored, + "errors": self.errors, "judge_unscored": self.judge_unscored, "check_unscored": self.check_unscored, "pass_rate": round(self.passed / self.n, 4) if self.n else None, @@ -107,7 +115,7 @@ def score(data: Any, questions: list[Mapping[str, Any]], *, ask: Optional[Ask] = for rec, per_condition in zip(graded, grades, strict=True): blocks = {} for name, g in per_condition.items(): - out.tally.setdefault(name, ConditionTally()).add(g) + out.tally.setdefault(name, ConditionTally()).add(g, errored=bool(rec.answers[name].error)) blocks[name] = {**g.as_dict(), "scorer": SCORER} out.evaluations.append(_evaluation(run, rec, blocks)) return out diff --git a/dokimi/dokimi/schemas/arms.v1.json b/dokimi/dokimi/schemas/arms.v1.json index c0555bb1f..4e7c1f176 100644 --- a/dokimi/dokimi/schemas/arms.v1.json +++ b/dokimi/dokimi/schemas/arms.v1.json @@ -104,6 +104,10 @@ "n": { "type": "integer" }, + "errors": { + "type": "integer", + "description": "Cells that errored (the answer call failed or the judge raised). Left out of n and avg_aggregate." + }, "delta_vs_baseline": { "type": [ "number", diff --git a/dokimi/tests/test_arm_cell_errors.py b/dokimi/tests/test_arm_cell_errors.py new file mode 100644 index 000000000..fd604513f --- /dev/null +++ b/dokimi/tests/test_arm_cell_errors.py @@ -0,0 +1,37 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""An arms cell whose answer call failed is an error, apart from the scores that count.""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +from dokimi.eval.arm_runner import ArmRunner +from dokimi.eval.arm_summary import cell_values + + +def test_arm_runner_judge_turns_an_errored_answer_into_an_error_cell(): + runner = ArmRunner.__new__(ArmRunner) + runner.judge_context = MagicMock() + runner.judge_context.resolve_with_source.return_value = ("", "snippets") + judge = MagicMock() + judge.score_answer.return_value = {"aggregate": 60.0} + rec = {"question_id": "q1", "question_text": "How?", + "arms": {"baseline": {"answer": "base"}, + "skill": {"answer": None, "error": "generation_failed: HTTP 500"}}} + [row] = runner.judge(judge, "lib", [rec], baseline_arm="baseline") + assert row["scores"]["skill"] == {"error": "generation_failed: HTTP 500"} + assert judge.score_answer.call_count == 1 + + +def test_an_errored_cell_is_counted_apart_and_left_out_of_the_mean(): + evaluations = [ + {"question_id": "1", "question_text": "Q1", + "scores": {"baseline": {"aggregate": 80}, "skill": {"error": "judge: timeout"}}}, + {"question_id": "2", "question_text": "Q2", + "scores": {"baseline": {"aggregate": 60}, "skill": {"aggregate": 70}}}, + ] + values, errors = cell_values(evaluations, ["baseline", "skill"]) + assert values == {"baseline": [80.0, 60.0], "skill": [70.0]} + assert errors == {"baseline": 0, "skill": 1} diff --git a/dokimi/tests/test_reference_checks.py b/dokimi/tests/test_reference_checks.py index 6c5200fa4..d737b97d8 100644 --- a/dokimi/tests/test_reference_checks.py +++ b/dokimi/tests/test_reference_checks.py @@ -156,7 +156,7 @@ def test_scored_grade_with_an_unjudged_check_is_counted_in_the_summary(): tally = ConditionTally() tally.add(grade({"checks": [{"type": "match", "values": ["1996"], "location": "any"}, {"type": "judge", "reference": "1996"}]}, "Q?", "In 1996.")) - assert tally.as_dict() == {"n": 1, "unscored": 0, "judge_unscored": 1, "check_unscored": 0, + assert tally.as_dict() == {"n": 1, "unscored": 0, "errors": 0, "judge_unscored": 1, "check_unscored": 0, "pass_rate": 1.0, "mean_score": 1.0} @@ -165,7 +165,7 @@ def test_a_timed_out_check_next_to_a_pass_is_counted_in_the_summary(): tally = ConditionTally() tally.add(grade({"checks": [{"type": "regex", "values": ["(a|aa)+$"]}, {"type": "match", "values": ["!"], "location": "any"}]}, "Q?", "a" * 40 + "!")) - assert tally.as_dict() == {"n": 1, "unscored": 0, "judge_unscored": 0, "check_unscored": 1, + assert tally.as_dict() == {"n": 1, "unscored": 0, "errors": 0, "judge_unscored": 0, "check_unscored": 1, "pass_rate": 1.0, "mean_score": 1.0} @@ -293,8 +293,9 @@ def test_score_binary_answers_gives_eval_v1(tmp_path): ]} result = score(data, _QUESTIONS) assert result.missing == ["3"] - assert result.tally["without_docs"].as_dict() == {"n": 2, "unscored": 0, "judge_unscored": 0, - "check_unscored": 0, "pass_rate": 0.5, "mean_score": 0.5} + assert result.tally["without_docs"].as_dict() == {"n": 2, "unscored": 0, "errors": 0, + "judge_unscored": 0, "check_unscored": 0, + "pass_rate": 0.5, "mean_score": 0.5} kind, payload = artifact(result, data, judge_model=None, judge_provider=None, questions_path="q.json") assert kind == "eval" first = payload["evaluations"][0] @@ -365,7 +366,7 @@ def test_read_inspect_json_log(tmp_path): save_artifact("eval", run.eval_artifact(), tmp_path / "e.json") -def test_errored_inspect_sample_stays_unscored_in_eval_reference(tmp_path): +def test_errored_inspect_sample_is_an_error_not_a_score_in_eval_reference(tmp_path): log = {"eval": {"model": "m"}, "samples": [ {"id": "q1", "input": "Q1", "output": {"completion": "Paris"}, "scores": {"match": {"value": "C"}}}, {"id": "q2", "input": "Q2", "output": {"completion": ""}, "scores": {}, @@ -378,7 +379,8 @@ def test_errored_inspect_sample_stays_unscored_in_eval_reference(tmp_path): checks = {"checks": [{"type": "match", "values": ["paris"], "location": "any"}]} questions = [{"id": q, "question": q, "grading": checks} for q in ("q1", "q2")] tally = score(run.answers_artifact(), questions).tally["without_docs"].as_dict() - assert (tally["n"], tally["unscored"], tally["pass_rate"]) == (1, 1, 1.0) + # The errored sample is counted as an error, apart from the unscored ones, and not in n or the rate. + assert (tally["n"], tally["unscored"], tally["errors"], tally["pass_rate"]) == (1, 0, 1, 1.0) def test_read_promptfoo_output_keys_by_test_and_prompt(tmp_path): From 89d19c96ac72a1260c66a161f4de3be717fca03c Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 28 Oct 2026 18:25:47 -0700 Subject: [PATCH 02/24] fix(cache): an empty reply is never stored or replayed A stored empty answer was replayed on every later run as a failed answer, so a one-off empty reply stuck. keeps() now refuses empty and whitespace-only text on store and on replay; a rerun asks the model again. A content-less reply (None, stored as null) is kept as before. Docs and changelog cover both commits of this branch. Note for replay-only runs: a cache that holds an empty reply now raises CacheMiss for that call instead of replaying the empty answer. Signed-off-by: Nikolay Petrov --- ...261028_120200_napetrov_feat-run-hygiene.md | 4 ++++ dokimi/docs/answer-formats.md | 6 ++++- dokimi/dokimi/caching.py | 6 +++++ dokimi/tests/test_caching.py | 24 +++++++++++++++++++ 4 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 dokimi/changelog.d/20261028_120200_napetrov_feat-run-hygiene.md diff --git a/dokimi/changelog.d/20261028_120200_napetrov_feat-run-hygiene.md b/dokimi/changelog.d/20261028_120200_napetrov_feat-run-hygiene.md new file mode 100644 index 000000000..330bc5d48 --- /dev/null +++ b/dokimi/changelog.d/20261028_120200_napetrov_feat-run-hygiene.md @@ -0,0 +1,4 @@ +### Fixed + +- **Errored answers are counted as errors, not as unscored.** An answer whose call failed is tallied as `errors` in `eval reference` summaries, and as `errors` per arm in arms summaries. Errors leave `n`, the pass rate and the mean out. Before, they were lumped with answers that no check could score. +- **Empty replies are not cached.** An empty or whitespace-only model reply is neither stored nor replayed from the response cache, so a later run asks the model again instead of replaying an empty answer as a failure. diff --git a/dokimi/docs/answer-formats.md b/dokimi/docs/answer-formats.md index 80482f95c..7386d0257 100644 --- a/dokimi/docs/answer-formats.md +++ b/dokimi/docs/answer-formats.md @@ -78,7 +78,11 @@ and `judge_provider` set to `reference`. The summary's `judge_unscored` counts t counted answers whose pass leaves out a judge check that had no verdict. The summary's `check_unscored` does the same for a deterministic check that did not run to a result (a `regex` past its limit): the pass is decided on the checks -that ran, and the count shows how many passes left one out. +that ran, and the count shows how many passes left one out. An answer whose +call failed is counted as `errors`, not `unscored`, and leaves the pass rate and +the mean out just as an unscored answer does. An arms summary has the same +count per arm as `errors`. An empty reply is never cached (on store or replay), +so a rerun asks the model again instead of replaying an empty answer. `eval reference` writes eval.v1 for an `answers generate` file and an arms artifact for an `arms run` file. Each block carries `scorer: reference`, and diff --git a/dokimi/dokimi/caching.py b/dokimi/dokimi/caching.py index 06fa96138..4c96b4d9a 100644 --- a/dokimi/dokimi/caching.py +++ b/dokimi/dokimi/caching.py @@ -286,7 +286,13 @@ def keeps(cache_if: Optional[Callable[[str], bool]], text: str) -> bool: all three of the judge's attempts are stored that way, every later run replays the failure without calling the model. Checking on replay too means entries stored before the check existed stop poisoning the run. + + An empty or whitespace-only reply is never kept, on store or on replay: a stored + empty answer would be replayed as a failed answer on every later run. ``None`` + (a content-less reply, stored as null) is kept as before. """ + if text is not None and not text.strip(): + return False return cache_if is None or cache_if(text) diff --git a/dokimi/tests/test_caching.py b/dokimi/tests/test_caching.py index e2e5e3e90..5157d1f69 100644 --- a/dokimi/tests/test_caching.py +++ b/dokimi/tests/test_caching.py @@ -388,3 +388,27 @@ def test_no_temp_files_are_left_behind(monkeypatch, tmp_path): monkeypatch.setenv(caching.RESPONSE_CACHE_ENV, str(tmp_path)) caching.store_response("k", "answer", _usage()) assert [p.name for p in tmp_path.iterdir()] == ["k.json"] + + +# --------------------------------------------------------------------------- +# keeps: empty replies are never stored or replayed +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("text", ["", " ", "\n\t"]) +def test_an_empty_reply_is_not_kept_even_without_a_judge_test(text): + assert caching.keeps(None, text) is False + + +def test_an_empty_reply_is_not_kept_even_when_the_judge_test_accepts_it(): + assert caching.keeps(lambda _text: True, "") is False + + +def test_a_content_less_reply_stored_as_null_is_still_kept(): + # llm.complete stores a content-less reply as None, as it always has. + assert caching.keeps(None, None) is True + + +def test_a_reply_with_text_is_kept_only_when_the_judge_test_accepts_it(): + assert caching.keeps(None, "Paris") is True + assert caching.keeps(lambda text: text.startswith("{"), "Paris") is False From 93bc14ac8963841d94fb1df76266b7607f28c31c Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 28 Oct 2026 20:32:34 -0700 Subject: [PATCH 03/24] fix(cache): a content-less reply is never stored or replayed keeps() treated None as acceptable, so a reply with no content was stored as null and replayed as "no answer" on every later run. It is now refused on store and on replay, like an empty or whitespace-only reply, and a rerun asks the model again. The one shared predicate is caching.is_empty_reply. No caller of complete() offers tools, and chat() (the tool-calling path) is never response-cached, so no tool-call-only reply is lost. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/caching.py | 19 +++++++++++++++---- dokimi/dokimi/llm.py | 2 +- dokimi/tests/test_caching.py | 11 +++++++---- dokimi/tests/test_llm.py | 16 +++++++++++++--- 4 files changed, 36 insertions(+), 12 deletions(-) diff --git a/dokimi/dokimi/caching.py b/dokimi/dokimi/caching.py index 4c96b4d9a..55e133d85 100644 --- a/dokimi/dokimi/caching.py +++ b/dokimi/dokimi/caching.py @@ -278,6 +278,16 @@ def draw_extra(extra: Dict[str, Any], draw: int) -> Dict[str, Any]: return {**extra, "draw": draw} if draw else extra +def is_empty_reply(text: Optional[str]) -> bool: + """Whether a reply carries no answer: no content (``None``) or only whitespace. + + The one test every path shares (response cache, single-turn and session arms, + ``eval reference``, the judge). An empty reply is an error on the cell: it is + never scored, never counted as a 0, and never stored or replayed. + """ + return text is None or not text.strip() + + def keeps(cache_if: Optional[Callable[[str], bool]], text: str) -> bool: """Whether a completion may be stored, or a stored one replayed. @@ -287,11 +297,12 @@ def keeps(cache_if: Optional[Callable[[str], bool]], text: str) -> bool: replays the failure without calling the model. Checking on replay too means entries stored before the check existed stop poisoning the run. - An empty or whitespace-only reply is never kept, on store or on replay: a stored - empty answer would be replayed as a failed answer on every later run. ``None`` - (a content-less reply, stored as null) is kept as before. + An empty reply (see :func:`is_empty_reply`) is never kept, on store or on + replay: a stored empty answer would be replayed as a failed answer on every + later run, and a content-less one (stored as null) as "no answer". ``text`` is + ``None`` at runtime for a content-less reply, whatever its annotation says. """ - if text is not None and not text.strip(): + if is_empty_reply(text): return False return cache_if is None or cache_if(text) diff --git a/dokimi/dokimi/llm.py b/dokimi/dokimi/llm.py index 0967a2011..0d91a78f3 100644 --- a/dokimi/dokimi/llm.py +++ b/dokimi/dokimi/llm.py @@ -718,7 +718,7 @@ def complete( if reply.response is not None and not getattr(reply.response, "choices", None): raise IndexError("reply has no choices") # as llm_call always did; not cached _warn_if_truncated(reply, self.model, max_tokens) - # A content-less reply is stored as null, as it always was. + # A content-less reply (None) is not stored: keeps() refuses it, so it is never replayed. text = cast(str, reply.text) if caching.keeps(cache_if, text): caching.store_response(key, text, reply.usage) diff --git a/dokimi/tests/test_caching.py b/dokimi/tests/test_caching.py index 5157d1f69..fd6373e16 100644 --- a/dokimi/tests/test_caching.py +++ b/dokimi/tests/test_caching.py @@ -297,7 +297,7 @@ def test_usage_values_of_every_field_type_replay(monkeypatch, tmp_path): def test_null_text_is_replayed_not_a_miss(monkeypatch, tmp_path): - """A reply without content is stored as null text and must still replay.""" + """A well-formed null entry loads (load_response); complete() still refuses to replay it (keeps).""" monkeypatch.setenv(caching.RESPONSE_CACHE_ENV, str(tmp_path)) (tmp_path / "empty.json").write_text( json.dumps({"text": None, "usage": {}}), encoding="utf-8" @@ -404,9 +404,12 @@ def test_an_empty_reply_is_not_kept_even_when_the_judge_test_accepts_it(): assert caching.keeps(lambda _text: True, "") is False -def test_a_content_less_reply_stored_as_null_is_still_kept(): - # llm.complete stores a content-less reply as None, as it always has. - assert caching.keeps(None, None) is True +def test_a_content_less_reply_is_never_stored_or_replayed(): + # A reply with no content (None) stored as null would replay as "no answer" on every + # later run. complete() takes no tools, so a tool-call reply never reaches this test: + # chat() is the tool-calling path and is never response-cached. + assert caching.keeps(None, None) is False + assert caching.keeps(lambda _text: True, None) is False def test_a_reply_with_text_is_kept_only_when_the_judge_test_accepts_it(): diff --git a/dokimi/tests/test_llm.py b/dokimi/tests/test_llm.py index 2710b450a..9326fc875 100644 --- a/dokimi/tests/test_llm.py +++ b/dokimi/tests/test_llm.py @@ -287,6 +287,15 @@ def test_a_cached_reply_is_replayed_with_its_stored_usage(cache_dir): assert second.usage.cost_usd == 0.001 # what the real call cost, not $0 +def test_a_content_less_reply_is_not_cached_so_the_next_call_asks_again(cache_dir): + usage = UsageRecord(prompt_tokens=1, completion_tokens=0, total_tokens=1) + fake = FakeAdapter(Reply(text=None, usage=usage), Reply(text="answer", usage=usage)) + llm = LLM("m", "openai", adapter=fake) + assert llm.complete("ask").text is None + assert llm.complete("ask").text == "answer" + assert len(fake.requests) == 2 + + def test_sampling_and_max_tokens_key_their_own_entries(cache_dir, fake_llm): fake_llm.push("free", "pinned", "seeded", "capped") assert LLM("m", "openai").ask("p") == "free" @@ -316,13 +325,14 @@ def test_cache_if_guards_both_storing_and_replaying(cache_dir, fake_llm): assert len(fake_llm.requests) == 2 -def test_a_reply_without_content_is_stored_and_replayed_as_before(cache_dir, fake_llm): - fake_llm.push(Reply(text=None, usage=UsageRecord())) +def test_a_reply_without_content_is_asked_again_not_replayed(cache_dir, fake_llm): + fake_llm.push(*(Reply(text=None, usage=UsageRecord()) for _ in range(3))) llm = LLM("m", "openai") assert llm.complete("p").text is None assert llm.complete("p").text is None assert llm.ask("p") == "" - assert len(fake_llm.requests) == 1 + assert len(fake_llm.requests) == 3 + assert not list(cache_dir.glob("*.json")) def test_chat_is_never_response_cached(cache_dir, fake_llm): From f99dfb296981068d8a6d0827755338e5f9ff62f5 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 28 Oct 2026 20:33:37 -0700 Subject: [PATCH 04/24] fix(arms): an empty reply is an error cell, never judged or scored A single-turn arm whose reply is empty or whitespace (or content-less) was recorded as an answer: the judge's own empty check skipped it silently (no score, not counted) or judged a whitespace reply. It is now an error cell, empty_reply, as a session that stopped on an empty turn already was; the call's usage still counts. The judge treats an empty answer the same way, with the shared is_empty_reply predicate. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/eval/arm_runner.py | 17 +++++++++-- dokimi/tests/test_arm_cell_errors.py | 43 ++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 3 deletions(-) diff --git a/dokimi/dokimi/eval/arm_runner.py b/dokimi/dokimi/eval/arm_runner.py index b1f75773c..c234f145b 100644 --- a/dokimi/dokimi/eval/arm_runner.py +++ b/dokimi/dokimi/eval/arm_runner.py @@ -18,6 +18,7 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Sequence +from dokimi.caching import is_empty_reply from dokimi.utils import question_id # noqa: F401 (re-exported: callers import it from here) from dokimi.artifacts import save_artifact from dokimi.llm import llm_call_with_usage @@ -398,6 +399,12 @@ def _answer_one_arm( final_chars = len(final_text or "") rec = to_record(usage, self.model, self.provider) + if is_empty_reply(final_text): + # As a session that stopped on an empty reply: an error cell, never an answer. + # The call was paid for, so its usage counts; no answer-length keys (see _session_spend). + return {"error": "empty_reply", "answer": None, + "token_usage": rec.as_token_usage_dict(), "metrics": rec.as_metrics_dict(), + "metadata": cfg_metadata, "elapsed_sec": round(time.time() - t0, 2)} return { "answer": final_text, "raw_answer": answer_text if shapers else None, @@ -614,9 +621,13 @@ def _score(rec: Record) -> Dict[str, Any]: gt = rec.ground_truth for arm_name in arm_names(rec): answer = rec.answer(arm_name) - if answer is None or not answer.text: - # An answer that errored is an error on the cell, not a missing score. - scores[arm_name] = {"error": answer.error} if answer is not None and answer.error else None + if answer is None or is_empty_reply(answer.text): + # An answer that errored is an error on the cell, not a missing score. An empty + # reply is never judged: it is an error on the cell too (as a session's empty turn). + if answer is None: + scores[arm_name] = None + else: + scores[arm_name] = {"error": answer.error or "empty_reply"} continue context, source = self.judge_context.resolve_with_source( rec.question_id, arm_name, dict(answer.raw) diff --git a/dokimi/tests/test_arm_cell_errors.py b/dokimi/tests/test_arm_cell_errors.py index fd604513f..40a751d66 100644 --- a/dokimi/tests/test_arm_cell_errors.py +++ b/dokimi/tests/test_arm_cell_errors.py @@ -7,8 +7,12 @@ from unittest.mock import MagicMock +import pytest + +import dokimi.eval.arm_runner as arm_runner_mod from dokimi.eval.arm_runner import ArmRunner from dokimi.eval.arm_summary import cell_values +from dokimi.treatments import BaselineTreatment def test_arm_runner_judge_turns_an_errored_answer_into_an_error_cell(): @@ -35,3 +39,42 @@ def test_an_errored_cell_is_counted_apart_and_left_out_of_the_mean(): values, errors = cell_values(evaluations, ["baseline", "skill"]) assert values == {"baseline": [80.0, 60.0], "skill": [70.0]} assert errors == {"baseline": 0, "skill": 1} + + +@pytest.mark.parametrize("reply", ["", " \n\t", None]) +def test_a_single_turn_empty_reply_is_an_error_cell_not_an_answer(monkeypatch, reply): + usage = {"prompt_tokens": 4, "completion_tokens": 0, "total_tokens": 4} + monkeypatch.setattr(arm_runner_mod, "llm_call_with_usage", lambda *a, **k: (reply, usage)) + runner = ArmRunner([BaselineTreatment()]) + [rec] = runner.run("lib", [{"id": "q1", "question": "How?"}], concurrency=1) + cell = rec["arms"]["baseline"] + assert cell["answer"] is None + assert cell["error"] == "empty_reply" + # The call was paid for, so its usage is still recorded, as for a failed session. + assert cell["token_usage"]["total_tokens"] == 4 + + +def test_judge_does_not_score_a_whitespace_reply_and_records_it_as_an_error(): + runner = ArmRunner.__new__(ArmRunner) + runner.judge_context = MagicMock() + runner.judge_context.resolve_with_source.return_value = ("", "snippets") + judge = MagicMock() + judge.score_answer.return_value = {"aggregate": 60.0} + rec = {"question_id": "q1", "question_text": "How?", + "arms": {"baseline": {"answer": "base"}, "skill": {"answer": " "}}} + [row] = runner.judge(judge, "lib", [rec], baseline_arm="baseline") + assert row["scores"]["skill"] == {"error": "empty_reply"} + assert judge.score_answer.call_count == 1 + + +def test_judge_records_a_legacy_empty_answer_without_error_as_an_error_cell(): + runner = ArmRunner.__new__(ArmRunner) + runner.judge_context = MagicMock() + runner.judge_context.resolve_with_source.return_value = ("", "snippets") + judge = MagicMock() + judge.score_answer.return_value = {"aggregate": 60.0} + rec = {"question_id": "q1", "question_text": "How?", + "arms": {"baseline": {"answer": "base"}, "skill": {"answer": ""}}} + [row] = runner.judge(judge, "lib", [rec], baseline_arm="baseline") + assert row["scores"]["skill"] == {"error": "empty_reply"} + assert cell_values([row], ["baseline", "skill"])[1] == {"baseline": 0, "skill": 1} From 32709c9e4ac19527e3c065260107e75d72fe6c56 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 28 Oct 2026 20:33:54 -0700 Subject: [PATCH 05/24] fix(arms): a judge error with an empty message is still an error cell The judge's exception handler stored str(exc). Some exceptions have an empty message (TimeoutError()), so the cell held {"error": ""}; the record reader drops an empty error, and the cell was neither scored nor counted as an error. The exception's class name is the error when the message is empty. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/eval/arm_runner.py | 3 ++- dokimi/tests/test_arm_cell_errors.py | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/eval/arm_runner.py b/dokimi/dokimi/eval/arm_runner.py index c234f145b..e6ca8c0eb 100644 --- a/dokimi/dokimi/eval/arm_runner.py +++ b/dokimi/dokimi/eval/arm_runner.py @@ -642,7 +642,8 @@ def _score(rec: Record) -> Dict[str, Any]: ) except Exception as exc: logger.exception("Judging arm %s failed", arm_name) - score = {"error": str(exc)} + # str() of some exceptions is empty (TimeoutError()); an empty error reads as no error. + score = {"error": str(exc) or type(exc).__name__} if isinstance(score, dict): score = {**score, "judge_context_source": source} digest = (self.judge_context.digest_of(rec.question_id, arm_name) diff --git a/dokimi/tests/test_arm_cell_errors.py b/dokimi/tests/test_arm_cell_errors.py index 40a751d66..bb124e974 100644 --- a/dokimi/tests/test_arm_cell_errors.py +++ b/dokimi/tests/test_arm_cell_errors.py @@ -78,3 +78,22 @@ def test_judge_records_a_legacy_empty_answer_without_error_as_an_error_cell(): [row] = runner.judge(judge, "lib", [rec], baseline_arm="baseline") assert row["scores"]["skill"] == {"error": "empty_reply"} assert cell_values([row], ["baseline", "skill"])[1] == {"baseline": 0, "skill": 1} + + +def test_a_judge_exception_with_an_empty_message_is_still_an_error_cell(): + runner = ArmRunner.__new__(ArmRunner) + runner.judge_context = MagicMock() + runner.judge_context.resolve_with_source.return_value = ("", "snippets") + + def score_answer(**kw): + if kw["answer"] == "x": + raise TimeoutError() # str() of this is "" + return {"aggregate": 60.0} + + judge = MagicMock() + judge.score_answer.side_effect = score_answer + rec = {"question_id": "q1", "question_text": "How?", + "arms": {"baseline": {"answer": "base"}, "skill": {"answer": "x"}}} + [row] = runner.judge(judge, "lib", [rec], baseline_arm="baseline") + assert row["scores"]["skill"]["error"] == "TimeoutError" + assert cell_values([row], ["baseline", "skill"])[1] == {"baseline": 0, "skill": 1} From 9547e8758318d55c8c1ea26631e8a04801939d67 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 28 Oct 2026 20:34:38 -0700 Subject: [PATCH 06/24] fix(eval): an empty reference reply is an error, not graded as "" eval reference passed an empty or whitespace-only reply to the checks as text, so a negated check scored it 1.0 and a plain one 0. Only a missing reply was treated as no answer. An empty reply is now an errored answer: it leaves n, the pass rate and the mean out, and is counted as an error, as an errored call is. The emptiness test is the shared is_empty_reply. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/eval/reference_run.py | 30 ++++++++++++++++++++------- dokimi/tests/test_reference_checks.py | 14 +++++++++++++ 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/dokimi/dokimi/eval/reference_run.py b/dokimi/dokimi/eval/reference_run.py index ac0ab1c10..f3e5f9494 100644 --- a/dokimi/dokimi/eval/reference_run.py +++ b/dokimi/dokimi/eval/reference_run.py @@ -21,8 +21,9 @@ from datetime import datetime, timezone from typing import Any, Mapping, Optional +from dokimi.caching import is_empty_reply from dokimi.eval.reference_checks import Ask, Grade, grade, validate_grading -from dokimi.records import NO_RETRIEVAL_SOURCES, Record, Run, read +from dokimi.records import NO_RETRIEVAL_SOURCES, Answer, Record, Run, read SCORER = "reference" @@ -91,12 +92,25 @@ def grading_by_id(questions: list[Mapping[str, Any]]) -> dict[str, Mapping[str, return out -def _grade_record(rec: Record, grading: Mapping[str, Any], ask: Optional[Ask]) -> dict[str, Grade]: - out: dict[str, Grade] = {} +def _answer_state(answer: Answer) -> tuple[Optional[str], bool]: + """The text to grade, and whether the answer is an error. + + An errored answer, and an empty one (no content, or only whitespace), has no answer to + grade: it is an error, never graded as "" (a negated check would pass it). A no-retrieval + placeholder ("[FALLBACK: ...]") is not the model's answer either, but it is unscored, not an error. + """ + if answer.doc_source in NO_RETRIEVAL_SOURCES and not answer.error: + return None, False + if answer.error or is_empty_reply(answer.text): + return None, True + return answer.text, False + + +def _grade_record(rec: Record, grading: Mapping[str, Any], ask: Optional[Ask]) -> dict[str, tuple[Grade, bool]]: + out: dict[str, tuple[Grade, bool]] = {} for name, answer in rec.answers.items(): - # A no-retrieval placeholder ("[FALLBACK: ...]") is not the model's answer: unscored. - text = None if answer.error or answer.doc_source in NO_RETRIEVAL_SOURCES else answer.text - out[name] = grade(grading, rec.question_text, text, ask) + text, errored = _answer_state(answer) + out[name] = (grade(grading, rec.question_text, text, ask), errored) return out @@ -114,8 +128,8 @@ def score(data: Any, questions: list[Mapping[str, Any]], *, ask: Optional[Ask] = out = ReferenceRun(run=run, evaluations=[], missing=missing) for rec, per_condition in zip(graded, grades, strict=True): blocks = {} - for name, g in per_condition.items(): - out.tally.setdefault(name, ConditionTally()).add(g, errored=bool(rec.answers[name].error)) + for name, (g, errored) in per_condition.items(): + out.tally.setdefault(name, ConditionTally()).add(g, errored=errored) blocks[name] = {**g.as_dict(), "scorer": SCORER} out.evaluations.append(_evaluation(run, rec, blocks)) return out diff --git a/dokimi/tests/test_reference_checks.py b/dokimi/tests/test_reference_checks.py index d737b97d8..ccc537b2c 100644 --- a/dokimi/tests/test_reference_checks.py +++ b/dokimi/tests/test_reference_checks.py @@ -400,3 +400,17 @@ def test_read_promptfoo_refuses_several_providers(tmp_path): path.write_text(json.dumps({"results": {"results": rows}}), encoding="utf-8") with pytest.raises(ValueError, match="2 providers"): read_log(path) + + +@pytest.mark.parametrize("reply", ["", " \n\t", None]) +def test_an_empty_reply_is_an_error_not_a_score_in_eval_reference(reply): + """An empty reply is no answer: never graded (a negated check would pass "" as a 1.0).""" + negated = {"checks": [{"type": "match", "values": ["i don't know"], "location": "any", "negate": True}]} + plain = {"checks": [{"type": "match", "values": ["paris"], "location": "any"}]} + for grading in (negated, plain): + data = {"baseline_arm": "baseline", "arms": ["baseline", "skill"], "answers": [ + {"question_id": "q1", "question_text": "Capital?", + "arms": {"baseline": {"answer": "Paris"}, "skill": {"answer": reply}}}]} + result = score(data, [{"id": "q1", "question": "Capital?", "grading": grading}]) + tally = result.tally["skill"].as_dict() + assert (tally["n"], tally["errors"], tally["pass_rate"], tally["mean_score"]) == (0, 1, None, None) From 73cdb3e31e1b74801f5aaaec28bca715c9dd1608 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 28 Oct 2026 20:35:20 -0700 Subject: [PATCH 07/24] fix(arms): delta_vs_baseline is paired over the questions both arms scored The headline delta was the difference of two unpaired means, so a question one arm errored on moved the delta (the repro gave -8.3 where the paired difference over the two shared questions is -10.0). It is now the mean of arm minus baseline over the questions both arms scored, via paired_delta. The unpaired avg_aggregate is still reported. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/eval/arm_runner.py | 7 ++++--- dokimi/dokimi/eval/arm_summary.py | 16 +++++++++++++++- dokimi/tests/test_arm_cell_errors.py | 20 ++++++++++++++++++++ 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/dokimi/dokimi/eval/arm_runner.py b/dokimi/dokimi/eval/arm_runner.py index e6ca8c0eb..f3687de51 100644 --- a/dokimi/dokimi/eval/arm_runner.py +++ b/dokimi/dokimi/eval/arm_runner.py @@ -26,7 +26,7 @@ from dokimi.eval import stats as arm_stats from dokimi.eval.interactions import interaction_significance from dokimi.eval.arm_summary import ( - cell_values, cost_summary, summarize_agentic_usage, summarize_skill_selection, + cell_values, cost_summary, paired_delta, summarize_agentic_usage, summarize_skill_selection, ) from dokimi.eval.cells import Cell from dokimi.eval.question_text import question_text_of, require_question_texts @@ -775,10 +775,11 @@ def _summarize( if arm_name == baseline_arm: base_avg = avg if base_avg is not None: + # Paired: over the questions both arms scored, so a question one arm errored on + # does not shift the delta. The unpaired means above are still reported. for arm_name, stats in summary["per_arm"].items(): - avg = stats["avg_aggregate"] stats["delta_vs_baseline"] = ( - round(avg - base_avg, 1) if avg is not None and arm_name != baseline_arm else None + paired_delta(evaluations, baseline_arm, arm_name) if arm_name != baseline_arm else None ) # Paired statistics. Every arm is tested against the *same* baseline diff --git a/dokimi/dokimi/eval/arm_summary.py b/dokimi/dokimi/eval/arm_summary.py index 4a960ceb4..db484ab58 100644 --- a/dokimi/dokimi/eval/arm_summary.py +++ b/dokimi/dokimi/eval/arm_summary.py @@ -13,7 +13,7 @@ from __future__ import annotations -from typing import Any, Dict, Iterable, List, Mapping, Sequence, Tuple +from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple from dokimi.records import RecordLike from dokimi.records import records as as_records @@ -266,6 +266,20 @@ def cost_summary(records: Iterable[RecordLike]) -> Dict[str, Any]: return out +def paired_delta(evaluations: Iterable[Any], baseline_arm: str, arm_name: str) -> Optional[float]: + """The mean of ``arm - baseline`` over the questions that both arms scored; ``None`` if none. + + A question one arm errored on (or left unscored) is not in the delta at all: the + difference of two unpaired means would compare different questions. + """ + diffs: List[float] = [] + for ev in as_records(evaluations, source="evaluations", kind="evaluations"): + base, arm = ev.score(baseline_arm), ev.score(arm_name) + if base is not None and arm is not None: + diffs.append(float(arm) - float(base)) + return round(sum(diffs) / len(diffs), 1) if diffs else None + + def cell_values(evaluations: Iterable[Any], arm_names: Sequence[str]) -> Tuple[Dict[str, List[float]], Dict[str, int]]: """Per arm, the aggregates that count, and how many cells errored instead. diff --git a/dokimi/tests/test_arm_cell_errors.py b/dokimi/tests/test_arm_cell_errors.py index bb124e974..ea5e89533 100644 --- a/dokimi/tests/test_arm_cell_errors.py +++ b/dokimi/tests/test_arm_cell_errors.py @@ -5,6 +5,7 @@ from __future__ import annotations +from types import SimpleNamespace from unittest.mock import MagicMock import pytest @@ -97,3 +98,22 @@ def score_answer(**kw): [row] = runner.judge(judge, "lib", [rec], baseline_arm="baseline") assert row["scores"]["skill"]["error"] == "TimeoutError" assert cell_values([row], ["baseline", "skill"])[1] == {"baseline": 0, "skill": 1} + + +def test_delta_vs_baseline_is_paired_over_the_questions_both_arms_scored(): + runner = ArmRunner.__new__(ArmRunner) + runner.treatments = [SimpleNamespace(name="baseline"), SimpleNamespace(name="skill")] + runner.turn_reminder = None + runner.stats_correction = "holm" + evaluations = [ + {"question_id": "1", "question_text": "Q1", + "scores": {"baseline": {"aggregate": 80}, "skill": {"aggregate": 70}}}, + # Scored by the baseline only: left out of the paired delta, not counted against the skill arm. + {"question_id": "2", "question_text": "Q2", + "scores": {"baseline": {"aggregate": 60}, "skill": {"error": "generation_failed: HTTP 500"}}}, + {"question_id": "3", "question_text": "Q3", + "scores": {"baseline": {"aggregate": 50}, "skill": {"aggregate": 40}}}, + ] + summary = runner._summarize(evaluations, "baseline") + assert summary["per_arm"]["skill"]["delta_vs_baseline"] == -10.0 + assert summary["per_arm"]["skill"]["avg_aggregate"] == 55.0 # the unpaired mean is still reported From 4a8b0ade67e570cd90a1887206503fb7a56fd55b Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 28 Oct 2026 20:40:11 -0700 Subject: [PATCH 08/24] refactor(arms): keep arm_runner under the growth cap The empty-reply cells added to arm_runner put it at 809 lines (cap 800) and pushed ArmRunner.judge past its complexity baseline. The unjudged cell (an error for an errored or empty answer, None for no answer) moves to arm_summary.unjudged_cell, and the single-turn empty-reply return is shorter. No behaviour change. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/eval/arm_runner.py | 23 +++++++---------------- dokimi/dokimi/eval/arm_summary.py | 9 ++++++++- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/dokimi/dokimi/eval/arm_runner.py b/dokimi/dokimi/eval/arm_runner.py index f3687de51..19f38cd22 100644 --- a/dokimi/dokimi/eval/arm_runner.py +++ b/dokimi/dokimi/eval/arm_runner.py @@ -26,7 +26,7 @@ from dokimi.eval import stats as arm_stats from dokimi.eval.interactions import interaction_significance from dokimi.eval.arm_summary import ( - cell_values, cost_summary, paired_delta, summarize_agentic_usage, summarize_skill_selection, + cell_values, cost_summary, paired_delta, summarize_agentic_usage, summarize_skill_selection, unjudged_cell, ) from dokimi.eval.cells import Cell from dokimi.eval.question_text import question_text_of, require_question_texts @@ -399,12 +399,10 @@ def _answer_one_arm( final_chars = len(final_text or "") rec = to_record(usage, self.model, self.provider) - if is_empty_reply(final_text): - # As a session that stopped on an empty reply: an error cell, never an answer. - # The call was paid for, so its usage counts; no answer-length keys (see _session_spend). - return {"error": "empty_reply", "answer": None, - "token_usage": rec.as_token_usage_dict(), "metrics": rec.as_metrics_dict(), - "metadata": cfg_metadata, "elapsed_sec": round(time.time() - t0, 2)} + if is_empty_reply(final_text): # an error cell, as a session's empty turn; the usage still counts + return {"error": "empty_reply", "answer": None, "token_usage": rec.as_token_usage_dict(), + "metrics": rec.as_metrics_dict(), "metadata": cfg_metadata, + "elapsed_sec": round(time.time() - t0, 2)} return { "answer": final_text, "raw_answer": answer_text if shapers else None, @@ -622,12 +620,7 @@ def _score(rec: Record) -> Dict[str, Any]: for arm_name in arm_names(rec): answer = rec.answer(arm_name) if answer is None or is_empty_reply(answer.text): - # An answer that errored is an error on the cell, not a missing score. An empty - # reply is never judged: it is an error on the cell too (as a session's empty turn). - if answer is None: - scores[arm_name] = None - else: - scores[arm_name] = {"error": answer.error or "empty_reply"} + scores[arm_name] = unjudged_cell(answer) # an errored or empty answer is never judged continue context, source = self.judge_context.resolve_with_source( rec.question_id, arm_name, dict(answer.raw) @@ -642,7 +635,6 @@ def _score(rec: Record) -> Dict[str, Any]: ) except Exception as exc: logger.exception("Judging arm %s failed", arm_name) - # str() of some exceptions is empty (TimeoutError()); an empty error reads as no error. score = {"error": str(exc) or type(exc).__name__} if isinstance(score, dict): score = {**score, "judge_context_source": source} @@ -775,8 +767,7 @@ def _summarize( if arm_name == baseline_arm: base_avg = avg if base_avg is not None: - # Paired: over the questions both arms scored, so a question one arm errored on - # does not shift the delta. The unpaired means above are still reported. + # Paired over the questions both arms scored; the unpaired means above are still reported. for arm_name, stats in summary["per_arm"].items(): stats["delta_vs_baseline"] = ( paired_delta(evaluations, baseline_arm, arm_name) if arm_name != baseline_arm else None diff --git a/dokimi/dokimi/eval/arm_summary.py b/dokimi/dokimi/eval/arm_summary.py index db484ab58..ed157899a 100644 --- a/dokimi/dokimi/eval/arm_summary.py +++ b/dokimi/dokimi/eval/arm_summary.py @@ -15,7 +15,7 @@ from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple -from dokimi.records import RecordLike +from dokimi.records import Answer, RecordLike from dokimi.records import records as as_records #: Stamp on an agentic answer's ``tool_usage["counts"]``: its tool counts skip @@ -266,6 +266,13 @@ def cost_summary(records: Iterable[RecordLike]) -> Dict[str, Any]: return out +def unjudged_cell(answer: Optional[Answer]) -> Optional[Dict[str, str]]: + """The cell of an answer that is not judged: an error for an errored or empty one, ``None`` for none.""" + if answer is None: + return None + return {"error": answer.error or "empty_reply"} + + def paired_delta(evaluations: Iterable[Any], baseline_arm: str, arm_name: str) -> Optional[float]: """The mean of ``arm - baseline`` over the questions that both arms scored; ``None`` if none. From a97c38b8e642877ea9dde2ed9f1a1e1c92a8ca12 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 28 Oct 2026 20:40:11 -0700 Subject: [PATCH 09/24] docs(changelog): empty and content-less replies are errors, judge errors count, delta is paired Describes the four fixes on this branch: empty reply as an error cell in arms and eval reference, the judge exception with an empty message, the response cache refusing content-less replies, and the paired delta_vs_baseline. Signed-off-by: Nikolay Petrov --- .../changelog.d/20261028_120200_napetrov_feat-run-hygiene.md | 5 ++++- dokimi/docs/answer-formats.md | 5 +++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/dokimi/changelog.d/20261028_120200_napetrov_feat-run-hygiene.md b/dokimi/changelog.d/20261028_120200_napetrov_feat-run-hygiene.md index 330bc5d48..43438dc2e 100644 --- a/dokimi/changelog.d/20261028_120200_napetrov_feat-run-hygiene.md +++ b/dokimi/changelog.d/20261028_120200_napetrov_feat-run-hygiene.md @@ -1,4 +1,7 @@ ### Fixed - **Errored answers are counted as errors, not as unscored.** An answer whose call failed is tallied as `errors` in `eval reference` summaries, and as `errors` per arm in arms summaries. Errors leave `n`, the pass rate and the mean out. Before, they were lumped with answers that no check could score. -- **Empty replies are not cached.** An empty or whitespace-only model reply is neither stored nor replayed from the response cache, so a later run asks the model again instead of replaying an empty answer as a failure. +- **Empty replies are errors, never scores.** An empty, whitespace-only or content-less model reply is an error on its cell in arms runs and in `eval reference`: it is never judged or graded (a negated check no longer scores `""` as 1.0), and it leaves `n`, the pass rate and the mean out. +- **Judge failures with an empty message are counted.** A judge exception whose message is empty (for example `TimeoutError()`) is recorded under its class name, so it is counted as an error rather than dropped. +- **Empty replies and content-less replies are not cached.** Neither is stored in or replayed from the response cache, so a later run asks the model again. +- **`delta_vs_baseline` is paired.** It is the mean of arm minus baseline over the questions both arms scored. Before, it was the difference of two unpaired means, so a question one arm errored on moved the delta. diff --git a/dokimi/docs/answer-formats.md b/dokimi/docs/answer-formats.md index 7386d0257..a5e6280fa 100644 --- a/dokimi/docs/answer-formats.md +++ b/dokimi/docs/answer-formats.md @@ -81,8 +81,9 @@ run to a result (a `regex` past its limit): the pass is decided on the checks that ran, and the count shows how many passes left one out. An answer whose call failed is counted as `errors`, not `unscored`, and leaves the pass rate and the mean out just as an unscored answer does. An arms summary has the same -count per arm as `errors`. An empty reply is never cached (on store or replay), -so a rerun asks the model again instead of replaying an empty answer. +count per arm as `errors`. An empty or content-less reply is an error, never graded: +it leaves the pass rate and the mean out, and is never cached (on store or replay), +so a rerun asks the model again. `eval reference` writes eval.v1 for an `answers generate` file and an arms artifact for an `arms run` file. Each block carries `scorer: reference`, and From 16710beba2d862bd5baaca4065c253ab756c76c1 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 28 Oct 2026 21:20:13 -0700 Subject: [PATCH 10/24] fix(eval): a judge failure is an error, not a pass; empty replies stay empty In eval reference, a judge check whose call raised, or whose reply had no YES or NO, was dropped, and the checks that ran decided the pass. The answer is now an error: no score and no pass, counted in errors, with the reason on the block as `error`. The checklist report keeps its per-point behaviour; the strict mode is opt-in. Arms: emptiness is read on the model's reply before the output shapers run, so a footer added to an empty reply is not an answer. delta_vs_baseline is always present (null when no question has both arms scored). The arms summary shows the errors count in the Markdown table and the console summary. Signed-off-by: Nikolay Petrov --- ...261028_120200_napetrov_feat-run-hygiene.md | 4 ++ dokimi/docs/answer-formats.md | 13 +++-- dokimi/dokimi/commands/arms.py | 3 +- dokimi/dokimi/eval/arm_runner.py | 25 ++++----- dokimi/dokimi/eval/checklist.py | 18 ++++++- dokimi/dokimi/eval/reference_checks.py | 41 ++++++++++---- dokimi/dokimi/eval/reference_run.py | 32 +++++------ dokimi/dokimi/report/arms_report.py | 6 +-- dokimi/tests/test_arm_cell_errors.py | 31 +++++++++++ dokimi/tests/test_arms_report.py | 15 ++++++ dokimi/tests/test_reference_checks.py | 53 +++++++++++++++++++ 11 files changed, 192 insertions(+), 49 deletions(-) diff --git a/dokimi/changelog.d/20261028_120200_napetrov_feat-run-hygiene.md b/dokimi/changelog.d/20261028_120200_napetrov_feat-run-hygiene.md index 43438dc2e..9f22c7cd6 100644 --- a/dokimi/changelog.d/20261028_120200_napetrov_feat-run-hygiene.md +++ b/dokimi/changelog.d/20261028_120200_napetrov_feat-run-hygiene.md @@ -5,3 +5,7 @@ - **Judge failures with an empty message are counted.** A judge exception whose message is empty (for example `TimeoutError()`) is recorded under its class name, so it is counted as an error rather than dropped. - **Empty replies and content-less replies are not cached.** Neither is stored in or replayed from the response cache, so a later run asks the model again. - **`delta_vs_baseline` is paired.** It is the mean of arm minus baseline over the questions both arms scored. Before, it was the difference of two unpaired means, so a question one arm errored on moved the delta. +- **A judge failure is an error, not a pass.** In `eval reference`, a judge check whose call raised, or whose reply had no YES or NO, makes the answer an error: no score and no pass, even when its other checks passed. Before, the other checks decided the pass. The block carries `error`, as an arm cell does. +- **An empty reply stays an error when an output shaper adds text.** Emptiness is read on the model's reply before the shapers run, so a footer on an empty reply is not an answer. +- **`delta_vs_baseline` is always present.** It is `null` when no question has both arms scored, instead of missing. +- **The arms summary shows errors.** The Markdown report has an `Errors` column, and the console summary shows the count per arm. diff --git a/dokimi/docs/answer-formats.md b/dokimi/docs/answer-formats.md index a5e6280fa..def75de9d 100644 --- a/dokimi/docs/answer-formats.md +++ b/dokimi/docs/answer-formats.md @@ -71,8 +71,8 @@ whose value is blank (whitespace included) imports as `unsupported` too. Every check accepts `mode` (`any`/`all` over its `values`), `negate` and `weight`. A question's score is the weighted mean of the checks that were counted. It passes when it reaches `threshold`; without a threshold, it passes -when every counted check passes. A failed answer call leaves its checks -unscored rather than failed. `--no-judge` leaves the `judge` checks unscored +when every counted check passes. A failed answer call is an error, not a failed +check: the answer gets no score and no pass. `--no-judge` leaves the `judge` checks unscored and makes no model calls; an arms artifact written that way has `judge_model` and `judge_provider` set to `reference`. The summary's `judge_unscored` counts the counted answers whose pass leaves out a judge check that had no verdict. The @@ -80,7 +80,10 @@ summary's `check_unscored` does the same for a deterministic check that did not run to a result (a `regex` past its limit): the pass is decided on the checks that ran, and the count shows how many passes left one out. An answer whose call failed is counted as `errors`, not `unscored`, and leaves the pass rate and -the mean out just as an unscored answer does. An arms summary has the same +the mean out just as an unscored answer does. A judge call that raises, or whose +reply has no YES or NO after its retries, is an error too: the answer gets no score +and no pass, even when its other checks passed. The block's `error` names it (for +example `judge: TimeoutError`), as an arm cell's `error` does. An arms summary has the same count per arm as `errors`. An empty or content-less reply is an error, never graded: it leaves the pass rate and the mean out, and is never cached (on store or replay), so a rerun asks the model again. @@ -119,7 +122,7 @@ because the native run had no dokimi doc context. The tool's own verdicts become an eval.v1 file with `scorer: native:`. Running `eval reference` on that answers file re-grades the same completions with dokimi's checks, which shows where the two graders disagree. An Inspect sample that errored -keeps its error on the answer, so `eval reference` leaves it unscored, as -Inspect does. A promptfoo output must come from +keeps its error on the answer, so `eval reference` counts it as an error, not a +score, as Inspect does. A promptfoo output must come from a single provider. Supported log formats are Inspect `json` logs (inspect-ai 0.3.283) and promptfoo `-o *.json` (0.125.0). diff --git a/dokimi/dokimi/commands/arms.py b/dokimi/dokimi/commands/arms.py index 5696708f8..fce372f82 100644 --- a/dokimi/dokimi/commands/arms.py +++ b/dokimi/dokimi/commands/arms.py @@ -103,7 +103,8 @@ def cmd_arms_run(args: argparse.Namespace) -> None: delta = stats.get("delta_vs_baseline") avg_s = "n/a" if avg is None else f"{avg:.1f}" delta_s = "" if (delta is None or arm == args.baseline_arm) else f" (delta {delta:+.1f})" - print(f" {arm:<24} {avg_s}{delta_s}") + err_s = f" ({stats['errors']} errors)" if stats.get("errors") else "" + print(f" {arm:<24} {avg_s}{delta_s}{err_s}") def cmd_arms_matrix_run(args: argparse.Namespace) -> None: diff --git a/dokimi/dokimi/eval/arm_runner.py b/dokimi/dokimi/eval/arm_runner.py index 19f38cd22..4ff63819b 100644 --- a/dokimi/dokimi/eval/arm_runner.py +++ b/dokimi/dokimi/eval/arm_runner.py @@ -394,15 +394,15 @@ def _answer_one_arm( return {"error": f"generation_failed: {exc}", "answer": None, "metadata": cfg_metadata} - raw_chars = len(answer_text or "") - final_text = self._apply_shapers(shapers, answer_text) - final_chars = len(final_text or "") - rec = to_record(usage, self.model, self.provider) - if is_empty_reply(final_text): # an error cell, as a session's empty turn; the usage still counts + # Read on the reply the model gave, before the shapers: a footer added to nothing is not an answer. + if is_empty_reply(answer_text): # an error cell, as a session's empty turn; the usage still counts return {"error": "empty_reply", "answer": None, "token_usage": rec.as_token_usage_dict(), "metrics": rec.as_metrics_dict(), "metadata": cfg_metadata, "elapsed_sec": round(time.time() - t0, 2)} + final_text = self._apply_shapers(shapers, answer_text) + raw_chars = len(answer_text or "") + final_chars = len(final_text or "") return { "answer": final_text, "raw_answer": answer_text if shapers else None, @@ -760,18 +760,15 @@ def _summarize( per_arm, errors = cell_values(evaluations, self.arm_names) summary: Dict[str, Any] = {"per_arm": {}} - base_avg = None for arm_name, vals in per_arm.items(): avg = round(sum(vals) / len(vals), 1) if vals else None summary["per_arm"][arm_name] = {"avg_aggregate": avg, "n": len(vals), "errors": errors.get(arm_name, 0)} - if arm_name == baseline_arm: - base_avg = avg - if base_avg is not None: - # Paired over the questions both arms scored; the unpaired means above are still reported. - for arm_name, stats in summary["per_arm"].items(): - stats["delta_vs_baseline"] = ( - paired_delta(evaluations, baseline_arm, arm_name) if arm_name != baseline_arm else None - ) + # Paired over the questions both arms scored; the unpaired means above are still reported. + # Always present: null for the baseline, and for an arm with no question both arms scored. + for arm_name, stats in summary["per_arm"].items(): + stats["delta_vs_baseline"] = ( + paired_delta(evaluations, baseline_arm, arm_name) if arm_name != baseline_arm else None + ) # Paired statistics. Every arm is tested against the *same* baseline # sample question-by-question, so the arms are mutually comparable and diff --git a/dokimi/dokimi/eval/checklist.py b/dokimi/dokimi/eval/checklist.py index 9a9631b06..aa1af1b4d 100644 --- a/dokimi/dokimi/eval/checklist.py +++ b/dokimi/dokimi/eval/checklist.py @@ -105,16 +105,30 @@ def consistent(self) -> Optional[bool]: return self.covered != self.absent -def check_answer(ask: Ask, question: str, answer: str, points: Sequence[str]) -> List[PointVerdict]: - """Both polarities for every point.""" +class JudgeFailed(RuntimeError): + """A judge call raised, or its reply had no YES or NO after every draw (``strict`` checks only).""" + + +def check_answer(ask: Ask, question: str, answer: str, points: Sequence[str], *, + strict: bool = False) -> List[PointVerdict]: + """Both polarities for every point. + + By default a failed call or an unparseable reply leaves its point unscored. With ``strict`` + it raises :class:`JudgeFailed` instead, for a caller that must not grade an answer on the + points that did parse. + """ def one(point: str, q: str) -> Optional[bool]: try: # Retried under new draws, and an unparseable reply is never cached. verdict, _ = ask_parsed(ask, _TEMPLATE.format(question=question, answer=answer, point=point, ask=q), parse_yes_no) except Exception as exc: # one failed call is an unparsed point, not a lost run + if strict: + raise JudgeFailed(str(exc) or type(exc).__name__) from exc logger.warning("Checklist call failed, point left unscored: %s", exc) return None + if verdict is None and strict: + raise JudgeFailed("no YES or NO in the reply") return verdict return [ diff --git a/dokimi/dokimi/eval/reference_checks.py b/dokimi/dokimi/eval/reference_checks.py index 3ad4e3f70..44321501b 100644 --- a/dokimi/dokimi/eval/reference_checks.py +++ b/dokimi/dokimi/eval/reference_checks.py @@ -77,6 +77,8 @@ class CheckResult: score: Optional[float] weight: float = 1.0 detail: str = "" + #: Set when the check could not be graded because a call failed (the judge), not because it has no verdict. + error: Optional[str] = None def as_dict(self) -> dict[str, Any]: out: dict[str, Any] = {"type": self.type, "passed": self.passed, "score": self.score} @@ -84,15 +86,22 @@ def as_dict(self) -> dict[str, Any]: out["weight"] = self.weight if self.detail: out["detail"] = self.detail + if self.error is not None: + out["error"] = self.error return out @dataclass class Grade: - """One answer graded against one question's checks.""" + """One answer graded against one question's checks. + + An ``error`` (the answer call failed, or a judge call did) means the answer has no grade: no + score and no pass, whatever the other checks said. It is counted as an error, not as unscored. + """ checks: list[CheckResult] = field(default_factory=list) threshold: Optional[float] = None + error: Optional[str] = None @property def counted(self) -> list[CheckResult]: @@ -110,6 +119,8 @@ def unrun(self) -> list[CheckResult]: @property def score(self) -> Optional[float]: + if self.error is not None: + return None counted = self.counted total = sum(c.weight for c in counted) if not counted or total <= 0: @@ -127,11 +138,14 @@ def passed(self) -> Optional[bool]: def as_dict(self) -> dict[str, Any]: score = self.score - return { + out: dict[str, Any] = { "aggregate": None if score is None else round(100 * score, 1), "passed": self.passed, "checks": [c.as_dict() for c in self.checks], } + if self.error is not None: + out["error"] = self.error + return out # --- normalisation ----------------------------------------------------------------------------- @@ -331,9 +345,12 @@ def grade_check(check: Mapping[str, Any], question: str, answer: str, ask: Optio if kind in JUDGED: if ask is None: return CheckResult(kind, None, None, weight, "not judged: no judge model") - from dokimi.eval.checklist import check_answer + from dokimi.eval.checklist import JudgeFailed, check_answer - verdict = check_answer(ask, question, answer, [judge_point(check)])[0] + try: + verdict = check_answer(ask, question, answer, [judge_point(check)], strict=True)[0] + except JudgeFailed as exc: + return CheckResult(kind, None, None, weight, "judge failed", error=f"judge: {exc}") score = verdict.score if score is not None and check.get("negate"): score = 1.0 - score @@ -412,18 +429,24 @@ def _check_problems(kind: str, c: Mapping[str, Any]) -> list[str]: return out -def grade(grading: Mapping[str, Any], question: str, answer: Optional[str], ask: Optional[Ask] = None) -> Grade: +def grade(grading: Mapping[str, Any], question: str, answer: Optional[str], ask: Optional[Ask] = None, + *, answer_error: Optional[str] = None) -> Grade: """Every check of ``grading`` on ``answer``. - A missing answer (the answer call failed) leaves every check unscored rather than failed: - a provider error is not the model's wrong answer, and the grade's score is then ``None``. + A missing answer leaves every check unscored rather than failed: a provider error is not the + model's wrong answer. ``answer_error`` names why there is none; it makes the grade an error. + A judge check whose call failed makes the grade an error too, so no pass is decided by the + checks that did run. """ threshold = grading.get("threshold") - out = Grade(threshold=float(threshold) if threshold is not None else None) + out = Grade(threshold=float(threshold) if threshold is not None else None, error=answer_error) for check in grading.get("checks", []): if answer is None: out.checks.append(CheckResult(str(check.get("type")), None, None, float(check.get("weight", 1.0)), "no answer")) continue - out.checks.append(grade_check(check, question, answer or "", ask)) + result = grade_check(check, question, answer or "", ask) + out.checks.append(result) + if out.error is None and result.error is not None: + out.error = result.error return out diff --git a/dokimi/dokimi/eval/reference_run.py b/dokimi/dokimi/eval/reference_run.py index f3e5f9494..9c3a81779 100644 --- a/dokimi/dokimi/eval/reference_run.py +++ b/dokimi/dokimi/eval/reference_run.py @@ -30,7 +30,7 @@ @dataclass class ConditionTally: - """Counts for one condition. An errored answer (the answer call failed) is ``errors``, apart from + """Counts for one condition. An errored grade (the answer call or a judge call failed) is ``errors``, apart from ``unscored`` (no check could score it, e.g. no judge): neither enters ``n``, the pass rate or the mean.""" n: int = 0 @@ -41,10 +41,10 @@ class ConditionTally: check_unscored: int = 0 score_sum: float = 0.0 - def add(self, g: Grade, *, errored: bool = False) -> None: + def add(self, g: Grade) -> None: score = g.score if score is None: - if errored: + if g.error is not None: self.errors += 1 else: self.unscored += 1 @@ -92,25 +92,27 @@ def grading_by_id(questions: list[Mapping[str, Any]]) -> dict[str, Mapping[str, return out -def _answer_state(answer: Answer) -> tuple[Optional[str], bool]: - """The text to grade, and whether the answer is an error. +def _answer_state(answer: Answer) -> tuple[Optional[str], Optional[str]]: + """The text to grade, and why there is none when it is an error. An errored answer, and an empty one (no content, or only whitespace), has no answer to grade: it is an error, never graded as "" (a negated check would pass it). A no-retrieval placeholder ("[FALLBACK: ...]") is not the model's answer either, but it is unscored, not an error. """ if answer.doc_source in NO_RETRIEVAL_SOURCES and not answer.error: - return None, False - if answer.error or is_empty_reply(answer.text): - return None, True - return answer.text, False + return None, None + if answer.error: + return None, str(answer.error) + if is_empty_reply(answer.text): + return None, "empty_reply" + return answer.text, None -def _grade_record(rec: Record, grading: Mapping[str, Any], ask: Optional[Ask]) -> dict[str, tuple[Grade, bool]]: - out: dict[str, tuple[Grade, bool]] = {} +def _grade_record(rec: Record, grading: Mapping[str, Any], ask: Optional[Ask]) -> dict[str, Grade]: + out: dict[str, Grade] = {} for name, answer in rec.answers.items(): - text, errored = _answer_state(answer) - out[name] = (grade(grading, rec.question_text, text, ask), errored) + text, error = _answer_state(answer) + out[name] = grade(grading, rec.question_text, text, ask, answer_error=error) return out @@ -128,8 +130,8 @@ def score(data: Any, questions: list[Mapping[str, Any]], *, ask: Optional[Ask] = out = ReferenceRun(run=run, evaluations=[], missing=missing) for rec, per_condition in zip(graded, grades, strict=True): blocks = {} - for name, (g, errored) in per_condition.items(): - out.tally.setdefault(name, ConditionTally()).add(g, errored=errored) + for name, g in per_condition.items(): + out.tally.setdefault(name, ConditionTally()).add(g) blocks[name] = {**g.as_dict(), "scorer": SCORER} out.evaluations.append(_evaluation(run, rec, blocks)) return out diff --git a/dokimi/dokimi/report/arms_report.py b/dokimi/dokimi/report/arms_report.py index da298f0ea..c10abe239 100644 --- a/dokimi/dokimi/report/arms_report.py +++ b/dokimi/dokimi/report/arms_report.py @@ -152,8 +152,8 @@ def render_arms_report(data: Dict[str, Any]) -> str: if summary and summary.get("per_arm"): lines.append("## Summary (avg aggregate score, 0–100)") lines.append("") - lines.append("| Arm | Avg | Δ vs baseline | n |") - lines.append("| --- | ---: | ---: | ---: |") + lines.append("| Arm | Avg | Δ vs baseline | n | Errors |") + lines.append("| --- | ---: | ---: | ---: | ---: |") for arm in arms: stats = summary["per_arm"].get(arm, {}) avg = stats.get("avg_aggregate") @@ -165,7 +165,7 @@ def render_arms_report(data: Dict[str, Any]) -> str: delta_s = "—" else: delta_s = f"{delta:+.1f}" - lines.append(f"| `{arm}` | {avg_s} | {delta_s} | {stats.get('n', 0)} |") + lines.append(f"| `{arm}` | {avg_s} | {delta_s} | {stats.get('n', 0)} | {stats.get('errors', 0)} |") lines.append("") lines.extend(_significance_lines(summary, arms, baseline)) lines.extend(_interaction_lines(summary)) diff --git a/dokimi/tests/test_arm_cell_errors.py b/dokimi/tests/test_arm_cell_errors.py index ea5e89533..d635b56c7 100644 --- a/dokimi/tests/test_arm_cell_errors.py +++ b/dokimi/tests/test_arm_cell_errors.py @@ -117,3 +117,34 @@ def test_delta_vs_baseline_is_paired_over_the_questions_both_arms_scored(): summary = runner._summarize(evaluations, "baseline") assert summary["per_arm"]["skill"]["delta_vs_baseline"] == -10.0 assert summary["per_arm"]["skill"]["avg_aggregate"] == 55.0 # the unpaired mean is still reported + + +@pytest.mark.parametrize("reply", ["", " \n\t", None]) +def test_an_empty_reply_stays_an_error_cell_when_an_output_shaper_adds_text(monkeypatch, reply): + """Emptiness is read on the reply the model gave, before the shapers run: a footer is not an answer.""" + import dokimi.plugins as plugins + + usage = {"prompt_tokens": 4, "completion_tokens": 0, "total_tokens": 4} + monkeypatch.setattr(arm_runner_mod, "llm_call_with_usage", lambda *a, **k: (reply, usage)) + monkeypatch.setattr(plugins, "take_output_shapers", lambda cfg: (["footer"], {})) + monkeypatch.setattr(plugins, "apply_output_shapers", lambda shapers, text: text + "\n\n-- Source: docs") + runner = ArmRunner([BaselineTreatment()]) + [rec] = runner.run("lib", [{"id": "q1", "question": "How?"}], concurrency=1) + cell = rec["arms"]["baseline"] + assert cell["answer"] is None + assert cell["error"] == "empty_reply" + + +def test_delta_vs_baseline_is_present_as_null_when_the_baseline_has_no_scored_question(): + runner = ArmRunner.__new__(ArmRunner) + runner.treatments = [SimpleNamespace(name="baseline"), SimpleNamespace(name="skill")] + runner.turn_reminder = None + runner.stats_correction = "holm" + evaluations = [ + {"question_id": "1", "question_text": "Q1", + "scores": {"baseline": {"error": "generation_failed: HTTP 500"}, "skill": {"aggregate": 90}}}, + ] + summary = runner._summarize(evaluations, "baseline") + assert "delta_vs_baseline" in summary["per_arm"]["skill"] + assert summary["per_arm"]["skill"]["delta_vs_baseline"] is None + assert summary["per_arm"]["skill"]["errors"] == 0 and summary["per_arm"]["baseline"]["errors"] == 1 diff --git a/dokimi/tests/test_arms_report.py b/dokimi/tests/test_arms_report.py index c04d861e2..37cd56579 100644 --- a/dokimi/tests/test_arms_report.py +++ b/dokimi/tests/test_arms_report.py @@ -256,3 +256,18 @@ def test_report_flags_tool_counts_that_include_failed_calls(): assert "Tool counts of `old`, `mixed` include failed calls" in md assert "Tool counts of `new`" not in md assert "include failed calls" not in _agentic_report({"new": {**row, "tool_counts": TOOL_COUNTS}}) + + +def test_summary_table_shows_the_errors_count_beside_n(): + data = { + "library_name": "oneTBB", "model": "m", "provider": "openai", + "arms": ["baseline", "skill"], "baseline_arm": "baseline", "total_questions": 3, + "summary": {"per_arm": { + "baseline": {"avg_aggregate": 70.0, "n": 2, "errors": 1}, + "skill": {"avg_aggregate": 60.0, "n": 1, "errors": 2, "delta_vs_baseline": None}, + }}, + } + md = render_arms_report(data) + assert "| Arm | Avg | Δ vs baseline | n | Errors |" in md + assert "| `baseline` | 70.0 | (baseline) | 2 | 1 |" in md + assert "| `skill` | 60.0 | — | 1 | 2 |" in md diff --git a/dokimi/tests/test_reference_checks.py b/dokimi/tests/test_reference_checks.py index ccc537b2c..c21ca21f9 100644 --- a/dokimi/tests/test_reference_checks.py +++ b/dokimi/tests/test_reference_checks.py @@ -414,3 +414,56 @@ def test_an_empty_reply_is_an_error_not_a_score_in_eval_reference(reply): result = score(data, [{"id": "q1", "question": "Capital?", "grading": grading}]) tally = result.tally["skill"].as_dict() assert (tally["n"], tally["errors"], tally["pass_rate"], tally["mean_score"]) == (0, 1, None, None) + + +# --- a judge that fails makes the answer an error, not a pass on the other checks ----------------- + +def _judge_raises(prompt, draw=0, cache_if=None): + raise TimeoutError() + + +def _judge_unparsed(prompt, draw=0, cache_if=None): + return "I cannot tell" + + +_MIXED = {"checks": [{"type": "match", "values": ["paris"], "location": "any"}, + {"type": "judge", "criterion": "names the capital"}], "threshold": 0.5} + + +@pytest.mark.parametrize("ask,error", [ + (_judge_raises, "judge: TimeoutError"), + (_judge_unparsed, "judge: no YES or NO in the reply"), +]) +def test_a_judge_failure_is_an_error_not_a_pass_decided_by_the_other_checks(ask, error): + g = grade(_MIXED, "Q?", "Paris", ask) + assert (g.score, g.passed, g.error) == (None, None, error) + assert g.as_dict()["error"] == error + tally = ConditionTally() + tally.add(g) + assert (tally.n, tally.errors, tally.unscored) == (0, 1, 0) + + +def test_eval_reference_counts_a_judge_failure_as_an_error_for_every_arm(): + data = {"baseline_arm": "baseline", "arms": ["baseline", "skill"], "answers": [ + {"question_id": "q1", "question_text": "Capital?", + "arms": {"baseline": {"answer": "Paris"}, "skill": {"answer": "Paris"}}}]} + result = score(data, [{"id": "q1", "question": "Capital?", "grading": _MIXED}], ask=_judge_raises) + for arm in ("baseline", "skill"): + tally = result.tally[arm].as_dict() + assert (tally["n"], tally["errors"], tally["pass_rate"]) == (0, 1, None) + cell = result.evaluations[0]["scores"][arm] + assert (cell["passed"], cell["aggregate"], cell["error"]) == (None, None, "judge: TimeoutError") + + +def test_a_reference_cell_carries_its_answer_error_as_an_arm_cell_does(tmp_path): + data = {"baseline_arm": "baseline", "arms": ["baseline", "skill"], "answers": [ + {"question_id": "q1", "question_text": "Capital?", + "arms": {"baseline": {"answer": "Paris"}, + "skill": {"answer": None, "error": "generation_failed: HTTP 500"}}}]} + grading = {"checks": [{"type": "match", "values": ["paris"], "location": "any"}]} + result = score(data, [{"id": "q1", "question": "Capital?", "grading": grading}]) + kind, payload = artifact(result, data, judge_model=None, judge_provider=None, questions_path="q.json") + scores = payload["evaluations"][0]["scores"] + assert scores["skill"]["error"] == "generation_failed: HTTP 500" + assert "error" not in scores["baseline"] + save_artifact(kind, payload, tmp_path / "arms.json") # validates arms.v1 with the error on the cell From 21ddc76d649fbb5dc5bc1cc3d539a0f5fe0092b9 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Thu, 29 Oct 2026 11:35:23 +0000 Subject: [PATCH 11/24] Show an absent errors count as unknown in the arms report Signed-off-by: Anton Volkov --- dokimi/dokimi/report/arms_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/dokimi/report/arms_report.py b/dokimi/dokimi/report/arms_report.py index c10abe239..8652e02bf 100644 --- a/dokimi/dokimi/report/arms_report.py +++ b/dokimi/dokimi/report/arms_report.py @@ -165,7 +165,7 @@ def render_arms_report(data: Dict[str, Any]) -> str: delta_s = "—" else: delta_s = f"{delta:+.1f}" - lines.append(f"| `{arm}` | {avg_s} | {delta_s} | {stats.get('n', 0)} | {stats.get('errors', 0)} |") + lines.append(f"| `{arm}` | {avg_s} | {delta_s} | {stats.get('n', 0)} | {stats.get('errors', '—')} |") lines.append("") lines.extend(_significance_lines(summary, arms, baseline)) lines.extend(_interaction_lines(summary)) From d13fcb3880b4d07a4d32e110007214900dd6ee68 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Thu, 29 Oct 2026 11:36:00 +0000 Subject: [PATCH 12/24] Describe the paired delta and counted errors where docs lagged Signed-off-by: Anton Volkov --- dokimi/docs/evaluating-treatments.md | 8 ++++---- dokimi/dokimi/task_formats/imports/native_logs.py | 2 +- dokimi/scripts/check_treatment_batch.py | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/dokimi/docs/evaluating-treatments.md b/dokimi/docs/evaluating-treatments.md index 691ab695a..1aa3d8ee8 100644 --- a/dokimi/docs/evaluating-treatments.md +++ b/dokimi/docs/evaluating-treatments.md @@ -717,10 +717,10 @@ Two gates are deliberately statistical rather than absolute: - **coverage, not zero errors.** A judge call fails occasionally (~1.25% observed against the Intel proxy), and a failed call leaves `aggregate: null`. The gate requires ≥95% of questions scored in *both* arms. -- **paired deltas, not the artifact's own number.** `delta_vs_baseline` averages each arm - over whatever it scored, so one failed call subtracts means taken over different - questions (39 vs 40 was observed, worth ~0.4 points). The script recomputes over - questions both arms scored. It pools questions; a mean of per-cell means differs slightly +- **paired deltas.** The artifact's `delta_vs_baseline` is paired over the questions both + arms scored, so one failed call does not subtract means taken over different + questions (39 vs 40 was observed, worth ~0.4 points). The script recomputes it over + those questions. It pools questions; a mean of per-cell means differs slightly when the sets have different sizes — say which one a report uses. ## Scope and caveats diff --git a/dokimi/dokimi/task_formats/imports/native_logs.py b/dokimi/dokimi/task_formats/imports/native_logs.py index d879c4989..64da32e5f 100644 --- a/dokimi/dokimi/task_formats/imports/native_logs.py +++ b/dokimi/dokimi/task_formats/imports/native_logs.py @@ -56,7 +56,7 @@ def _add(run: NativeRun, qid: Any, text: str, completion: Optional[str], score: passed: Optional[bool], detail: Any = None, error: Optional[str] = None) -> None: answer = None if completion is None else {"answer": completion, "model": run.model} if answer is not None and error: - answer["error"] = error # a failed sample stays unscored in `eval reference` + answer["error"] = error # a failed sample is counted under `errors` in `eval reference` run.answers.append({"question_id": str(qid), "question_text": text, "without_docs": answer}) run.evaluations.append({"question_id": str(qid), "question_text": text, NATIVE_CONDITION: _block(score, passed, run.tool, detail)}) diff --git a/dokimi/scripts/check_treatment_batch.py b/dokimi/scripts/check_treatment_batch.py index 9a165e2d5..32de9d663 100755 --- a/dokimi/scripts/check_treatment_batch.py +++ b/dokimi/scripts/check_treatment_batch.py @@ -12,9 +12,9 @@ `used_system_prompt` (the whole point of a `profile:` arm is that a system prompt reaches the model -- 7720/7720 pre-existing records have it false). 2. **Report paired deltas.** A failed judge call leaves a score record with - `aggregate: None`, so the artifact's own `delta_vs_baseline` can subtract - means taken over different question counts (39 vs 40 was observed). Only - questions where *both* arms scored are comparable. + `aggregate: None`, and the per-arm means are taken over different question + counts (39 vs 40 was observed). Only questions where *both* arms scored are + comparable, as in the artifact's `delta_vs_baseline`. Usage: python scripts/check_treatment_batch.py results/unified_judge/persona-batch \\ From c026063e126ba89c338aba77e250f1e662a3d04f Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:39:34 -0700 Subject: [PATCH 13/24] test(llm): pin the replay half of the keeps rule An empty or null entry stored before the rule must not be replayed as the answer: complete() asks the model again. The test fails when the replay guard is reduced to cache_if alone. Thread: PRRT_kwDOVBg7g86xh-Dg Signed-off-by: Nikolay Petrov --- dokimi/tests/test_llm.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/dokimi/tests/test_llm.py b/dokimi/tests/test_llm.py index 9326fc875..209b0cfae 100644 --- a/dokimi/tests/test_llm.py +++ b/dokimi/tests/test_llm.py @@ -335,6 +335,19 @@ def test_a_reply_without_content_is_asked_again_not_replayed(cache_dir, fake_llm assert not list(cache_dir.glob("*.json")) +@pytest.mark.parametrize("stale", [None, "", " \n"]) +def test_an_empty_entry_stored_before_the_rule_is_not_replayed(stale, cache_dir, fake_llm): + """A leftover empty entry must not replay as the answer with no live call.""" + fake_llm.push("seed", "live answer") + llm = LLM("m", "openai") + assert llm.ask("p") == "seed" + [entry] = cache_dir.glob("*.json") + stored = json.loads(entry.read_text(encoding="utf-8")) + entry.write_text(json.dumps({**stored, "text": stale}), encoding="utf-8") + assert llm.complete("p").text == "live answer" + assert len(fake_llm.requests) == 2 + + def test_chat_is_never_response_cached(cache_dir, fake_llm): fake_llm.push("a", "b") llm = LLM("m", "openai") From 823787c007fbec7002f8c38d78c96bad03c8d9de Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:39:49 -0700 Subject: [PATCH 14/24] test(report): pin the unknown errors count of an older arms artifact A row without an errors key reads as unknown, not as a false zero, and a real zero stays a zero. The test fails when the fallback is changed to 0. Thread: PRRT_kwDOVBg7g86xlq7F Review: PRR_kwDOVBg7g88AAAABU7EbFQ Comment: IC_kwDOVBg7g88AAAABg13P_A Signed-off-by: Nikolay Petrov --- dokimi/tests/test_arms_report.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/dokimi/tests/test_arms_report.py b/dokimi/tests/test_arms_report.py index 37cd56579..71572f112 100644 --- a/dokimi/tests/test_arms_report.py +++ b/dokimi/tests/test_arms_report.py @@ -271,3 +271,19 @@ def test_summary_table_shows_the_errors_count_beside_n(): assert "| Arm | Avg | Δ vs baseline | n | Errors |" in md assert "| `baseline` | 70.0 | (baseline) | 2 | 1 |" in md assert "| `skill` | 60.0 | — | 1 | 2 |" in md + + +def test_summary_table_reads_an_absent_errors_count_as_unknown_not_zero(): + """An artifact written before the count has no `errors` key; a real zero is still a zero.""" + data = { + "library_name": "oneTBB", "model": "m", "provider": "openai", + "arms": ["baseline", "old", "clean"], "baseline_arm": "baseline", "total_questions": 3, + "summary": {"per_arm": { + "baseline": {"avg_aggregate": 70.0, "n": 2, "errors": 0}, + "old": {"avg_aggregate": 60.0, "n": 1, "delta_vs_baseline": -10.0}, + "clean": {"avg_aggregate": 65.0, "n": 2, "errors": 0, "delta_vs_baseline": -5.0}, + }}, + } + md = render_arms_report(data) + assert "| `old` | 60.0 | -10.0 | 1 | — |" in md + assert "| `clean` | 65.0 | -5.0 | 2 | 0 |" in md From f3717a92ffd0b057e38c88341cf9ec71c079d8b8 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:40:36 -0700 Subject: [PATCH 15/24] fix(eval): an agentic arm's empty reply is an error cell before the shapers run _answer_agentic_arm applied the output shapers before any emptiness check, so a footer added to nothing made the text non-empty and the judge scored it. Read the loop's reply first, as the single-turn path does, and build both error cells with one empty_reply_cell in arm_summary.py (arm_runner.py is at its 800 line limit). Covers the answered and max_iterations stops. The test is red on the old code. Thread: PRRT_kwDOVBg7g86xh-DN Review: PRR_kwDOVBg7g88AAAABU5TpUw Signed-off-by: Nikolay Petrov --- dokimi/dokimi/eval/arm_runner.py | 13 +++++++----- dokimi/dokimi/eval/arm_summary.py | 12 +++++++++++ dokimi/tests/test_arm_cell_errors.py | 30 ++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 5 deletions(-) diff --git a/dokimi/dokimi/eval/arm_runner.py b/dokimi/dokimi/eval/arm_runner.py index 4ff63819b..919a91554 100644 --- a/dokimi/dokimi/eval/arm_runner.py +++ b/dokimi/dokimi/eval/arm_runner.py @@ -26,7 +26,7 @@ from dokimi.eval import stats as arm_stats from dokimi.eval.interactions import interaction_significance from dokimi.eval.arm_summary import ( - cell_values, cost_summary, paired_delta, summarize_agentic_usage, summarize_skill_selection, unjudged_cell, + cell_values, cost_summary, empty_reply_cell, paired_delta, summarize_agentic_usage, summarize_skill_selection, unjudged_cell, ) from dokimi.eval.cells import Cell from dokimi.eval.question_text import question_text_of, require_question_texts @@ -397,9 +397,7 @@ def _answer_one_arm( rec = to_record(usage, self.model, self.provider) # Read on the reply the model gave, before the shapers: a footer added to nothing is not an answer. if is_empty_reply(answer_text): # an error cell, as a session's empty turn; the usage still counts - return {"error": "empty_reply", "answer": None, "token_usage": rec.as_token_usage_dict(), - "metrics": rec.as_metrics_dict(), "metadata": cfg_metadata, - "elapsed_sec": round(time.time() - t0, 2)} + return empty_reply_cell(rec, cfg_metadata, t0) final_text = self._apply_shapers(shapers, answer_text) raw_chars = len(answer_text or "") final_chars = len(final_text or "") @@ -571,8 +569,13 @@ def _answer_agentic_arm( "tool_usage": tool_usage, "metadata": cfg_metadata} raw_answer = result["answer"] - final_answer = self._apply_shapers(shapers, raw_answer) rec = to_record(result.get("usage") or result["token_usage"], self.model, self.provider) + if is_empty_reply(raw_answer): # read before the shapers: a footer added to nothing is not an answer + return empty_reply_cell( + rec, cfg_metadata, t0, agentic=True, transcript=result["transcript"], + tool_call_count=result["tool_call_count"], tool_usage=result.get("tool_usage", {}), + iterations=result["iterations"], stopped_reason=result["stopped_reason"]) + final_answer = self._apply_shapers(shapers, raw_answer) metrics = rec.as_metrics_dict( answer_chars=len(raw_answer or ""), final_chars=len(final_answer or "") ) diff --git a/dokimi/dokimi/eval/arm_summary.py b/dokimi/dokimi/eval/arm_summary.py index ed157899a..9a0a360e2 100644 --- a/dokimi/dokimi/eval/arm_summary.py +++ b/dokimi/dokimi/eval/arm_summary.py @@ -13,6 +13,7 @@ from __future__ import annotations +import time from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple from dokimi.records import Answer, RecordLike @@ -273,6 +274,17 @@ def unjudged_cell(answer: Optional[Answer]) -> Optional[Dict[str, str]]: return {"error": answer.error or "empty_reply"} +def empty_reply_cell(rec: Any, metadata: Any, started: float, **extra: Any) -> Dict[str, Any]: + """The answer cell of a call that returned no text: an error, never an answer. + + The call was paid for, so its usage (*rec*) is still recorded. *started* is the ``time.time()`` + at the start of the arm; *extra* carries what the agent loop adds to a cell. + """ + return {"error": "empty_reply", "answer": None, "token_usage": rec.as_token_usage_dict(), + "metrics": rec.as_metrics_dict(), "metadata": metadata, + "elapsed_sec": round(time.time() - started, 2), **extra} + + def paired_delta(evaluations: Iterable[Any], baseline_arm: str, arm_name: str) -> Optional[float]: """The mean of ``arm - baseline`` over the questions that both arms scored; ``None`` if none. diff --git a/dokimi/tests/test_arm_cell_errors.py b/dokimi/tests/test_arm_cell_errors.py index d635b56c7..9b31d5381 100644 --- a/dokimi/tests/test_arm_cell_errors.py +++ b/dokimi/tests/test_arm_cell_errors.py @@ -135,6 +135,36 @@ def test_an_empty_reply_stays_an_error_cell_when_an_output_shaper_adds_text(monk assert cell["error"] == "empty_reply" +@pytest.mark.parametrize("reply, stopped", [("", "answered"), (" \n", "answered"), (None, "max_iterations")]) +def test_an_agentic_empty_reply_stays_an_error_cell_when_an_output_shaper_adds_text( + monkeypatch, demo_skill, reply, stopped): + """The agent loop's empty reply is read before the shapers, as on the single-turn path.""" + import dokimi.eval.agent_runner as agent_runner_mod + import dokimi.plugins as plugins + from dokimi.skills import load_skill + from dokimi.treatments import SkillAgentTreatment + + usage = {"prompt_tokens": 4, "completion_tokens": 0, "total_tokens": 4} + loop_result = {"answer": reply, "transcript": [], "tool_call_count": 0, "tool_usage": {}, + "iterations": 1, "stopped_reason": stopped, "usage": usage, "token_usage": usage, + "per_turn": []} + monkeypatch.setattr(agent_runner_mod, "run_agent_loop", lambda **kw: dict(loop_result)) + monkeypatch.setattr(plugins, "take_output_shapers", lambda cfg: (["footer"], {})) + monkeypatch.setattr(plugins, "apply_output_shapers", lambda shapers, text: (text or "") + "\n\n-- Source: docs") + runner = ArmRunner([SkillAgentTreatment(load_skill(demo_skill))]) + [rec] = runner.run("lib", [{"id": "q1", "question": "How?"}], concurrency=1) + cell = rec["arms"]["skill_agent:demo-skill"] + assert cell["answer"] is None + assert cell["error"] == "empty_reply" + assert cell["token_usage"]["total_tokens"] == 4 # the loop was paid for, so its usage still counts + assert cell["stopped_reason"] == stopped + judge = MagicMock() + judge.score_answer.return_value = {"aggregate": 60.0} + [row] = runner.judge(judge, "lib", [rec], baseline_arm="skill_agent:demo-skill") + assert row["scores"]["skill_agent:demo-skill"] == {"error": "empty_reply"} + assert judge.score_answer.call_count == 0 + + def test_delta_vs_baseline_is_present_as_null_when_the_baseline_has_no_scored_question(): runner = ArmRunner.__new__(ArmRunner) runner.treatments = [SimpleNamespace(name="baseline"), SimpleNamespace(name="skill")] From cbb879f7d121137fdeb6ff6ba65761567d27ef19 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:40:43 -0700 Subject: [PATCH 16/24] Review: PRR_kwDOVBg7g88AAAABU5FakQ: already fixed in d13fcb38 The docs nit of the review (three places describing the old behaviour) was fixed by the reviewer in d13fcb38; no change in this round. Signed-off-by: Nikolay Petrov From 0f31dadc7f08c06323231b03c237a3fdb9c7ce8d Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 17:54:08 -0700 Subject: [PATCH 17/24] fix(eval): a judge check after a failed one makes no call in eval reference Once a judge call fails the grade is an error, so no further judge check can change it. The remaining judge checks are now recorded as not judged instead of each sending its paid calls, usually into the same outage. Deterministic checks still run. Thread: PRRT_kwDOVBg7g86xypBx Signed-off-by: Nikolay Petrov --- dokimi/dokimi/eval/reference_checks.py | 10 +++++++++- dokimi/tests/test_reference_checks.py | 19 +++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/eval/reference_checks.py b/dokimi/dokimi/eval/reference_checks.py index 44321501b..4a46cc1ec 100644 --- a/dokimi/dokimi/eval/reference_checks.py +++ b/dokimi/dokimi/eval/reference_checks.py @@ -440,13 +440,21 @@ def grade(grading: Mapping[str, Any], question: str, answer: Optional[str], ask: """ threshold = grading.get("threshold") out = Grade(threshold=float(threshold) if threshold is not None else None, error=answer_error) + judge_failed = False for check in grading.get("checks", []): + kind = str(check.get("type")) if answer is None: - out.checks.append(CheckResult(str(check.get("type")), None, None, + out.checks.append(CheckResult(kind, None, None, float(check.get("weight", 1.0)), "no answer")) continue + if judge_failed and kind in JUDGED: + # The grade is already an error, so a further judge call could not change it: skip the paid call. + out.checks.append(CheckResult(kind, None, None, float(check.get("weight", 1.0)), + "not judged: an earlier judge check failed")) + continue result = grade_check(check, question, answer or "", ask) out.checks.append(result) if out.error is None and result.error is not None: out.error = result.error + judge_failed = judge_failed or (kind in JUDGED and result.error is not None) return out diff --git a/dokimi/tests/test_reference_checks.py b/dokimi/tests/test_reference_checks.py index c21ca21f9..d9a18b734 100644 --- a/dokimi/tests/test_reference_checks.py +++ b/dokimi/tests/test_reference_checks.py @@ -443,6 +443,25 @@ def test_a_judge_failure_is_an_error_not_a_pass_decided_by_the_other_checks(ask, assert (tally.n, tally.errors, tally.unscored) == (0, 1, 0) +def test_judge_checks_after_a_failed_judge_check_make_no_paid_calls(): + """The grade is an error once a judge call fails, so the judge checks after it are not asked.""" + four = {"checks": [{"type": "judge", "criterion": f"names point {i}"} for i in range(4)]} + counted = {"one": [], "four": []} + + def counting(key): + def ask(prompt, draw=0, cache_if=None): + counted[key].append(prompt) + raise TimeoutError() + return ask + + grade({"checks": four["checks"][:1]}, "Q?", "Paris", counting("one")) + g = grade(four, "Q?", "Paris", counting("four")) + assert len(counted["four"]) == len(counted["one"]) > 0 + assert g.error == "judge: TimeoutError" + assert [c.error for c in g.checks[1:]] == [None, None, None] + assert [c.detail for c in g.checks[1:]] == ["not judged: an earlier judge check failed"] * 3 + + def test_eval_reference_counts_a_judge_failure_as_an_error_for_every_arm(): data = {"baseline_arm": "baseline", "arms": ["baseline", "skill"], "answers": [ {"question_id": "q1", "question_text": "Capital?", From 753e5fb72f2651554f0685277ef56b428ad818da Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 17:54:55 -0700 Subject: [PATCH 18/24] fix(eval): an empty final reply keeps its skill choice in the selection summary The loop loads its skills before the final reply comes back empty. That reply is an error cell, and the selection summary skipped every errored cell, so the choice the transcript recorded was dropped. The run still counts under n_failed; its selection now counts under n. Thread: PRRT_kwDOVBg7g86xypBu Signed-off-by: Nikolay Petrov --- dokimi/dokimi/eval/arm_summary.py | 5 ++++- dokimi/tests/test_skill_selection.py | 8 ++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/eval/arm_summary.py b/dokimi/dokimi/eval/arm_summary.py index 9a0a360e2..8746c0ae1 100644 --- a/dokimi/dokimi/eval/arm_summary.py +++ b/dokimi/dokimi/eval/arm_summary.py @@ -155,7 +155,10 @@ def summarize_skill_selection(records: Iterable[RecordLike]) -> Dict[str, Any]: }) if failed: s["failed"] += 1 - continue + # An empty final reply fails the answer, not the choice: the skills were + # loaded before it came back empty, so the choice is still scored. + if arm.get("error") != "empty_reply": + continue first = skill.get("selected") # Skills loaded together in the first turn are one choice. It is a hit # only if every one of them is acceptable: crediting "the right one is diff --git a/dokimi/tests/test_skill_selection.py b/dokimi/tests/test_skill_selection.py index f720d9d2c..8feef1c05 100644 --- a/dokimi/tests/test_skill_selection.py +++ b/dokimi/tests/test_skill_selection.py @@ -113,6 +113,14 @@ def test_selection_summary_skips_arms_without_skills(): assert summarize_skill_selection([rec]) == {} +def test_an_empty_final_reply_keeps_the_skill_choice_it_made(): + """The loop loaded the skill before its final reply came back empty: the choice is still scored.""" + cell = _rec("linux-perf", "linux-perf") + cell["arms"]["cat"].update(error="empty_reply", answer=None) + s = summarize_skill_selection([cell])["cat"] + assert (s["n"], s["n_failed"], s["selection_accuracy"], s["recall"]) == (1, 1, 1.0, 1.0) + + def test_report_renders_selection_table(): md = render_arms_report({ "arms": ["cat"], "answers": [], From 2170fb80142ed02db95f79b3d8c3e77997781a97 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 17:55:38 -0700 Subject: [PATCH 19/24] refactor(eval): paired_delta reads the stats pairing The complete-case rule lived in both paired_delta and stats.paired_vectors, so a later change to one could make delta_vs_baseline and the significance delta_mean disagree. paired_delta now takes its pairs from paired_vectors. The test fails until paired_delta uses it; the numbers do not change. Thread: PRRT_kwDOVBg7g86xypBy Signed-off-by: Nikolay Petrov --- dokimi/dokimi/eval/arm_summary.py | 8 +++----- dokimi/tests/test_arm_cell_errors.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/dokimi/dokimi/eval/arm_summary.py b/dokimi/dokimi/eval/arm_summary.py index 8746c0ae1..be7790dea 100644 --- a/dokimi/dokimi/eval/arm_summary.py +++ b/dokimi/dokimi/eval/arm_summary.py @@ -18,6 +18,7 @@ from dokimi.records import Answer, RecordLike from dokimi.records import records as as_records +from dokimi.eval.stats import paired_vectors #: Stamp on an agentic answer's ``tool_usage["counts"]``: its tool counts skip #: calls that failed (``error`` in the transcript). Unstamped answers counted @@ -294,11 +295,8 @@ def paired_delta(evaluations: Iterable[Any], baseline_arm: str, arm_name: str) - A question one arm errored on (or left unscored) is not in the delta at all: the difference of two unpaired means would compare different questions. """ - diffs: List[float] = [] - for ev in as_records(evaluations, source="evaluations", kind="evaluations"): - base, arm = ev.score(baseline_arm), ev.score(arm_name) - if base is not None and arm is not None: - diffs.append(float(arm) - float(base)) + arm_vals, base_vals = paired_vectors(evaluations, arm_name, baseline_arm) + diffs = [a - b for a, b in zip(arm_vals, base_vals, strict=True)] return round(sum(diffs) / len(diffs), 1) if diffs else None diff --git a/dokimi/tests/test_arm_cell_errors.py b/dokimi/tests/test_arm_cell_errors.py index 9b31d5381..64f984b92 100644 --- a/dokimi/tests/test_arm_cell_errors.py +++ b/dokimi/tests/test_arm_cell_errors.py @@ -165,6 +165,21 @@ def test_an_agentic_empty_reply_stays_an_error_cell_when_an_output_shaper_adds_t assert judge.score_answer.call_count == 0 +def test_delta_vs_baseline_is_built_on_the_stats_pairing(monkeypatch): + """One pairing rule for the delta and the significance test: paired_delta reads stats.paired_vectors.""" + import dokimi.eval.arm_summary as arm_summary_mod + + calls = [] + + def stub(evaluations, arm_a, arm_b): + calls.append((arm_a, arm_b)) + return [5.0, 1.0], [2.0, 0.0] + + monkeypatch.setattr(arm_summary_mod, "paired_vectors", stub) + assert arm_summary_mod.paired_delta([], "baseline", "skill") == 2.0 + assert calls == [("skill", "baseline")] + + def test_delta_vs_baseline_is_present_as_null_when_the_baseline_has_no_scored_question(): runner = ArmRunner.__new__(ArmRunner) runner.treatments = [SimpleNamespace(name="baseline"), SimpleNamespace(name="skill")] From d3fae291db0dfe6288dff52926fbf98fc50c2fc3 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 17:57:19 -0700 Subject: [PATCH 20/24] refactor(cache): keeps() takes the Optional reply text, no cast in complete() keeps() was annotated text: str while its callers passed a content-less reply's None, so complete() cast the text twice to fit. It is now typed Optional[str] and refuses None before cache_if runs, and complete() passes Reply.text through, narrowed by a None test before the store. Thread: PRRT_kwDOVBg7g86xypB0 Signed-off-by: Nikolay Petrov --- dokimi/dokimi/caching.py | 6 +++--- dokimi/dokimi/llm.py | 6 +++--- dokimi/tests/test_caching.py | 7 +++++++ 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/dokimi/dokimi/caching.py b/dokimi/dokimi/caching.py index 55e133d85..98e779a9e 100644 --- a/dokimi/dokimi/caching.py +++ b/dokimi/dokimi/caching.py @@ -288,7 +288,7 @@ def is_empty_reply(text: Optional[str]) -> bool: return text is None or not text.strip() -def keeps(cache_if: Optional[Callable[[str], bool]], text: str) -> bool: +def keeps(cache_if: Optional[Callable[[str], bool]], text: Optional[str]) -> bool: """Whether a completion may be stored, or a stored one replayed. *cache_if* is the caller's own acceptance test. The judge passes "this reply @@ -300,9 +300,9 @@ def keeps(cache_if: Optional[Callable[[str], bool]], text: str) -> bool: An empty reply (see :func:`is_empty_reply`) is never kept, on store or on replay: a stored empty answer would be replayed as a failed answer on every later run, and a content-less one (stored as null) as "no answer". ``text`` is - ``None`` at runtime for a content-less reply, whatever its annotation says. + ``None`` for a content-less reply. """ - if is_empty_reply(text): + if text is None or is_empty_reply(text): # the None test narrows the type for cache_if return False return cache_if is None or cache_if(text) diff --git a/dokimi/dokimi/llm.py b/dokimi/dokimi/llm.py index 0d91a78f3..1f6310c7a 100644 --- a/dokimi/dokimi/llm.py +++ b/dokimi/dokimi/llm.py @@ -710,7 +710,7 @@ def complete( extra=caching.draw_extra(self._cache_extra(stream, max_tokens), draw), ) hit = caching.load_response(key) - if hit is not None and caching.keeps(cache_if, cast(str, hit[0])): + if hit is not None and caching.keeps(cache_if, hit[0]): return self._metered(Reply(text=hit[0], usage=hit[1])) if caching.replaying_only(): raise caching.CacheMiss(key) @@ -719,8 +719,8 @@ def complete( raise IndexError("reply has no choices") # as llm_call always did; not cached _warn_if_truncated(reply, self.model, max_tokens) # A content-less reply (None) is not stored: keeps() refuses it, so it is never replayed. - text = cast(str, reply.text) - if caching.keeps(cache_if, text): + text = reply.text + if text is not None and caching.keeps(cache_if, text): caching.store_response(key, text, reply.usage) return reply diff --git a/dokimi/tests/test_caching.py b/dokimi/tests/test_caching.py index df69b7885..541fba3dc 100644 --- a/dokimi/tests/test_caching.py +++ b/dokimi/tests/test_caching.py @@ -405,6 +405,13 @@ def test_an_empty_reply_is_not_kept_even_when_the_judge_test_accepts_it(): assert caching.keeps(lambda _text: True, "") is False +def test_keeps_is_annotated_for_the_none_of_a_content_less_reply(): + """Typed Optional[str], so a caller passes Reply.text as it is: no cast before the check.""" + import typing + + assert typing.get_type_hints(caching.keeps)["text"] == typing.Optional[str] + + def test_a_content_less_reply_is_never_stored_or_replayed(): # A reply with no content (None) stored as null would replay as "no answer" on every # later run. complete() takes no tools, so a tool-call reply never reaches this test: From fcc5698ea6fe23bdd08d372f4fe751dfd70d27ab Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 17:57:51 -0700 Subject: [PATCH 21/24] test(cache): replay-only treats an empty stored entry as a miss; name the null-entry test The null-entry test was named "replayed, not a miss" while the rule it pins is the opposite: the entry loads, and complete() refuses to replay it. Rename it. Add the replay-only case: an empty or null stored entry raises CacheMiss instead of replaying, so a replay-only run asks for a fresh answer. The new test fails when the keeps() replay guard is removed. Thread: PRRT_kwDOVBg7g86xtVB4 Signed-off-by: Nikolay Petrov --- dokimi/tests/test_caching.py | 2 +- dokimi/tests/test_llm.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/dokimi/tests/test_caching.py b/dokimi/tests/test_caching.py index 541fba3dc..c2953a6f7 100644 --- a/dokimi/tests/test_caching.py +++ b/dokimi/tests/test_caching.py @@ -297,7 +297,7 @@ def test_usage_values_of_every_field_type_replay(monkeypatch, tmp_path): assert hit == ("a", UsageRecord(**usage, replayed=True)) -def test_null_text_is_replayed_not_a_miss(monkeypatch, tmp_path): +def test_a_null_entry_still_loads(monkeypatch, tmp_path): """A well-formed null entry loads (load_response); complete() still refuses to replay it (keeps).""" monkeypatch.setenv(caching.RESPONSE_CACHE_ENV, str(tmp_path)) (tmp_path / "empty.json").write_text( diff --git a/dokimi/tests/test_llm.py b/dokimi/tests/test_llm.py index 209b0cfae..c3b10ef0f 100644 --- a/dokimi/tests/test_llm.py +++ b/dokimi/tests/test_llm.py @@ -348,6 +348,20 @@ def test_an_empty_entry_stored_before_the_rule_is_not_replayed(stale, cache_dir, assert len(fake_llm.requests) == 2 +@pytest.mark.parametrize("stale", [None, "", " \n"]) +def test_an_empty_entry_is_a_miss_in_replay_only(stale, cache_dir, fake_llm): + """Replay-only serves no empty entry either: it is a miss, not an answer.""" + fake_llm.push("seed", "live answer") + llm = LLM("m", "openai") + assert llm.ask("p") == "seed" + [entry] = cache_dir.glob("*.json") + stored = json.loads(entry.read_text(encoding="utf-8")) + entry.write_text(json.dumps({**stored, "text": stale}), encoding="utf-8") + with caching.replay_only(), pytest.raises(caching.CacheMiss): + llm.complete("p") + assert len(fake_llm.requests) == 1 + + def test_chat_is_never_response_cached(cache_dir, fake_llm): fake_llm.push("a", "b") llm = LLM("m", "openai") From b301b5a89a86c73d1ddd1135b1f14bb4f3ee2a38 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 17:58:16 -0700 Subject: [PATCH 22/24] docs(changelog): file the run-hygiene changes that move scores under Changed Changes that move pass rates, deltas or selection rates for the same inputs go under Changed, as changelog.d/README.md says, with a note that earlier figures are not comparable. The errors fields and the Errors column are Added. The fragment has no test to fail; this is a prose change. Thread: PRRT_kwDOVBg7g86xypBs Signed-off-by: Nikolay Petrov --- .../20261028_120200_napetrov_feat-run-hygiene.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/dokimi/changelog.d/20261028_120200_napetrov_feat-run-hygiene.md b/dokimi/changelog.d/20261028_120200_napetrov_feat-run-hygiene.md index 9f22c7cd6..233629a08 100644 --- a/dokimi/changelog.d/20261028_120200_napetrov_feat-run-hygiene.md +++ b/dokimi/changelog.d/20261028_120200_napetrov_feat-run-hygiene.md @@ -1,11 +1,18 @@ -### Fixed +### Added + +- **Errors are counted apart.** An answer whose call failed is tallied as `errors` in `eval reference` summaries and per arm in arms summaries. The arms summary shows an `Errors` column, and the console summary shows the count per arm. -- **Errored answers are counted as errors, not as unscored.** An answer whose call failed is tallied as `errors` in `eval reference` summaries, and as `errors` per arm in arms summaries. Errors leave `n`, the pass rate and the mean out. Before, they were lumped with answers that no check could score. +### Changed + +- **Earlier figures are not comparable.** Pass rates, deltas and selection rates from runs before this change do not match new ones for the same inputs. Re-run `arms judge` or `eval reference` to compare. +- **Errors leave the scores out.** An answer whose call failed leaves `n`, the pass rate and the mean out. Before, it was lumped with answers that no check could score. - **Empty replies are errors, never scores.** An empty, whitespace-only or content-less model reply is an error on its cell in arms runs and in `eval reference`: it is never judged or graded (a negated check no longer scores `""` as 1.0), and it leaves `n`, the pass rate and the mean out. - **Judge failures with an empty message are counted.** A judge exception whose message is empty (for example `TimeoutError()`) is recorded under its class name, so it is counted as an error rather than dropped. -- **Empty replies and content-less replies are not cached.** Neither is stored in or replayed from the response cache, so a later run asks the model again. - **`delta_vs_baseline` is paired.** It is the mean of arm minus baseline over the questions both arms scored. Before, it was the difference of two unpaired means, so a question one arm errored on moved the delta. - **A judge failure is an error, not a pass.** In `eval reference`, a judge check whose call raised, or whose reply had no YES or NO, makes the answer an error: no score and no pass, even when its other checks passed. Before, the other checks decided the pass. The block carries `error`, as an arm cell does. - **An empty reply stays an error when an output shaper adds text.** Emptiness is read on the model's reply before the shapers run, so a footer on an empty reply is not an answer. + +### Fixed + +- **Empty replies and content-less replies are not cached.** Neither is stored in or replayed from the response cache, so a later run asks the model again. - **`delta_vs_baseline` is always present.** It is `null` when no question has both arms scored, instead of missing. -- **The arms summary shows errors.** The Markdown report has an `Errors` column, and the console summary shows the count per arm. From fb603801a5f5c065c390b50bb7ab2a3a1bb2113d Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 19:54:12 -0700 Subject: [PATCH 23/24] test(caching): keep the content-less reply comment to two lines Signed-off-by: Nikolay Petrov --- dokimi/tests/test_caching.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dokimi/tests/test_caching.py b/dokimi/tests/test_caching.py index c2953a6f7..212a36b24 100644 --- a/dokimi/tests/test_caching.py +++ b/dokimi/tests/test_caching.py @@ -413,9 +413,8 @@ def test_keeps_is_annotated_for_the_none_of_a_content_less_reply(): def test_a_content_less_reply_is_never_stored_or_replayed(): - # A reply with no content (None) stored as null would replay as "no answer" on every - # later run. complete() takes no tools, so a tool-call reply never reaches this test: - # chat() is the tool-calling path and is never response-cached. + # A None reply stored as null would replay as "no answer" on every later run. Tool-call + # replies never get here: complete() takes no tools, and chat() is never response-cached. assert caching.keeps(None, None) is False assert caching.keeps(lambda _text: True, None) is False From a3a74df62c67a1f8da0917efe982051a10944a08 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 30 Oct 2026 08:19:07 -0700 Subject: [PATCH 24/24] test(arms): an agentic empty reply keeps the tool usage the selection summary scores Signed-off-by: Nikolay Petrov --- dokimi/tests/test_arm_cell_errors.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/dokimi/tests/test_arm_cell_errors.py b/dokimi/tests/test_arm_cell_errors.py index 64f984b92..65fa331d6 100644 --- a/dokimi/tests/test_arm_cell_errors.py +++ b/dokimi/tests/test_arm_cell_errors.py @@ -165,6 +165,33 @@ def test_an_agentic_empty_reply_stays_an_error_cell_when_an_output_shaper_adds_t assert judge.score_answer.call_count == 0 +def test_an_agentic_empty_reply_keeps_the_tool_usage_the_selection_summary_scores(monkeypatch, demo_skill): + """The loop loaded the skill before its final reply came back empty: the cell keeps that load.""" + import dokimi.eval.agent_runner as agent_runner_mod + import dokimi.plugins as plugins + from dokimi.eval.arm_summary import summarize_skill_selection + from dokimi.skills import load_skill + from dokimi.treatments import SkillAgentTreatment + + usage = {"prompt_tokens": 4, "completion_tokens": 0, "total_tokens": 4} + tool_usage = {"skill": {"offered": ["demo-skill"], "selected": "demo-skill", "first_turn_loaded": ["demo-skill"]}} + loop_result = {"answer": "", "transcript": [], "tool_call_count": 1, "tool_usage": tool_usage, + "iterations": 2, "stopped_reason": "answered", "usage": usage, "token_usage": usage, + "per_turn": []} + monkeypatch.setattr(agent_runner_mod, "run_agent_loop", lambda **kw: dict(loop_result)) + monkeypatch.setattr(plugins, "take_output_shapers", lambda cfg: (["footer"], {})) + monkeypatch.setattr(plugins, "apply_output_shapers", lambda shapers, text: (text or "") + "\n\n-- Source: docs") + runner = ArmRunner([SkillAgentTreatment(load_skill(demo_skill))]) + [rec] = runner.run("lib", [{"id": "q1", "question": "How?"}], concurrency=1) + arm = "skill_agent:demo-skill" + cell = rec["arms"][arm] + assert cell["error"] == "empty_reply" + assert cell["tool_usage"] == tool_usage + rec["expected_skill"] = "demo-skill" + s = summarize_skill_selection([rec])[arm] + assert (s["n"], s["n_failed"], s["selection_accuracy"]) == (1, 1, 1.0) + + def test_delta_vs_baseline_is_built_on_the_stats_pairing(monkeypatch): """One pairing rule for the delta and the significance test: paired_delta reads stats.paired_vectors.""" import dokimi.eval.arm_summary as arm_summary_mod