From 968f2972caf59489d6fd2a822206d5ffa6db457e Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Wed, 28 Oct 2026 16:22:10 -0700
Subject: [PATCH 01/53] fix(answers): Stack P asks the doc source for top_k
 docs (#228 step 2)

The answerer stored top_k and never sent it, so retrieval asked the doc source for its default max_results. It now sends max_results=top_k, the same request DocTreatment makes on Stack A. With the default top_k of 3 the reranker sees 3 raw docs, not 5, so with-docs answers and their judge scores move.

Flips the PINNED rows D1 retrieval.passed and D1 retrieval.max_results. Test fakes that take the doc-source call now accept max_results.

Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/dokimi/eval/answerer.py              | 5 ++---
 dokimi/tests/test_answering_stack_parity.py | 6 +++---
 dokimi/tests/test_eval_answerer.py          | 3 ++-
 dokimi/tests/test_provider_call_budgets.py  | 2 +-
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/dokimi/dokimi/eval/answerer.py b/dokimi/dokimi/eval/answerer.py
index 1dbdc0612..563b61659 100644
--- a/dokimi/dokimi/eval/answerer.py
+++ b/dokimi/dokimi/eval/answerer.py
@@ -315,12 +315,11 @@ def _retrieve_docs(
         }
         
         try:
-            # Retrieve top-k docs
-            # Note: Context7 client returns single doc, so we call it once for now
-            # TODO: Update Context7 client to support top_k natively
+            # Retrieve top-k docs, the same request the arms stack sends (max_results=top_k)
             raw_docs = self.mcp_client.get_library_docs(
                 library_id=library_id,
                 query=question,
+                max_results=self.top_k,
                 max_tokens=max_tokens
             )
             metadata["raw_count"] = len(raw_docs)
diff --git a/dokimi/tests/test_answering_stack_parity.py b/dokimi/tests/test_answering_stack_parity.py
index 0a2daca21..12c2d5a3a 100644
--- a/dokimi/tests/test_answering_stack_parity.py
+++ b/dokimi/tests/test_answering_stack_parity.py
@@ -61,9 +61,9 @@
     ("shared", "answer.system", None, None),
     ("shared", "answer.cache_blocks", False, False),
     # D1 retrieval request
-    ("D1", "retrieval.passed", ("max_tokens",), ("max_results",)),
-    # what each side asked for, not what came back: both get 4 docs; P keeps 3, A keeps 3
-    ("D1", "retrieval.max_results", 5, 3),
+    ("D1", "retrieval.passed", ("max_results", "max_tokens"), ("max_results",)),
+    # what each side asked for, not what came back: both get 4 docs; both ask for top_k=3
+    ("D1", "retrieval.max_results", 3, 3),
     ("D1", "retrieval.max_tokens", 4000, 8000),
     # D2 no docs found
     ("D2", "empty.answer_kinds", ("plain",), ("plain", "plain")),
diff --git a/dokimi/tests/test_eval_answerer.py b/dokimi/tests/test_eval_answerer.py
index 7a9b62568..5c8f247b1 100644
--- a/dokimi/tests/test_eval_answerer.py
+++ b/dokimi/tests/test_eval_answerer.py
@@ -197,7 +197,8 @@ def test_calls_mcp_client(self, mock_mcp_client):
             docs = answerer._retrieve_docs("uxlfoundation/oneTBB", "parallel_for", 4000)
 
             mock_mcp_client.get_library_docs.assert_called_once_with(
-                library_id="uxlfoundation/oneTBB", query="parallel_for", max_tokens=4000
+                library_id="uxlfoundation/oneTBB", query="parallel_for",
+                max_results=5, max_tokens=4000,
             )
             assert [d["content"] for d in docs] == [
                 mock_mcp_client.get_library_docs.return_value[0]["content"]
diff --git a/dokimi/tests/test_provider_call_budgets.py b/dokimi/tests/test_provider_call_budgets.py
index cbcd7627a..47229e1e5 100644
--- a/dokimi/tests/test_provider_call_budgets.py
+++ b/dokimi/tests/test_provider_call_budgets.py
@@ -328,7 +328,7 @@ def test_eval_panel_score_of_1_record_with_3_judges_makes_6_provider_calls(
 class _DocSource:
     """A doc-source client with one relevant chunk per query."""
 
-    def get_library_docs(self, library_id, query, max_tokens):
+    def get_library_docs(self, library_id, query, max_tokens, max_results=5):
         return [{"content": f"{query} Use parallel_for with a blocked_range.", "source": "s"}]
 
 

From 1c97ebc51e453a11c4337bd46e7b283b20578b78 Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Wed, 28 Oct 2026 16:24:14 -0700
Subject: [PATCH 02/53] fix(benchmark): benchmark run takes and passes
 --judge-temperature (#228 step 2)

benchmark run had no judge temperature flag, so its judge always ran at the provider default while evaluate and arms run took one. The flag is added (default unset, so runs without it are unchanged) and reaches the EvaluationPipeline judge.

Flips the PINNED row D6 cli.judge_temperature_flag. benchmark batch does not get the flag yet.

Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/dokimi/commands/benchmark.py          |  7 ++-
 dokimi/tests/test_answering_stack_p_step2.py | 53 ++++++++++++++++++++
 dokimi/tests/test_answering_stack_parity.py  |  4 +-
 3 files changed, 61 insertions(+), 3 deletions(-)
 create mode 100644 dokimi/tests/test_answering_stack_p_step2.py

diff --git a/dokimi/dokimi/commands/benchmark.py b/dokimi/dokimi/commands/benchmark.py
index 2c50c724e..ed58b50c7 100644
--- a/dokimi/dokimi/commands/benchmark.py
+++ b/dokimi/dokimi/commands/benchmark.py
@@ -27,7 +27,7 @@
 from dokimi.commands.library import _load_registry
 
 
-def _run_single_library(entry, output_dir: str, model: str, provider: str, judge_model: str, judge_provider: str = DEFAULT_JUDGE_PROVIDER, doc_source_override=None, max_tokens_per_question: int = DEFAULT_MAX_TOKENS_PER_QUESTION, force_regen: bool = False, concurrency: int = DEFAULT_CONCURRENCY, questions_from=None, questions_root=None) -> dict:
+def _run_single_library(entry, output_dir: str, model: str, provider: str, judge_model: str, judge_provider: str = DEFAULT_JUDGE_PROVIDER, doc_source_override=None, max_tokens_per_question: int = DEFAULT_MAX_TOKENS_PER_QUESTION, force_regen: bool = False, concurrency: int = DEFAULT_CONCURRENCY, questions_from=None, questions_root=None, judge_temperature=None) -> dict:
     """Run full evaluation pipeline for one ProductEntry. Returns result dict."""
     from dokimi.orchestrator import EvaluationPipeline
     from pathlib import Path as _Path
@@ -58,6 +58,7 @@ def _run_single_library(entry, output_dir: str, model: str, provider: str, judge
         provider=provider,
         judge_model=judge_model,
         judge_provider=judge_provider,
+        judge_temperature=judge_temperature,
         context7_id=entry.context7_id,
         doc_source=doc_source,
         max_tokens_per_question=max_tokens_per_question,
@@ -247,6 +248,7 @@ def cmd_benchmark_run(args: argparse.Namespace) -> None:
             concurrency=concurrency,
             questions_from=questions_from,
             questions_root=questions_root,
+            judge_temperature=getattr(args, "judge_temperature", None),
         )
     else:
         print(f"\n🔁 Multi-run mode: {n_runs} evaluation passes")
@@ -269,6 +271,7 @@ def cmd_benchmark_run(args: argparse.Namespace) -> None:
                 concurrency=concurrency,
                 questions_from=qfrom,
                 questions_root=questions_root,
+                judge_temperature=getattr(args, "judge_temperature", None),
             )
             try:
                 eval_summary = r["result"]["steps"]["evaluation"]["summary"]
@@ -369,6 +372,8 @@ def register(sub, positive_int) -> None:
                              help=JUDGE_MODEL_HELP)
     bench_run_p.add_argument("--judge-provider", default=DEFAULT_JUDGE_PROVIDER, dest="judge_provider",
                              choices=JUDGE_PROVIDERS)
+    bench_run_p.add_argument("--judge-temperature", type=float, default=None, dest="judge_temperature",
+                             help="Judge sampling temperature, recorded as judge_temperature in the eval output (default: provider default)")
     bench_run_p.add_argument("--registry", default=None, help="Path to custom products.yaml")
     bench_run_p.add_argument("--max-tokens", type=positive_int, default=DEFAULT_MAX_TOKENS_PER_QUESTION, dest="max_tokens",
                              help="Max tokens to retrieve per question from doc source (default: 4000)")
diff --git a/dokimi/tests/test_answering_stack_p_step2.py b/dokimi/tests/test_answering_stack_p_step2.py
new file mode 100644
index 000000000..00b0007c1
--- /dev/null
+++ b/dokimi/tests/test_answering_stack_p_step2.py
@@ -0,0 +1,53 @@
+# Copyright (C) 2026 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+"""Wiring checks for the Stack P fixes of #228 step 2.
+
+The parity table in ``test_answering_stack_parity.py`` pins what each fix changes
+in numbers. These tests cover the parts that table cannot see: the value a flag
+parses to, and where it lands.
+"""
+
+from types import SimpleNamespace
+
+from tests.test_answering_stack_parity import _subparser
+
+
+def test_benchmark_run_judge_temperature_reaches_the_pipeline(monkeypatch, tmp_path):
+    from dokimi.commands import benchmark
+    from dokimi import orchestrator
+
+    args = _subparser("benchmark", "run").parse_args(
+        ["--library", "onetbb", "--judge-temperature", "0.2"]
+    )
+    assert args.judge_temperature == 0.2
+
+    seen = {}
+
+    class FakePipeline:
+        def __init__(self, **kwargs):
+            seen.update(kwargs)
+
+        def run(self, concurrency):
+            return {"steps": {}}
+
+    monkeypatch.setattr(orchestrator, "EvaluationPipeline", FakePipeline)
+    entry = SimpleNamespace(
+        name="oneTBB", key="onetbb", repo=None, description="d",
+        doc_sources=["context7"], context7_id="uxlfoundation/oneTBB",
+    )
+    benchmark._run_single_library(
+        entry,
+        output_dir=str(tmp_path),
+        model="m",
+        provider="openai",
+        judge_model="jm",
+        judge_provider="openai",
+        judge_temperature=args.judge_temperature,
+    )
+    assert seen["judge_temperature"] == 0.2
+
+
+def test_benchmark_run_judge_temperature_defaults_to_provider_default():
+    args = _subparser("benchmark", "run").parse_args(["--library", "onetbb"])
+    assert args.judge_temperature is None
diff --git a/dokimi/tests/test_answering_stack_parity.py b/dokimi/tests/test_answering_stack_parity.py
index 12c2d5a3a..01b94a65a 100644
--- a/dokimi/tests/test_answering_stack_parity.py
+++ b/dokimi/tests/test_answering_stack_parity.py
@@ -102,8 +102,8 @@
         "(No documentation provided)",
         "(No documentation retrieved)",
     ),
-    # D6 judge temperature (CLI: P = benchmark run, A = arms run; benchmark run has no such flag)
-    ("D6", "cli.judge_temperature_flag", False, True),
+    # D6 judge temperature (CLI: P = benchmark run, A = arms run)
+    ("D6", "cli.judge_temperature_flag", True, True),
     # D7 --max-tokens (CLI defaults; where the configured value lands)
     ("D7", "cli.max_tokens_default", 4000, None),
     ("D7", "max_tokens.lands_in", "retrieval", "answer"),

From cbb73500779d411d9ffeed9c9ff199980ffd36f7 Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Wed, 28 Oct 2026 16:26:49 -0700
Subject: [PATCH 03/53] fix(pipeline): resolve the doc library id through the
 registry (#228)

EvaluationPipeline took the library id from --context7-id or the client's
own resolver, so a product whose products.yaml declares context7_id was
still looked up by its registry key. Resolve it with the shared
resolve_library_id_for_product helper, the same order the arms stack uses:
explicit id, then the registry's context7_id, then the client.

Question generation and answer generation both go through the helper.
The registry is injectable (registry=) and defaults to the bundled
products.yaml. Parity row D8 flips from client/resolved to registry/lib.

Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/dokimi/orchestrator/pipeline.py      | 22 +++++++++++++----
 dokimi/tests/answering_stack_parity.py      |  2 ++
 dokimi/tests/test_answering_stack_parity.py | 27 +++++++++++++--------
 3 files changed, 36 insertions(+), 15 deletions(-)

diff --git a/dokimi/dokimi/orchestrator/pipeline.py b/dokimi/dokimi/orchestrator/pipeline.py
index 5badfda32..b0bdc7fdc 100644
--- a/dokimi/dokimi/orchestrator/pipeline.py
+++ b/dokimi/dokimi/orchestrator/pipeline.py
@@ -10,7 +10,7 @@
 from pathlib import Path
 
 from dokimi.defaults import DEFAULT_ANSWER_MODEL, DEFAULT_ANSWER_PROVIDER, DEFAULT_JUDGE_MODEL, DEFAULT_JUDGE_PROVIDER, DEFAULT_RETRIEVAL_TOP_K
-from dokimi.mcp.factory import create_doc_source_client
+from dokimi.mcp.factory import create_doc_source_client, resolve_library_id_for_product
 from dokimi.records import BASELINE, CONTEXT, records as read_records
 
 logger = logging.getLogger(__name__)
@@ -95,6 +95,7 @@ def __init__(
         questions_from: Optional[Path] = None,
         product_key: Optional[str] = None,
         questions_root: Optional[Path] = None,
+        registry: Optional[Any] = None,
     ):
         """
         Initialize pipeline.
@@ -199,6 +200,8 @@ def __init__(
         self.debug_retrieval = debug_retrieval
         self.doc_source = doc_source
         self.context7_id = context7_id
+        # Product registry for the library id (defaults to the bundled products.yaml).
+        self.registry = registry
         self.max_tokens_per_question = max_tokens_per_question
         self.force_regen = force_regen
 
@@ -444,14 +447,23 @@ def _discover_personas(self) -> Dict[str, Any]:
         
         return personas
     
+    def _library_id(self, mcp_client) -> str:
+        """The doc-source library id: explicit id, then the registry's ``context7_id``, then the client.
+
+        The same order the arms stack uses (``commands/arms.py``, ``_resolve_doc_library_id``).
+        """
+        return resolve_library_id_for_product(
+            self.product_key, mcp_client, explicit_id=self.context7_id, registry=self.registry,
+        )
+
     def _generate_questions(self, personas: Dict[str, Any]) -> List[Dict[str, Any]]:
         """Generate questions from personas."""
         from dokimi.questions import RagasSeedExtractor, QuestionGenerator
 
         # Extract topics
         mcp_client = create_doc_source_client(self.doc_source)
-        library_id = self.context7_id or mcp_client.resolve_library_id(self.product)
-        
+        library_id = self._library_id(mcp_client)
+
         extractor = RagasSeedExtractor(mcp_client=mcp_client, cache_dir=Path(".cache/topics"))
         topics = extractor.extract_topics(
             library_id=library_id,
@@ -619,8 +631,8 @@ def _generate_answers(self, questions: List[Dict[str, Any]], concurrency: int =
 
         # Setup documentation source client
         mcp_client = create_doc_source_client(self.doc_source)
-        library_id = self.context7_id or mcp_client.resolve_library_id(self.product)
-        
+        library_id = self._library_id(mcp_client)
+
         # Generate answers
         answerer = Answerer(
             mcp_client=mcp_client,
diff --git a/dokimi/tests/answering_stack_parity.py b/dokimi/tests/answering_stack_parity.py
index 148596523..7d099c2d3 100644
--- a/dokimi/tests/answering_stack_parity.py
+++ b/dokimi/tests/answering_stack_parity.py
@@ -259,6 +259,7 @@ def run_stack_p(
     max_tokens: int = 4000,
     context7_id: Optional[str] = LIBRARY_ID,
     questions: Optional[List[Dict[str, Any]]] = None,
+    registry: Any = None,
 ) -> Observed:
     """Stack P end to end on the fixture: answer, then judge, through the real pipeline code."""
     from dokimi.orchestrator import pipeline as pipeline_mod
@@ -281,6 +282,7 @@ def run_stack_p(
         max_tokens_per_question=max_tokens,
         context7_id=context7_id,
         product_key=PRODUCT_KEY,
+        registry=registry,
     )
     qs = QUESTIONS if questions is None else questions
     mark = len(provider_calls.calls)
diff --git a/dokimi/tests/test_answering_stack_parity.py b/dokimi/tests/test_answering_stack_parity.py
index 01b94a65a..516d99bb5 100644
--- a/dokimi/tests/test_answering_stack_parity.py
+++ b/dokimi/tests/test_answering_stack_parity.py
@@ -109,7 +109,7 @@
     ("D7", "max_tokens.lands_in", "retrieval", "answer"),
     # D8 library id: EvaluationPipeline with no context7_id (the `orchestrate` path without
     # --context7-id). `benchmark run` passes the registry's context7_id, so it is not this row.
-    ("D8", "library_id", "client/resolved", "registry/lib"),
+    ("D8", "library_id", "registry/lib", "registry/lib"),
     # D9 statistics. The delta also moves with D3: P's q-rerank is judged with no docs
     # (delta 0), A's falls back to raw docs (delta 20), so a D3 fix flips this row too.
     ("D9", "stats.delta_mean", 13.33, 16.67),
@@ -253,15 +253,6 @@ def _where_4000_lands(obs: Observed) -> str:
 
 
 def _observe(tmp_path: Path, provider_calls) -> Dict[str, Tuple[Any, Any]]:
-    with pytest.MonkeyPatch.context() as mp:
-        p = run_stack_p(mp, tmp_path / "p", provider_calls)
-    with pytest.MonkeyPatch.context() as mp:
-        a = run_stack_a(mp, tmp_path / "a", provider_calls)
-    with pytest.MonkeyPatch.context() as mp:
-        p_noid = run_stack_p(
-            mp, tmp_path / "p-noid", provider_calls, context7_id=None, questions=QUESTIONS[:1]
-        )
-
     from dokimi.commands.arms import _resolve_doc_library_id
     from dokimi.registry import ProductRegistry
     from dokimi.treatments.arms import DocTreatment
@@ -272,6 +263,22 @@ def _observe(tmp_path: Path, provider_calls) -> Dict[str, Tuple[Any, Any]]:
         f"    description: fixture\n    context7_id: {REGISTRY_ID}\n",
         encoding="utf-8",
     )
+    registry = ProductRegistry(registry_path)
+
+    with pytest.MonkeyPatch.context() as mp:
+        p = run_stack_p(mp, tmp_path / "p", provider_calls)
+    with pytest.MonkeyPatch.context() as mp:
+        a = run_stack_a(mp, tmp_path / "a", provider_calls)
+    with pytest.MonkeyPatch.context() as mp:
+        p_noid = run_stack_p(
+            mp,
+            tmp_path / "p-noid",
+            provider_calls,
+            context7_id=None,
+            questions=QUESTIONS[:1],
+            registry=registry,
+        )
+
     a_lib = _resolve_doc_library_id(
         [DocTreatment(a.client, name="docs", top_k=3, reranker=a.reranker)],
         PRODUCT,

From b68b0ef3f6631c4a5923b823bab0ee0c8a29591a Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Wed, 28 Oct 2026 16:44:32 -0700
Subject: [PATCH 04/53] fix(judge): do not score the no-retrieval placeholder
 by default (#228)

When no document reached the prompt (no hit, or the reranker raised), the
answerer records "[FALLBACK: No relevant documentation found]" as the
with-docs answer, and the judge scored that text like a model answer. It
inflated the with-docs side of the paired delta on questions the docs did
not answer.

The judge now leaves a context answer whose doc_source is a no-retrieval
source (NO_RETRIEVAL_SOURCES) unjudged, so its with-docs score and delta
are None and the question drops out of the paired statistics. The record
and its retrieval_hit stay. Opt back in with judge_placeholder=True, or
--judge-placeholder on benchmark run, orchestrate evaluate and eval score.

The trust gate's retrieval_hit_rate read only the both-arms-scored
evaluations. Unjudged misses would then vanish from it and an all-miss run
would read as "not measured". It now counts every evaluation's
retrieval_hit. The thresholds are unchanged.

Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/dokimi/commands/benchmark.py          |  7 ++-
 dokimi/dokimi/commands/evaluate.py           |  3 ++
 dokimi/dokimi/commands/orchestrate.py        |  3 ++
 dokimi/dokimi/eval/judge.py                  | 16 ++++--
 dokimi/dokimi/gate/trust_gate.py             | 12 +++--
 dokimi/dokimi/orchestrator/pipeline.py       |  4 ++
 dokimi/tests/answering_stack_parity.py       |  2 +
 dokimi/tests/test_answering_stack_p_step2.py | 16 ++++++
 dokimi/tests/test_answering_stack_parity.py  | 57 +++++++++++++-------
 dokimi/tests/test_trust_gate.py              |  8 +--
 10 files changed, 97 insertions(+), 31 deletions(-)

diff --git a/dokimi/dokimi/commands/benchmark.py b/dokimi/dokimi/commands/benchmark.py
index ed58b50c7..43315db86 100644
--- a/dokimi/dokimi/commands/benchmark.py
+++ b/dokimi/dokimi/commands/benchmark.py
@@ -27,7 +27,7 @@
 from dokimi.commands.library import _load_registry
 
 
-def _run_single_library(entry, output_dir: str, model: str, provider: str, judge_model: str, judge_provider: str = DEFAULT_JUDGE_PROVIDER, doc_source_override=None, max_tokens_per_question: int = DEFAULT_MAX_TOKENS_PER_QUESTION, force_regen: bool = False, concurrency: int = DEFAULT_CONCURRENCY, questions_from=None, questions_root=None, judge_temperature=None) -> dict:
+def _run_single_library(entry, output_dir: str, model: str, provider: str, judge_model: str, judge_provider: str = DEFAULT_JUDGE_PROVIDER, doc_source_override=None, max_tokens_per_question: int = DEFAULT_MAX_TOKENS_PER_QUESTION, force_regen: bool = False, concurrency: int = DEFAULT_CONCURRENCY, questions_from=None, questions_root=None, judge_temperature=None, judge_placeholder=False) -> dict:
     """Run full evaluation pipeline for one ProductEntry. Returns result dict."""
     from dokimi.orchestrator import EvaluationPipeline
     from pathlib import Path as _Path
@@ -59,6 +59,7 @@ def _run_single_library(entry, output_dir: str, model: str, provider: str, judge
         judge_model=judge_model,
         judge_provider=judge_provider,
         judge_temperature=judge_temperature,
+        judge_placeholder=judge_placeholder,
         context7_id=entry.context7_id,
         doc_source=doc_source,
         max_tokens_per_question=max_tokens_per_question,
@@ -249,6 +250,7 @@ def cmd_benchmark_run(args: argparse.Namespace) -> None:
             questions_from=questions_from,
             questions_root=questions_root,
             judge_temperature=getattr(args, "judge_temperature", None),
+            judge_placeholder=getattr(args, "judge_placeholder", False),
         )
     else:
         print(f"\n🔁 Multi-run mode: {n_runs} evaluation passes")
@@ -272,6 +274,7 @@ def cmd_benchmark_run(args: argparse.Namespace) -> None:
                 questions_from=qfrom,
                 questions_root=questions_root,
                 judge_temperature=getattr(args, "judge_temperature", None),
+                judge_placeholder=getattr(args, "judge_placeholder", False),
             )
             try:
                 eval_summary = r["result"]["steps"]["evaluation"]["summary"]
@@ -374,6 +377,8 @@ def register(sub, positive_int) -> None:
                              choices=JUDGE_PROVIDERS)
     bench_run_p.add_argument("--judge-temperature", type=float, default=None, dest="judge_temperature",
                              help="Judge sampling temperature, recorded as judge_temperature in the eval output (default: provider default)")
+    bench_run_p.add_argument("--judge-placeholder", action="store_true", dest="judge_placeholder",
+                             help="Judge the no-retrieval placeholder answers too (default: they are not scored)")
     bench_run_p.add_argument("--registry", default=None, help="Path to custom products.yaml")
     bench_run_p.add_argument("--max-tokens", type=positive_int, default=DEFAULT_MAX_TOKENS_PER_QUESTION, dest="max_tokens",
                              help="Max tokens to retrieve per question from doc source (default: 4000)")
diff --git a/dokimi/dokimi/commands/evaluate.py b/dokimi/dokimi/commands/evaluate.py
index 9d3602f31..62bfeeb0d 100644
--- a/dokimi/dokimi/commands/evaluate.py
+++ b/dokimi/dokimi/commands/evaluate.py
@@ -203,6 +203,7 @@ def cmd_eval_score(args: argparse.Namespace) -> None:
         model=args.judge_model,
         provider=args.judge_provider,
         temperature=args.judge_temperature,
+        judge_placeholder=getattr(args, "judge_placeholder", False),
         run_metadata={
             "question_model": run.meta.get("question_model", "unknown"),
             "question_provider": run.meta.get("question_provider", "unknown"),
@@ -478,6 +479,8 @@ def register(sub, positive_int) -> None:
     score_p.add_argument("--concurrency", type=positive_int, default=DEFAULT_CONCURRENCY, help="Parallel judge calls (default: 5)")
     score_p.add_argument("--judge-temperature", type=float, default=None, dest="judge_temperature",
                          help="Judge sampling temperature, recorded as judge_temperature in the output")
+    score_p.add_argument("--judge-placeholder", action="store_true", dest="judge_placeholder",
+                         help="Judge the no-retrieval placeholder answers too (default: they are not scored)")
     score_p.set_defaults(func=cmd_eval_score)
 
     # eval ragas
diff --git a/dokimi/dokimi/commands/orchestrate.py b/dokimi/dokimi/commands/orchestrate.py
index f31b9bbac..d7cf40771 100644
--- a/dokimi/dokimi/commands/orchestrate.py
+++ b/dokimi/dokimi/commands/orchestrate.py
@@ -60,6 +60,7 @@ def cmd_evaluate(args: argparse.Namespace) -> None:
             judge_model=args.judge_model,
             judge_provider=args.judge_provider,
             judge_temperature=args.judge_temperature,
+            judge_placeholder=getattr(args, "judge_placeholder", False),
             personas_count=args.personas_count,
             questions_per_topic=args.questions_per_topic,
             # Only pass top_k when explicitly set; otherwise let pipeline use config default
@@ -122,6 +123,8 @@ def register(sub, positive_int) -> None:
     eval_p.add_argument("--judge-provider", default=DEFAULT_JUDGE_PROVIDER, choices=JUDGE_PROVIDERS)
     eval_p.add_argument("--judge-temperature", type=float, default=None, dest="judge_temperature",
                         help="Judge sampling temperature, recorded as judge_temperature in the eval output")
+    eval_p.add_argument("--judge-placeholder", action="store_true", dest="judge_placeholder",
+                        help="Judge the no-retrieval placeholder answers too (default: they are not scored)")
     eval_p.add_argument("--personas-count", type=positive_int, default=5, help="Target number of personas")
     eval_p.add_argument("--questions-per-topic", type=positive_int, default=2, help="Questions per topic per persona")
     eval_p.add_argument("--top-k", type=positive_int, default=None, help="Docs to retrieve before reranking (default from config/products.yaml retrieval.top_k)")
diff --git a/dokimi/dokimi/eval/judge.py b/dokimi/dokimi/eval/judge.py
index f07c9cc4d..457819683 100644
--- a/dokimi/dokimi/eval/judge.py
+++ b/dokimi/dokimi/eval/judge.py
@@ -21,7 +21,7 @@
 from dokimi.llm import ChatOpenAI, ChatAnthropic, LANGCHAIN_AVAILABLE
 from dokimi.eval.diagnoser import diagnose
 from dokimi.eval.question_text import require_question_texts
-from dokimi.records import BASELINE, CONTEXT, Record, RecordLike, records
+from dokimi.records import BASELINE, CONTEXT, NO_RETRIEVAL_SOURCES, Record, RecordLike, records
 from dokimi.defaults import DEFAULT_JUDGE_MODEL, DEFAULT_JUDGE_PROVIDER
 # One calibration scale for both the single judge and the panel, so their
 # numbers mean the same thing and can be compared.
@@ -195,6 +195,9 @@ class Judge:
     - Code Quality
     - Actionability
     """
+
+    #: Class default, so a Judge built without __init__ (tests build one with __new__) keeps the default.
+    judge_placeholder: bool = False
     
     def __init__(
         self,
@@ -205,6 +208,7 @@ def __init__(
         temperature: Optional[float] = None,
         seed: Optional[int] = None,
         prompt_template: Optional[str] = None,
+        judge_placeholder: bool = False,
     ):
         """
         Args:
@@ -218,6 +222,9 @@ def __init__(
             prompt_template: Replaces :data:`JUDGE_PROMPT` on the ungrounded
                 path. Exists so the judge can be measured under a rewording of
                 its own prompt; production scoring leaves it unset.
+            judge_placeholder: Judge a context answer that is the no-retrieval
+                placeholder ("[FALLBACK: ...]") too. Off by default: a question
+                with no document reaching the prompt is not scored as an answer.
         """
         if not LANGCHAIN_AVAILABLE:
             raise ImportError(
@@ -233,6 +240,7 @@ def __init__(
         self.prompt_sha256 = JUDGE_PROMPT_SHA256
         self.run_metadata = dict(run_metadata) if run_metadata else {}
         self.prompt_template = prompt_template
+        self.judge_placeholder = judge_placeholder
         # Accumulate judge-side token/cost telemetry across all judge calls
         # (Phase-C.0 covered answering; the judge path was untracked).
         self._usage_lock = threading.Lock()
@@ -371,9 +379,11 @@ def _evaluate_answer_pair(
         context_answer = answer.answer(CONTEXT)
         baseline_answer = answer.answer(BASELINE)
 
-        # Evaluate context arm
+        # Evaluate context arm. The no-retrieval placeholder is not an answer: it is
+        # left unjudged unless the caller opted in with judge_placeholder.
         with_docs_eval = None
-        if context_answer is not None and context_answer.text:
+        placeholder = context_answer is not None and context_answer.doc_source in NO_RETRIEVAL_SOURCES
+        if context_answer is not None and context_answer.text and (self.judge_placeholder or not placeholder):
             with_docs_eval = self._judge_answer(
                 library_name=library_name,
                 question=question_text,
diff --git a/dokimi/dokimi/gate/trust_gate.py b/dokimi/dokimi/gate/trust_gate.py
index fda12ff5f..316b8d72d 100644
--- a/dokimi/dokimi/gate/trust_gate.py
+++ b/dokimi/dokimi/gate/trust_gate.py
@@ -201,15 +201,19 @@ def evaluate_trust(
     # artifacts judged before that carry no retrieval outcome at all, and the
     # check reports "not measured" for them rather than a pass. So does a mix of
     # the two below ``min_retrieval_coverage``: the rate would describe a subset.
-    measured = [h for r in valid if (h := r.retrieval_hit) is not None]
-    if measured and len(measured) / n < cfg["min_retrieval_coverage"]:
+    # Over every evaluation, not only the both-arms-scored ones: a question with no
+    # document is not judged by default, and it still records that it missed.
+    every = records(evaluations, kind="evaluations")
+    measured = [h for r in every if (h := r.retrieval_hit) is not None]
+    n_all = len(every)
+    if measured and len(measured) / n_all < cfg["min_retrieval_coverage"]:
         checks.append(TrustCheck(
             name="retrieval_hit_rate",
             passed=True,
             value=None,
             threshold=cfg["min_retrieval_hit_rate"],
             message=(
-                f"Not measured: only {len(measured)} of {n} evaluations record a retrieval "
+                f"Not measured: only {len(measured)} of {n_all} evaluations record a retrieval "
                 f"outcome (retrieval_hit), under the {cfg['min_retrieval_coverage']:.0%} coverage "
                 f"floor; {sum(measured)} of those {len(measured)} hit"
             ),
@@ -219,7 +223,7 @@ def evaluate_trust(
     elif measured:
         retrieval_hits = sum(measured)
         hit_rate = retrieval_hits / len(measured)
-        coverage = "" if len(measured) == n else f"; {n - len(measured)} unrecorded"
+        coverage = "" if len(measured) == n_all else f"; {n_all - len(measured)} unrecorded"
         checks.append(TrustCheck(
             name="retrieval_hit_rate",
             passed=hit_rate >= cfg["min_retrieval_hit_rate"],
diff --git a/dokimi/dokimi/orchestrator/pipeline.py b/dokimi/dokimi/orchestrator/pipeline.py
index b0bdc7fdc..4def1d9c9 100644
--- a/dokimi/dokimi/orchestrator/pipeline.py
+++ b/dokimi/dokimi/orchestrator/pipeline.py
@@ -96,6 +96,7 @@ def __init__(
         product_key: Optional[str] = None,
         questions_root: Optional[Path] = None,
         registry: Optional[Any] = None,
+        judge_placeholder: bool = False,
     ):
         """
         Initialize pipeline.
@@ -192,6 +193,8 @@ def __init__(
         self.judge_model = judge_model
         self.judge_provider = judge_provider
         self.judge_temperature = judge_temperature
+        # Judge the no-retrieval placeholder answers too (off: they are not scored).
+        self.judge_placeholder = judge_placeholder
 
         self.personas_count = personas_count
         self.questions_per_topic = questions_per_topic
@@ -676,6 +679,7 @@ def _evaluate_answers(self, answers: List[Dict[str, Any]] | Dict[str, Any],
             model=self.judge_model,
             provider=self.judge_provider,
             temperature=self.judge_temperature,
+            judge_placeholder=self.judge_placeholder,
             run_metadata={
                 "question_model": self.model,
                 "question_provider": self.provider,
diff --git a/dokimi/tests/answering_stack_parity.py b/dokimi/tests/answering_stack_parity.py
index 7d099c2d3..da7af8793 100644
--- a/dokimi/tests/answering_stack_parity.py
+++ b/dokimi/tests/answering_stack_parity.py
@@ -260,6 +260,7 @@ def run_stack_p(
     context7_id: Optional[str] = LIBRARY_ID,
     questions: Optional[List[Dict[str, Any]]] = None,
     registry: Any = None,
+    judge_placeholder: bool = False,
 ) -> Observed:
     """Stack P end to end on the fixture: answer, then judge, through the real pipeline code."""
     from dokimi.orchestrator import pipeline as pipeline_mod
@@ -283,6 +284,7 @@ def run_stack_p(
         context7_id=context7_id,
         product_key=PRODUCT_KEY,
         registry=registry,
+        judge_placeholder=judge_placeholder,
     )
     qs = QUESTIONS if questions is None else questions
     mark = len(provider_calls.calls)
diff --git a/dokimi/tests/test_answering_stack_p_step2.py b/dokimi/tests/test_answering_stack_p_step2.py
index 00b0007c1..ef918ca9a 100644
--- a/dokimi/tests/test_answering_stack_p_step2.py
+++ b/dokimi/tests/test_answering_stack_p_step2.py
@@ -51,3 +51,19 @@ def run(self, concurrency):
 def test_benchmark_run_judge_temperature_defaults_to_provider_default():
     args = _subparser("benchmark", "run").parse_args(["--library", "onetbb"])
     assert args.judge_temperature is None
+
+
+def test_judge_placeholder_flag_is_off_by_default_and_parses_when_given():
+    run = _subparser("benchmark", "run")
+    assert run.parse_args(["--library", "onetbb"]).judge_placeholder is False
+    assert run.parse_args(["--library", "onetbb", "--judge-placeholder"]).judge_placeholder is True
+
+
+def test_judge_placeholder_flag_judges_the_placeholder_answers(monkeypatch, tmp_path, provider_calls):
+    from tests.answering_stack_parity import run_stack_p
+
+    # q-empty and q-rerank carry the placeholder as their with-docs answer: with the flag they
+    # are judged again, so P's judge calls go back to the 12 that Stack A makes.
+    p = run_stack_p(monkeypatch, tmp_path, provider_calls, judge_placeholder=True)
+    assert len(p.judge_inputs) == 12
+    assert len([e for e in p.evaluations if e["delta"] is not None]) == 6
diff --git a/dokimi/tests/test_answering_stack_parity.py b/dokimi/tests/test_answering_stack_parity.py
index 516d99bb5..dc962f3c6 100644
--- a/dokimi/tests/test_answering_stack_parity.py
+++ b/dokimi/tests/test_answering_stack_parity.py
@@ -19,7 +19,7 @@
 
 import argparse
 from pathlib import Path
-from typing import Any, Dict, Tuple
+from typing import Any, Dict, Optional, Tuple
 
 import pytest
 
@@ -55,7 +55,6 @@
 PINNED: Tuple[Tuple[str, str, Any, Any], ...] = (
     # shared: the two stacks agree today
     ("shared", "retrieval.count", 6, 6),
-    ("shared", "judge.calls", 12, 12),
     ("shared", "reranker.calls", 5, 5),
     ("shared", "answer.roles", ("user",), ("user",)),
     ("shared", "answer.system", None, None),
@@ -110,23 +109,27 @@
     # D8 library id: EvaluationPipeline with no context7_id (the `orchestrate` path without
     # --context7-id). `benchmark run` passes the registry's context7_id, so it is not this row.
     ("D8", "library_id", "registry/lib", "registry/lib"),
-    # D9 statistics. The delta also moves with D3: P's q-rerank is judged with no docs
-    # (delta 0), A's falls back to raw docs (delta 20), so a D3 fix flips this row too.
-    ("D9", "stats.delta_mean", 13.33, 16.67),
-    ("D9", "significance.n", 6, 6),
+    # D2 judge calls: P no longer judges the placeholder with-docs answers of q-empty and
+    # q-rerank (D2, D3), so 2 of the 12 calls are gone. A still judges all 12.
+    ("D2", "judge.calls", 10, 12),
+    # D9 statistics. P's paired deltas are q-docs, q-docs-2, q-docs-3, q-gt: 4 pairs, below
+    # MIN_PAIRS_FOR_TEST (6), so P has no significance test. A has 6 pairs.
+    ("D9", "stats.delta_mean", 20.0, 16.67),
+    ("D9", "stats.pairs", 4, 6),
+    ("D9", "significance.n", None, 6),
     ("D9", "significance.correction", None, "holm"),
-    ("D9", "significance.significant", False, False),
+    ("D9", "significance.significant", None, False),
     # what the judge sees per case: no docs (D2), reranker raises (D3), ground-truth chunk (D5)
     (
         "D2",
         "judge.contexts.q-empty",
-        ("(No documentation provided)", "(No documentation retrieved)"),
+        ("(No documentation provided)",),
         ("(No documentation retrieved)",),
     ),
     (
         "D3",
         "judge.contexts.q-rerank",
-        ("(No documentation provided)", "(No documentation retrieved)"),
+        ("(No documentation provided)",),
         ("(No documentation retrieved)", ("docs", "Passage", 3, True)),
     ),
     (
@@ -139,7 +142,8 @@
     ("shared", "judge.ground_truth.q-gt", (GROUND_TRUTH_CHUNK,), (GROUND_TRUTH_CHUNK,)),
     ("shared", "judge.ground_truth.others", (None,), (None,)),
     # the per-question deltas that feed D9's statistics (P: stored "delta"; A: "docs" vs "baseline").
-    # q-rerank is the one that moves with D3: P judges it with no docs (0), A with the raw docs (20).
+    # q-empty and q-rerank have no P delta (no with-docs judgement, D2/D3); A's q-empty is 0 and
+    # its q-rerank falls back to the raw docs (20).
     (
         "D9",
         "delta.per_question",
@@ -147,9 +151,9 @@
             ("q-docs", 20.0),
             ("q-docs-2", 20.0),
             ("q-docs-3", 20.0),
-            ("q-empty", 0.0),
+            ("q-empty", None),
             ("q-gt", 20.0),
-            ("q-rerank", 0.0),
+            ("q-rerank", None),
         ),
         (
             ("q-docs", 20.0),
@@ -310,6 +314,7 @@ def sampling(pairs, field):
 
     sig_p = _significance_p(p)
     sig_a = a.artifact["summary"]["per_arm"]["docs"]["significance"]
+    paired_p = _paired_deltas(p)
 
     return {
         "retrieval.count": (len(p.client.calls), len(a.client.calls)),
@@ -318,7 +323,8 @@ def sampling(pairs, field):
         "answer.roles": (sampling(p_all, "roles"), sampling(a_all, "roles")),
         "answer.system": (sampling(p_all, "system"), sampling(a_all, "system")),
         "answer.cache_blocks": (sampling(p_all, "cache_blocks"), sampling(a_all, "cache_blocks")),
-        "stats.delta_mean": (sig_p["delta_mean"], sig_a["delta_mean"]),
+        "stats.delta_mean": (round(sum(paired_p) / len(paired_p), 2), sig_a["delta_mean"]),
+        "stats.pairs": (len(paired_p), sig_a["n"]),
         "retrieval.passed": (
             distinct([c["passed"] for c in p.client.calls]),
             distinct([c["passed"] for c in a.client.calls]),
@@ -372,9 +378,16 @@ def sampling(pairs, field):
         ),
         "max_tokens.lands_in": (_where_4000_lands(p), _where_4000_lands(a)),
         "library_id": (p_noid.client.calls[0]["library_id"], a_lib),
-        "significance.n": (sig_p["n"], sig_a["n"]),
-        "significance.correction": (sig_p.get("correction"), sig_a.get("correction")),
-        "significance.significant": (sig_p["significant"], sig_a["significant"]),
+        # None: the paired test does not run below MIN_PAIRS_FOR_TEST pairs (P has 4 here)
+        "significance.n": (sig_p["n"] if sig_p else None, sig_a["n"]),
+        "significance.correction": (
+            sig_p.get("correction") if sig_p else None,
+            sig_a.get("correction"),
+        ),
+        "significance.significant": (
+            sig_p["significant"] if sig_p else None,
+            sig_a["significant"],
+        ),
         "record.shape": (_record_shape(p_rec_empty), _record_shape(a_rec_empty)),
         "answer.token_usage.context_chars": (
             "context_chars" in docs_block(p, p_rec_docs)["token_usage"],
@@ -410,12 +423,16 @@ def sampling(pairs, field):
     }
 
 
-def _significance_p(obs: Observed) -> Dict[str, Any]:
+def _significance_p(obs: Observed) -> Optional[Dict[str, Any]]:
+    """Stack P's paired test; None when it does not run (fewer pairs than MIN_PAIRS_FOR_TEST)."""
     from dokimi.report.eval_report import significance_test
 
-    result = significance_test(obs.evaluations)
-    assert result is not None, "P paired test did not run on the fixture"
-    return result
+    return significance_test(obs.evaluations)
+
+
+def _paired_deltas(obs: Observed) -> List[float]:
+    """The deltas of the questions both arms were judged on."""
+    return [e["delta"] for e in obs.evaluations if e.get("delta") is not None]
 
 
 def _record_shape(rec: Dict[str, Any]) -> str:
diff --git a/dokimi/tests/test_trust_gate.py b/dokimi/tests/test_trust_gate.py
index 8bc9a3f1b..174fb2e63 100644
--- a/dokimi/tests/test_trust_gate.py
+++ b/dokimi/tests/test_trust_gate.py
@@ -182,11 +182,13 @@ def _judged(n_hit, n_miss):
         ]
 
     def test_fails_when_retrieval_never_returned_a_doc(self):
-        evals = self._judged(n_hit=0, n_miss=12)
-        assert all(e["with_docs"]["aggregate"] > 0 for e in evals)
+        evals = self._judged(n_hit=2, n_miss=12)
+        # The placeholder is not judged by default: only the two hits carry a with-docs score,
+        # and the twelve misses still count against the rate.
+        assert sum(e["with_docs"] is None for e in evals) == 12
         check = next(c for c in evaluate_trust(evals).checks if c.name == "retrieval_hit_rate")
         assert not check.passed, check.message
-        assert check.value == 0.0
+        assert check.value == round(2 / 14, 3)
         assert check.measured
 
     def test_rate_counts_recorded_hits(self):

From 5469f7ae75db43387f42fc91a1ea5140b6a526be Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Wed, 28 Oct 2026 16:46:39 -0700
Subject: [PATCH 05/53] refactor(answerer): delete dead code in the context arm
 (#228)

Answerer built a self.llm client in __init__ that nothing read: the answer
call goes through llm_call_with_usage, which picks the provider's client
itself. Only the unsupported-provider ValueError is kept; the client
construction and its imports are gone.

_generate_with_docs had an "if not docs" branch that could not run, because
_generate_answer_pair calls it only when retrieval returned documents. The
branch is deleted with its unreachable "[No documentation retrieved]" record.

No parity row changes: nothing observable on either stack moved.

Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/dokimi/eval/answerer.py     | 34 +++++-------------------------
 dokimi/tests/test_eval_answerer.py | 22 +++----------------
 2 files changed, 8 insertions(+), 48 deletions(-)

diff --git a/dokimi/dokimi/eval/answerer.py b/dokimi/dokimi/eval/answerer.py
index 563b61659..3f876c573 100644
--- a/dokimi/dokimi/eval/answerer.py
+++ b/dokimi/dokimi/eval/answerer.py
@@ -15,8 +15,7 @@
 
 from dokimi.utils import question_id
 from dokimi.artifacts import save_artifact
-from dokimi.llm import ChatModel
-from dokimi.llm import llm_call_with_usage, ChatOpenAI, ChatAnthropic, LANGCHAIN_AVAILABLE
+from dokimi.llm import llm_call_with_usage, LANGCHAIN_AVAILABLE
 from dokimi.defaults import DEFAULT_ANSWER_MODEL, DEFAULT_ANSWER_PROVIDER
 from dokimi.metrics.usage import UsageRecord, to_record
 
@@ -95,15 +94,9 @@ def __init__(
         self.provider = provider
         self.api_key = api_key
 
-        self.llm: Optional[ChatModel]
-        if provider == "openai":
-            self.llm = ChatOpenAI(model=model, api_key=api_key)
-        elif provider == "anthropic":
-            self.llm = ChatAnthropic(model=model, api_key=api_key)
-        elif provider in ("google", "gemini", "google-vertex", "openrouter",
-                          "amazon-bedrock", "vertex_ai"):
-            self.llm = None  # llm_call_with_usage handles these natively
-        else:
+        # The answer call goes through llm_call_with_usage, which picks the provider's client itself.
+        if provider not in ("openai", "anthropic", "google", "gemini", "google-vertex",
+                            "openrouter", "amazon-bedrock", "vertex_ai"):
             raise ValueError(
                 f"Unsupported provider: '{provider}'. "
                 "Use 'openai', 'anthropic', 'google', 'openrouter', or 'amazon-bedrock'."
@@ -357,24 +350,7 @@ def _generate_with_docs(
         retrieval_metadata: Optional[Dict[str, Any]] = None,
         verbatim: bool = False,
     ) -> Dict[str, Any]:
-        """Generate context-arm answer."""
-        if not docs:
-            # No LLM call is made — emit a zero-cost/zero-token metrics block so
-            # this row carries the same telemetry shape as answered rows.
-            empty = UsageRecord(model=self.model, provider=self.provider,
-                                cost_usd=0.0, n_calls=0)
-            token_usage = empty.as_token_usage_dict()
-            token_usage["context_chars"] = 0
-            return {
-                "answer": "[No documentation retrieved]",
-                "retrieved_docs": [],
-                "model": self.model,
-                "doc_source": "none",
-                "retrieval_metadata": retrieval_metadata if self.debug_retrieval else None,
-                "token_usage": token_usage,
-                "metrics": empty.as_metrics_dict(answer_chars=0),
-            }
-
+        """Generate context-arm answer. Called only with a non-empty *docs* (see ``_generate_answer_pair``)."""
         # ~4k tokens at ~4 chars/token; keeps prompt within most models' context window
         MAX_DOC_CHARS = 15_000
 
diff --git a/dokimi/tests/test_eval_answerer.py b/dokimi/tests/test_eval_answerer.py
index 5c8f247b1..a36a853e0 100644
--- a/dokimi/tests/test_eval_answerer.py
+++ b/dokimi/tests/test_eval_answerer.py
@@ -75,23 +75,16 @@ def mock_llm_call():
 
 class TestAnswererInit:
     def test_init_openai(self):
-        mock_cls = Mock()
-        with patch('dokimi.eval.answerer.ChatOpenAI', mock_cls, create=True), \
-             patch('dokimi.eval.answerer.LANGCHAIN_AVAILABLE', True):
+        with patch('dokimi.eval.answerer.LANGCHAIN_AVAILABLE', True):
             answerer = Answerer(model="gpt-4o", provider="openai")
             assert answerer.model == "gpt-4o"
             assert answerer.provider == "openai"
-            mock_cls.assert_called_once()
 
     def test_init_anthropic(self):
-        mock_cls = Mock()
-        with patch('dokimi.eval.answerer.ChatAnthropic', mock_cls, create=True), \
-             patch('dokimi.eval.answerer.LANGCHAIN_AVAILABLE', True):
+        with patch('dokimi.eval.answerer.LANGCHAIN_AVAILABLE', True):
             answerer = Answerer(model="claude-opus", provider="anthropic", api_key="test")
-            # The anthropic client is the one built, with the caller's model and key.
-            mock_cls.assert_called_once_with(model="claude-opus", api_key="test")
-            assert answerer.llm is mock_cls.return_value
             assert answerer.provider == "anthropic"
+            assert answerer.api_key == "test"
 
     def test_init_unsupported_provider(self):
         mock_cls = Mock()
@@ -229,15 +222,6 @@ def test_with_docs(self, mock_llm_call):
             assert result["doc_source"] == "context7"
             assert result["answer"] == "This is a test answer from LLM."
 
-    def test_empty_docs(self):
-        with patch('dokimi.eval.answerer.LANGCHAIN_AVAILABLE', True):
-            answerer = Answerer()
-            result = answerer._generate_with_docs("Test question?", [])
-
-            assert result["answer"] == "[No documentation retrieved]"
-            assert result["retrieved_docs"] == []
-            assert "token_usage" in result
-
 
 class TestGenerateWithoutDocs:
     def test_generates_answer(self, mock_llm_call):

From f4c2ade603c199bd65b65e57b1a2b2be713e609f Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Wed, 28 Oct 2026 16:49:47 -0700
Subject: [PATCH 06/53] feat(artifacts): stamp method_version in the answers
 and judge output (#228)

The Stack P answers artifact and the judge's evaluation output now carry
method_version, from defaults.ANSWERING_METHOD_VERSION (2). The steps
before this one are version 1 and have no stamp, so a later compare can
refuse to put runs of different versions side by side. compare does not
read the stamp yet.

answers.v1 and eval.v1 document the field. The arms artifact is not
stamped in this change; the new parity row records that.

Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 .../20261028_120000_napetrov_stack-p-step2.md | 29 +++++++++++++++++++
 dokimi/dokimi/defaults.py                     |  5 ++++
 dokimi/dokimi/eval/answerer.py                |  2 ++
 dokimi/dokimi/eval/judge.py                   |  3 +-
 dokimi/dokimi/schemas/answers.v1.json         |  4 +++
 dokimi/dokimi/schemas/eval.v1.json            |  4 +++
 dokimi/tests/test_answering_stack_p_step2.py  |  9 ++++++
 dokimi/tests/test_answering_stack_parity.py   |  3 ++
 8 files changed, 58 insertions(+), 1 deletion(-)
 create mode 100644 dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md

diff --git a/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md b/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md
new file mode 100644
index 000000000..5fc6d7893
--- /dev/null
+++ b/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md
@@ -0,0 +1,29 @@
+### Changed
+
+- **Stack P scores move (#228 step 2)** — `benchmark run` and `orchestrate evaluate` now
+  answer with the `--top-k` they were given (retrieval asked for `max_results` 5 and the
+  client default before), take the judge temperature from `--judge-temperature` (it used to
+  be dropped), and resolve the doc library id through `products.yaml` `context7_id` before the
+  client's own resolver. A question with no document (no hit, or the reranker raised) is no
+  longer judged on its `[FALLBACK: ...]` placeholder by default, so it drops out of the paired
+  statistics; the record and its `retrieval_hit` stay. Pass `--judge-placeholder` to judge the
+  placeholder again, which restores the old numbers. `eval score` takes the same flag.
+- **Trust gate** — `retrieval_hit_rate` counts every evaluation's recorded retrieval outcome,
+  not only the ones scored in both arms, so unjudged misses still count against the rate. The
+  thresholds are unchanged.
+- **Stamped artifacts** — the answers and eval artifacts carry `method_version` (2). Older
+  artifacts have none and read as version 1.
+
+### Fixed
+
+- **Stack P no longer ignores `top_k`** — `Answerer` passes `max_results=top_k` to the doc
+  source; the context arm's retrieval request is now the one the arms stack makes.
+- **`benchmark run --judge-temperature`** reaches the judge pipeline.
+- **Library id from the registry** — `EvaluationPipeline` resolves the library id through
+  `resolve_library_id_for_product` (explicit id, then `context7_id`, then the client). A
+  registered product without `context7_id` now fails for a client that needs a declared id.
+
+### Removed
+
+- Dead code in the answering path: the unused `Answerer.llm` client and the unreachable
+  "no documents" branch of `_generate_with_docs`.
diff --git a/dokimi/dokimi/defaults.py b/dokimi/dokimi/defaults.py
index 71056525c..05dd80122 100644
--- a/dokimi/dokimi/defaults.py
+++ b/dokimi/dokimi/defaults.py
@@ -90,6 +90,11 @@ class _JudgeModelForProvider(str):
 
 # Decision (2026-03-04): one fixed semantic retrieval config.
 DEFAULT_RETRIEVAL_TOP_K = 3
+
+# Version of the answering and judging method, stamped in the answers and eval artifacts.
+# Bump it when a change moves scores, so that compare can refuse runs of different versions.
+# An artifact without the stamp predates it (#228 step 2); treat it as version 1.
+ANSWERING_METHOD_VERSION = 2
 DEFAULT_MAX_TOKENS_PER_QUESTION = 4000
 DEFAULT_CONCURRENCY = 5
 DEFAULT_RESULTS_SUFFIX = "_final"
diff --git a/dokimi/dokimi/eval/answerer.py b/dokimi/dokimi/eval/answerer.py
index 3f876c573..74e6f0252 100644
--- a/dokimi/dokimi/eval/answerer.py
+++ b/dokimi/dokimi/eval/answerer.py
@@ -15,6 +15,7 @@
 
 from dokimi.utils import question_id
 from dokimi.artifacts import save_artifact
+from dokimi.defaults import ANSWERING_METHOD_VERSION
 from dokimi.llm import llm_call_with_usage, LANGCHAIN_AVAILABLE
 from dokimi.defaults import DEFAULT_ANSWER_MODEL, DEFAULT_ANSWER_PROVIDER
 from dokimi.metrics.usage import UsageRecord, to_record
@@ -433,6 +434,7 @@ def _build_output(self, answers: List[Dict[str, Any]]) -> Dict[str, Any]:
 
         return {
             "generated_at": self._get_timestamp(),
+            "method_version": ANSWERING_METHOD_VERSION,
             "model": self.model,
             "provider": self.provider,
             "total_questions": len(answers),
diff --git a/dokimi/dokimi/eval/judge.py b/dokimi/dokimi/eval/judge.py
index 457819683..3e387f52c 100644
--- a/dokimi/dokimi/eval/judge.py
+++ b/dokimi/dokimi/eval/judge.py
@@ -22,7 +22,7 @@
 from dokimi.eval.diagnoser import diagnose
 from dokimi.eval.question_text import require_question_texts
 from dokimi.records import BASELINE, CONTEXT, NO_RETRIEVAL_SOURCES, Record, RecordLike, records
-from dokimi.defaults import DEFAULT_JUDGE_MODEL, DEFAULT_JUDGE_PROVIDER
+from dokimi.defaults import ANSWERING_METHOD_VERSION, DEFAULT_JUDGE_MODEL, DEFAULT_JUDGE_PROVIDER
 # One calibration scale for both the single judge and the panel, so their
 # numbers mean the same thing and can be compared.
 from dokimi.eval.rubric import (
@@ -648,6 +648,7 @@ def _build_evaluation_output(self, evaluations: List[Dict[str, Any]]) -> Dict[st
         """Build the serialisable output structure for a set of evaluations."""
         output: Dict[str, Any] = {
             "evaluated_at": self._get_timestamp(),
+            "method_version": ANSWERING_METHOD_VERSION,
             "judge_model": self.model,
             "judge_provider": self.provider,
             "judge_temperature": self.temperature,
diff --git a/dokimi/dokimi/schemas/answers.v1.json b/dokimi/dokimi/schemas/answers.v1.json
index 48a6f34d8..fd9abc965 100644
--- a/dokimi/dokimi/schemas/answers.v1.json
+++ b/dokimi/dokimi/schemas/answers.v1.json
@@ -25,6 +25,10 @@
         "null"
       ]
     },
+    "method_version": {
+      "type": "integer",
+      "description": "Answering method version (defaults.ANSWERING_METHOD_VERSION); absent on artifacts from before it was stamped."
+    },
     "total_questions": {
       "type": "integer"
     },
diff --git a/dokimi/dokimi/schemas/eval.v1.json b/dokimi/dokimi/schemas/eval.v1.json
index 61ae91b8c..ebbb0f479 100644
--- a/dokimi/dokimi/schemas/eval.v1.json
+++ b/dokimi/dokimi/schemas/eval.v1.json
@@ -85,6 +85,10 @@
         }
       }
     },
+    "method_version": {
+      "type": "integer",
+      "description": "Answering method version (defaults.ANSWERING_METHOD_VERSION); absent on artifacts from before it was stamped."
+    },
     "judge_temperature": {
       "description": "Sampling temperature the judge ran at; null when left to the provider default."
     },
diff --git a/dokimi/tests/test_answering_stack_p_step2.py b/dokimi/tests/test_answering_stack_p_step2.py
index ef918ca9a..ac33ffe23 100644
--- a/dokimi/tests/test_answering_stack_p_step2.py
+++ b/dokimi/tests/test_answering_stack_p_step2.py
@@ -67,3 +67,12 @@ def test_judge_placeholder_flag_judges_the_placeholder_answers(monkeypatch, tmp_
     p = run_stack_p(monkeypatch, tmp_path, provider_calls, judge_placeholder=True)
     assert len(p.judge_inputs) == 12
     assert len([e for e in p.evaluations if e["delta"] is not None]) == 6
+
+
+def test_eval_output_is_stamped_with_the_method_version(monkeypatch):
+    from dokimi.defaults import ANSWERING_METHOD_VERSION
+    from dokimi.eval.judge import Judge
+
+    monkeypatch.setenv("OPENAI_API_KEY", "test-key-not-used")
+    out = Judge(model="gpt-x", provider="openai")._build_evaluation_output([])
+    assert out["method_version"] == ANSWERING_METHOD_VERSION == 2
diff --git a/dokimi/tests/test_answering_stack_parity.py b/dokimi/tests/test_answering_stack_parity.py
index dc962f3c6..1ccf5f772 100644
--- a/dokimi/tests/test_answering_stack_parity.py
+++ b/dokimi/tests/test_answering_stack_parity.py
@@ -168,6 +168,8 @@
     ("only", "record.shape", "with_docs/without_docs", "arms"),
     ("only", "answer.token_usage.context_chars", True, False),
     ("only", "artifact.cost_summary", False, True),
+    # #228 step 2 stamp: the answers artifact carries method_version; the arms artifact does not
+    ("only", "artifact.method_version", 2, None),
 )
 
 
@@ -394,6 +396,7 @@ def sampling(pairs, field):
             "context_chars" in docs_block(a, a_rec_docs)["token_usage"],
         ),
         "artifact.cost_summary": ("cost_summary" in p.artifact, "cost_summary" in a.artifact),
+        "artifact.method_version": (p.artifact.get("method_version"), a.artifact.get("method_version")),
         "judge.contexts.q-empty": (_context_kinds(p, EMPTY_QUERY), _context_kinds(a, EMPTY_QUERY)),
         "judge.contexts.q-rerank": (
             _context_kinds(p, RERANK_QUERY),

From 65f6995beb8e711ba022aa05a8ab9dce5119a6f9 Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Wed, 28 Oct 2026 16:56:25 -0700
Subject: [PATCH 07/53] test(parity): import List for the paired-delta helper
 (#228)

ruff F821 flagged the annotation on _paired_deltas. The module uses
`from __future__ import annotations`, so the missing name did not fail at
import time and the parity tests passed without it.

Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/tests/test_answering_stack_parity.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dokimi/tests/test_answering_stack_parity.py b/dokimi/tests/test_answering_stack_parity.py
index 1ccf5f772..744804a41 100644
--- a/dokimi/tests/test_answering_stack_parity.py
+++ b/dokimi/tests/test_answering_stack_parity.py
@@ -19,7 +19,7 @@
 
 import argparse
 from pathlib import Path
-from typing import Any, Dict, Optional, Tuple
+from typing import Any, Dict, List, Optional, Tuple
 
 import pytest
 

From 6a29f10584e7ed70f68b99bcdd4e605bce0aeaf1 Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Wed, 28 Oct 2026 16:56:25 -0700
Subject: [PATCH 08/53] test(answerer): one provider allow-list smoke test
 replaces the init set-then-read tests (#228)

With the llm construction removed from Answerer.__init__, test_init_openai and
test_init_anthropic only set attributes and read them back, which the test
quality audit flags (set_then_read). The new parametrized test checks what
init does now: each supported provider is accepted. It is marked as a
must-not-raise smoke test.

Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/tests/test_eval_answerer.py | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/dokimi/tests/test_eval_answerer.py b/dokimi/tests/test_eval_answerer.py
index a36a853e0..84d1c7c7d 100644
--- a/dokimi/tests/test_eval_answerer.py
+++ b/dokimi/tests/test_eval_answerer.py
@@ -74,17 +74,14 @@ def mock_llm_call():
 
 
 class TestAnswererInit:
-    def test_init_openai(self):
+    @pytest.mark.parametrize(
+        "provider",
+        ["openai", "anthropic", "google", "gemini", "google-vertex", "openrouter",
+         "amazon-bedrock", "vertex_ai"],
+    )
+    def test_init_accepts_supported_provider(self, provider):  # test-quality: ok must-not-raise smoke test of the provider allow-list
         with patch('dokimi.eval.answerer.LANGCHAIN_AVAILABLE', True):
-            answerer = Answerer(model="gpt-4o", provider="openai")
-            assert answerer.model == "gpt-4o"
-            assert answerer.provider == "openai"
-
-    def test_init_anthropic(self):
-        with patch('dokimi.eval.answerer.LANGCHAIN_AVAILABLE', True):
-            answerer = Answerer(model="claude-opus", provider="anthropic", api_key="test")
-            assert answerer.provider == "anthropic"
-            assert answerer.api_key == "test"
+            Answerer(model="m", provider=provider)
 
     def test_init_unsupported_provider(self):
         mock_cls = Mock()

From 8a775b2e7429c0879ca386dd95775f82714c45a6 Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Wed, 28 Oct 2026 17:08:18 -0700
Subject: [PATCH 09/53] test(answerer): pin the top_k value at the Answerer
 call site (#228)

test_calls_mcp_client builds the Answerer with the default top_k, so a literal
max_results=5 would pass it. The new test builds it with top_k=3 and asserts
the doc source is asked for 3. On intel/main the same probe sends no
max_results at all, so the test fails there.

Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/tests/test_eval_answerer.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/dokimi/tests/test_eval_answerer.py b/dokimi/tests/test_eval_answerer.py
index 84d1c7c7d..a514d8d2a 100644
--- a/dokimi/tests/test_eval_answerer.py
+++ b/dokimi/tests/test_eval_answerer.py
@@ -181,6 +181,16 @@ def test_token_usage_summary_in_output(self, sample_questions, mock_mcp_client,
 
 
 class TestRetrieveDocs:
+    def test_asks_the_doc_source_for_top_k_docs(self, mock_mcp_client):
+        with patch('dokimi.eval.answerer.LANGCHAIN_AVAILABLE', True):
+            answerer = Answerer(mcp_client=mock_mcp_client, top_k=3)
+            answerer._retrieve_docs("uxlfoundation/oneTBB", "parallel_for", 4000)
+
+            mock_mcp_client.get_library_docs.assert_called_once_with(
+                library_id="uxlfoundation/oneTBB", query="parallel_for",
+                max_results=3, max_tokens=4000,
+            )
+
     def test_calls_mcp_client(self, mock_mcp_client):
         with patch('dokimi.eval.answerer.LANGCHAIN_AVAILABLE', True):
             answerer = Answerer(mcp_client=mock_mcp_client)

From 0fe7452da2253267247e79dad0df85bff66c279e Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Wed, 28 Oct 2026 17:08:18 -0700
Subject: [PATCH 10/53] docs(gate): say which trust-gate checks run on
 both-arm-scored questions (#228)

The methodology list and the method_version comment described the trust gate
and the stamp more broadly than the code does. The score checks run over the
both-arm-scored questions, so a question with no document leaves them. Nothing
reads method_version yet; the comment now says so.

Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/docs/methodology.md | 10 +++++-----
 dokimi/dokimi/defaults.py  |  4 ++--
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/dokimi/docs/methodology.md b/dokimi/docs/methodology.md
index 952f3ca56..1d0385ea6 100644
--- a/dokimi/docs/methodology.md
+++ b/dokimi/docs/methodology.md
@@ -199,11 +199,11 @@ RAGAS meta-evaluation can be run after answer generation to add faithfulness and
 
 Every serious run should answer "Can we trust this run?" before interpreting deltas. The trust gate checks:
 
-- minimum evaluated question count,
-- zero-score fraction,
-- minimum `with_docs` average,
-- average delta,
-- score coefficient of variation,
+- minimum evaluated question count (both arms scored; a question with no document is not judged by default, so it is not counted here),
+- zero-score fraction (over the both-arm-scored questions only, so questions with no document leave it too),
+- minimum `with_docs` average (same subset),
+- average delta (same subset),
+- score coefficient of variation (same subset),
 - inter-rater agreement when panel data exists,
 - retrieval hit rate — the share of questions where at least one document reached the `with_docs` prompt, from the per-evaluation `retrieval_hit` the judge and panel record; eval files judged before that field existed show it as not measured, and so do files where fewer than 80% of evaluations record it (`min_retrieval_coverage`), since the rate would then describe a subset,
 - multi-run variance when `--multi-run N` is used.
diff --git a/dokimi/dokimi/defaults.py b/dokimi/dokimi/defaults.py
index 05dd80122..a7a6c85a5 100644
--- a/dokimi/dokimi/defaults.py
+++ b/dokimi/dokimi/defaults.py
@@ -92,8 +92,8 @@ class _JudgeModelForProvider(str):
 DEFAULT_RETRIEVAL_TOP_K = 3
 
 # Version of the answering and judging method, stamped in the answers and eval artifacts.
-# Bump it when a change moves scores, so that compare can refuse runs of different versions.
-# An artifact without the stamp predates it (#228 step 2); treat it as version 1.
+# Bump it when a change moves scores. Nothing reads it yet: a later compare is meant to refuse
+# runs of different versions. An artifact without the stamp predates it (#228 step 2); treat it as version 1.
 ANSWERING_METHOD_VERSION = 2
 DEFAULT_MAX_TOKENS_PER_QUESTION = 4000
 DEFAULT_CONCURRENCY = 5

From c517c72709a784ee09c1388700847a6d9c4131b9 Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Wed, 28 Oct 2026 17:23:18 -0700
Subject: [PATCH 11/53] fix(gate): count unjudged retrieval misses as unusable
 in zero_score_fraction (#228)

A question whose retrieval missed is no longer judged with docs, so it has
no context score. The zero-score check counted only scored questions, so a
run where most questions miss passed it: 4 misses of 14 gave 0.0 PASS, where
intel/main gave 0.286 FAIL (the misses were judged at zero there).

The check now counts a miss (no context score, a baseline score, and
retrieval_hit False) as unusable, in the numerator and the denominator. The
message says such rows had no document. Thresholds, min_questions and the
early return are unchanged. The new fail-severity retrieval check is not
added here.

Regression test: tests/test_trust_gate_unjudged_misses.py. It fails on the
parent commit (4 of 14 passes at 0.0) and passes with this change.

Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/dokimi/gate/trust_gate.py              | 19 ++++-
 .../tests/test_trust_gate_unjudged_misses.py  | 70 +++++++++++++++++++
 2 files changed, 86 insertions(+), 3 deletions(-)
 create mode 100644 dokimi/tests/test_trust_gate_unjudged_misses.py

diff --git a/dokimi/dokimi/gate/trust_gate.py b/dokimi/dokimi/gate/trust_gate.py
index 316b8d72d..c3e59409b 100644
--- a/dokimi/dokimi/gate/trust_gate.py
+++ b/dokimi/dokimi/gate/trust_gate.py
@@ -136,14 +136,27 @@ def evaluate_trust(
         return TrustVerdict(checks=checks)  # can't compute anything else
 
     # ── 2. Zero-score fraction ─────────────────────────────────────────────
-    zero_with = sum(1 for v in with_scores if v == 0)
-    zero_frac = zero_with / n
+    # A retrieval miss is not judged by default (the no-document placeholder is not a
+    # with-docs answer). It still counts here as unusable: the zero-score check
+    # has always counted those questions, judged at zero. Leaving them out would
+    # let a run with many misses pass this fail-severity check.
+    unjudged_misses = [
+        r for r in records(evaluations, kind="evaluations")
+        if r.score(CONTEXT) is None and r.score(BASELINE) is not None
+        and r.retrieval_hit is False
+    ]
+    zero_with = sum(1 for v in with_scores if v == 0) + len(unjudged_misses)
+    unusable_n = n + len(unjudged_misses)
+    zero_frac = zero_with / unusable_n
     checks.append(TrustCheck(
         name="zero_score_fraction",
         passed=zero_frac <= cfg["max_zero_fraction"],
         value=round(zero_frac, 3),
         threshold=cfg["max_zero_fraction"],
-        message=f"{zero_with}/{n} questions scored 0 in the context arm ({zero_frac:.1%})",
+        message=(
+            f"{zero_with}/{unusable_n} questions scored 0 or had no document in the context "
+            f"arm ({zero_frac:.1%})"
+        ),
         severity="fail",
     ))
 
diff --git a/dokimi/tests/test_trust_gate_unjudged_misses.py b/dokimi/tests/test_trust_gate_unjudged_misses.py
new file mode 100644
index 000000000..3fbfafced
--- /dev/null
+++ b/dokimi/tests/test_trust_gate_unjudged_misses.py
@@ -0,0 +1,70 @@
+# Copyright (C) 2026 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+"""The zero-score check counts unjudged retrieval misses as unusable.
+
+A question whose retrieval missed gets no with-docs judgement, so it has no
+context score. The zero-score check must still count it. Otherwise a run where
+most questions miss passes a fail-severity check it fails on intel/main.
+"""
+
+from unittest.mock import MagicMock
+
+from dokimi.eval.judge import Judge
+from dokimi.gate.trust_gate import evaluate_trust
+
+GOOD = MagicMock(content=(
+    '{"correctness": 60, "completeness": 60, "specificity": 60, '
+    '"code_quality": 60, "actionability": 60, "aggregate": 0}'
+))
+ZERO = MagicMock(content=(
+    '{"correctness": 0, "completeness": 0, "specificity": 0, '
+    '"code_quality": 0, "actionability": 0, "aggregate": 0}'
+))
+HIT = {
+    "answer": "use parallel_for",
+    "doc_source": "context7",
+    "retrieved_docs": [{"source": "context7", "snippet": "parallel_for ...",
+                        "relevance_score": 0.8}],
+}
+MISS = {
+    "answer": "[FALLBACK: No relevant documentation found]",
+    "doc_source": "fallback_none",
+    "retrieved_docs": [],
+}
+
+
+def _rows(n_hit, n_miss):
+    """Judge n_hit questions with a document and n_miss without, as the judge does."""
+    judge = Judge.__new__(Judge)
+    judge._accumulate_usage = lambda usage: None
+    judge.temperature = judge.seed = None
+    judge.llm = MagicMock()
+    judge.llm.invoke.side_effect = lambda msgs, *a, **k: ZERO if "FALLBACK" in str(msgs) else GOOD
+    rows = []
+    for i in range(n_hit + n_miss):
+        rows.append(judge._evaluate_answer_pair("L", {
+            "question_id": f"q{i}",
+            "question_text": "q?",
+            "with_docs": dict(HIT if i < n_hit else MISS),
+            "without_docs": {"answer": "baseline"},
+        }))
+    return rows
+
+
+def _zero_check(verdict):
+    (check,) = [c for c in verdict.checks if c.name == "zero_score_fraction"]
+    return check
+
+
+def test_four_misses_of_fourteen_fail_the_zero_check():
+    check = _zero_check(evaluate_trust(_rows(10, 4)))
+    assert check.severity == "fail"
+    assert check.value == 0.286
+    assert check.passed is False
+
+
+def test_all_hits_pass_the_zero_check():
+    check = _zero_check(evaluate_trust(_rows(14, 0)))
+    assert check.value == 0.0
+    assert check.passed is True

From 6bec8910704e07866d1f86747348f090823129ab Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Wed, 28 Oct 2026 17:30:19 -0700
Subject: [PATCH 12/53] docs(gate): zero-score bullet counts retrieval misses;
 parse evaluations once (#228)

The methodology bullet still said the zero-score fraction leaves out
questions with no document. It counts them now (c517c727). Update it.

Also parse the evaluations once in evaluate_trust. The input may be a
one-shot iterable, and the old code parsed it again for the retrieval check.

Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/docs/methodology.md       | 2 +-
 dokimi/dokimi/gate/trust_gate.py | 9 +++++----
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/dokimi/docs/methodology.md b/dokimi/docs/methodology.md
index 1d0385ea6..cc1e9785f 100644
--- a/dokimi/docs/methodology.md
+++ b/dokimi/docs/methodology.md
@@ -200,7 +200,7 @@ RAGAS meta-evaluation can be run after answer generation to add faithfulness and
 Every serious run should answer "Can we trust this run?" before interpreting deltas. The trust gate checks:
 
 - minimum evaluated question count (both arms scored; a question with no document is not judged by default, so it is not counted here),
-- zero-score fraction (over the both-arm-scored questions only, so questions with no document leave it too),
+- zero-score fraction (over the both-arm-scored questions, plus the questions whose retrieval missed: those count as unusable, scored 0 or no document),
 - minimum `with_docs` average (same subset),
 - average delta (same subset),
 - score coefficient of variation (same subset),
diff --git a/dokimi/dokimi/gate/trust_gate.py b/dokimi/dokimi/gate/trust_gate.py
index c3e59409b..e98618a68 100644
--- a/dokimi/dokimi/gate/trust_gate.py
+++ b/dokimi/dokimi/gate/trust_gate.py
@@ -114,9 +114,11 @@ def evaluate_trust(
     cfg = {**DEFAULTS, **(thresholds or {})}
     checks: List[TrustCheck] = []
 
-    # Every check is over the questions scored in both arms.
+    # Parsed once: ``evaluations`` may be a one-shot iterable.
+    every = records(evaluations, kind="evaluations")
+    # The score checks below are over the questions scored in both arms.
     valid = [
-        r for r in records(evaluations, kind="evaluations")
+        r for r in every
         if r.score(CONTEXT) is not None and r.score(BASELINE) is not None
     ]
     with_scores: List[float] = [r.score(CONTEXT) or 0 for r in valid]
@@ -141,7 +143,7 @@ def evaluate_trust(
     # has always counted those questions, judged at zero. Leaving them out would
     # let a run with many misses pass this fail-severity check.
     unjudged_misses = [
-        r for r in records(evaluations, kind="evaluations")
+        r for r in every
         if r.score(CONTEXT) is None and r.score(BASELINE) is not None
         and r.retrieval_hit is False
     ]
@@ -216,7 +218,6 @@ def evaluate_trust(
     # the two below ``min_retrieval_coverage``: the rate would describe a subset.
     # Over every evaluation, not only the both-arms-scored ones: a question with no
     # document is not judged by default, and it still records that it missed.
-    every = records(evaluations, kind="evaluations")
     measured = [h for r in every if (h := r.retrieval_hit) is not None]
     n_all = len(every)
     if measured and len(measured) / n_all < cfg["min_retrieval_coverage"]:

From 715b6e5026c5560f3eeb3c143cbc15c4c1df5c55 Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Thu, 29 Oct 2026 11:39:21 -0700
Subject: [PATCH 13/53] fix(judge): a judge-error row keeps the retrieval
 outcome of its answer

The retrieval coverage floor counted every judge-error row as an unrecorded
outcome, so a run with more than 20% judge errors lost the retrieval check.

Thread: PRRT_kwDOVBg7g86xgwoY
Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/dokimi/eval/judge.py                   |  5 ++-
 .../tests/test_trust_gate_unjudged_misses.py  | 32 +++++++++++++++++++
 2 files changed, 36 insertions(+), 1 deletion(-)

diff --git a/dokimi/dokimi/eval/judge.py b/dokimi/dokimi/eval/judge.py
index 3e387f52c..dfad323a0 100644
--- a/dokimi/dokimi/eval/judge.py
+++ b/dokimi/dokimi/eval/judge.py
@@ -328,7 +328,10 @@ def _process(idx: int, answer: Record) -> None:
                     "error": str(e),
                     "with_docs": None,
                     "without_docs": None,
-                    "delta": None
+                    "delta": None,
+                    # What retrieval returned does not depend on the judge: the trust
+                    # gate's coverage floor must not count this row as unrecorded.
+                    "retrieval_hit": answer.retrieval_hit,
                 }
             elapsed = time.time() - t0
             with lock:
diff --git a/dokimi/tests/test_trust_gate_unjudged_misses.py b/dokimi/tests/test_trust_gate_unjudged_misses.py
index 3fbfafced..c805aaf21 100644
--- a/dokimi/tests/test_trust_gate_unjudged_misses.py
+++ b/dokimi/tests/test_trust_gate_unjudged_misses.py
@@ -68,3 +68,35 @@ def test_all_hits_pass_the_zero_check():
     check = _zero_check(evaluate_trust(_rows(14, 0)))
     assert check.value == 0.0
     assert check.passed is True
+
+
+def _retrieval_check(verdict):
+    (check,) = [c for c in verdict.checks if c.name == "retrieval_hit_rate"]
+    return check
+
+
+def test_judge_error_rows_keep_the_retrieval_outcome_of_their_answer(monkeypatch):
+    """5 of 20 answers fail in the judge: the retrieval rate is still measured over all 20."""
+    monkeypatch.setenv("OPENAI_API_KEY", "test-key-not-used")
+    judge = Judge(model="gpt-x", provider="openai")
+    judge.llm = MagicMock()
+    judge.llm.invoke.side_effect = lambda msgs, *a, **k: GOOD
+    real = judge._evaluate_answer_pair
+
+    def flaky(library_name, item):
+        if int(item.question_id[1:]) >= 15:
+            raise RuntimeError("judge down")
+        return real(library_name, item)
+
+    judge._evaluate_answer_pair = flaky
+    answers = [
+        {"question_id": f"q{i}", "question_text": "q?",
+         "with_docs": dict(HIT), "without_docs": {"answer": "baseline"}}
+        for i in range(20)
+    ]
+    rows = judge.evaluate_answers("L", answers)
+    assert sum("error" in r for r in rows) == 5
+    check = _retrieval_check(evaluate_trust(rows))
+    assert check.value == 1.0
+    assert check.measured is not False
+    assert "(20/20)" in check.message

From 2f5cc956700e79f75e8f3156cc4c7f4eccdf5b8e Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Thu, 29 Oct 2026 11:40:06 -0700
Subject: [PATCH 14/53] fix(diagnoser): an unjudged retrieval miss is an empty
 retrieval

With the no-retrieval placeholder left unjudged its delta is None, and
diagnose() returned insufficient_data before it looked at the doc source.

Thread: PRRT_kwDOVBg7g86xgwog
Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/dokimi/eval/diagnoser.py                 | 9 +++++++--
 dokimi/tests/test_diagnoser.py                  | 8 ++++++++
 dokimi/tests/test_trust_gate_unjudged_misses.py | 7 +++++++
 3 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/dokimi/dokimi/eval/diagnoser.py b/dokimi/dokimi/eval/diagnoser.py
index 15e194e6d..4bad7a466 100644
--- a/dokimi/dokimi/eval/diagnoser.py
+++ b/dokimi/dokimi/eval/diagnoser.py
@@ -30,12 +30,13 @@
     added noise rather than signal.
 
 insufficient_data
-    Delta could not be computed (one or both answers missing).
+    Delta could not be computed (one or both answers missing or errored). A
+    retrieval miss that was left unjudged is ``empty_retrieval``, not this.
 """
 
 from typing import Any, Dict, Iterable, List, Mapping, Optional
 
-from dokimi.records import Answer, RecordLike, records
+from dokimi.records import NO_RETRIEVAL_SOURCES, Answer, RecordLike, records
 
 # Keyword-overlap score below which retrieved docs are considered off-topic
 LOW_RELEVANCE_CUTOFF = 0.3
@@ -95,6 +96,10 @@ def diagnose(
         - ``evidence``: dict of signals used to reach the diagnosis
     """
     if delta is None:
+        # A retrieval miss is left unjudged, so it has no delta; it is still an
+        # empty retrieval, not a missing or errored answer.
+        if context_answer is not None and not context_answer.error and context_answer.doc_source in NO_RETRIEVAL_SOURCES:
+            return _result(EMPTY_RETRIEVAL, {"doc_source": context_answer.doc_source, "retrieved_count": 0})
         return _result(INSUFFICIENT_DATA, {})
 
     if delta >= 0:
diff --git a/dokimi/tests/test_diagnoser.py b/dokimi/tests/test_diagnoser.py
index 5fabb7db2..e0d3362e5 100644
--- a/dokimi/tests/test_diagnoser.py
+++ b/dokimi/tests/test_diagnoser.py
@@ -53,6 +53,14 @@ def test_empty_retrieval_no_docs():
     assert result["label"] == EMPTY_RETRIEVAL
 
 
+def test_unjudged_retrieval_miss_is_an_empty_retrieval_not_insufficient_data():
+    """The judge leaves the no-document placeholder unjudged, so its delta is None."""
+    answer = _make_answer(doc_source="fallback_none", retrieved_docs=[])
+    result = diagnose(answer, None)
+    assert result["label"] == EMPTY_RETRIEVAL
+    assert result["evidence"]["doc_source"] == "fallback_none"
+
+
 def test_empty_retrieval_no_mcp_client():
     answer = _make_answer(doc_source="none", retrieved_docs=[])
     result = diagnose(answer, -2.0)
diff --git a/dokimi/tests/test_trust_gate_unjudged_misses.py b/dokimi/tests/test_trust_gate_unjudged_misses.py
index c805aaf21..4cb7118a3 100644
--- a/dokimi/tests/test_trust_gate_unjudged_misses.py
+++ b/dokimi/tests/test_trust_gate_unjudged_misses.py
@@ -10,6 +10,7 @@
 
 from unittest.mock import MagicMock
 
+from dokimi.eval.diagnoser import EMPTY_RETRIEVAL, INSUFFICIENT_DATA, summarise_diagnoses
 from dokimi.eval.judge import Judge
 from dokimi.gate.trust_gate import evaluate_trust
 
@@ -100,3 +101,9 @@ def flaky(library_name, item):
     assert check.value == 1.0
     assert check.measured is not False
     assert "(20/20)" in check.message
+
+
+def test_unjudged_misses_stay_empty_retrievals_in_the_failure_analysis():
+    summary = summarise_diagnoses(_rows(10, 4))
+    assert summary["counts"][EMPTY_RETRIEVAL] == 4
+    assert summary["counts"][INSUFFICIENT_DATA] == 0

From d0b383c2d2d245b974a5b0506453fe4193edef6f Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Thu, 29 Oct 2026 11:41:02 -0700
Subject: [PATCH 15/53] fix(pipeline): say how many questions the run summary
 left out

The summary averages the questions scored in both arms (paired_summary, #250),
so an unjudged retrieval miss is in none of the three numbers. The run result
now carries paired and unpaired, and the printed line names them.

Thread: PRRT_kwDOVBg7g86xgwon
Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/dokimi/orchestrator/pipeline.py |  8 +++++++-
 dokimi/tests/test_pipeline_summary.py  | 15 +++++++++++++++
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/dokimi/dokimi/orchestrator/pipeline.py b/dokimi/dokimi/orchestrator/pipeline.py
index ca9d1ebe6..a66e678b4 100644
--- a/dokimi/dokimi/orchestrator/pipeline.py
+++ b/dokimi/dokimi/orchestrator/pipeline.py
@@ -389,6 +389,10 @@ def run(self, concurrency: int = 5) -> Dict[str, Any]:
         
         # Summary stats over the complete pairs, the numbers the reports print for the same file.
         summary = paired_summary(evaluations)
+        # Questions scored in one arm only (a retrieval miss is left unjudged, a judge may fail)
+        # are in none of the averages: say how many, so the line is not read as covering all.
+        results["steps"]["evaluation"]["paired"] = summary.paired
+        results["steps"]["evaluation"]["unpaired"] = summary.unpaired
 
         if summary.paired:
             results["steps"]["evaluation"]["summary"] = {
@@ -399,7 +403,9 @@ def run(self, concurrency: int = 5) -> Dict[str, Any]:
             print(f"✓ Evaluated {len(evaluations)} answers: "
                   f"context={results['steps']['evaluation']['summary']['with_avg']}, "
                   f"baseline={results['steps']['evaluation']['summary']['without_avg']}, "
-                  f"delta={results['steps']['evaluation']['summary']['delta_avg']:+.1f}")
+                  f"delta={results['steps']['evaluation']['summary']['delta_avg']:+.1f}"
+                  + (f" (over the {summary.paired} scored in both arms; {summary.unpaired} left out)"
+                     if summary.unpaired else ""))
         else:
             print(f"✓ Evaluated {len(evaluations)} answers (summary unavailable)")
         
diff --git a/dokimi/tests/test_pipeline_summary.py b/dokimi/tests/test_pipeline_summary.py
index 35e802037..e7dcd57ec 100644
--- a/dokimi/tests/test_pipeline_summary.py
+++ b/dokimi/tests/test_pipeline_summary.py
@@ -70,3 +70,18 @@ def test_summary_is_left_out_when_no_question_was_scored(tmp_path, monkeypatch):
     result = _run(tmp_path, monkeypatch, [_evaluation("q1", None, None)])
 
     assert "summary" not in result["steps"]["evaluation"]
+
+
+def test_summary_says_how_many_questions_it_left_out(tmp_path, monkeypatch, capsys):
+    """3 of 10 questions are unjudged retrieval misses: the means are over 7, and the
+    run result and the printed line say so instead of reading as a result over all 10."""
+    evaluations = [_evaluation(f"q{i}", 80, 60) for i in range(7)]
+    evaluations += [_evaluation(f"m{i}", None, 60) for i in range(3)]
+
+    result = _run(tmp_path, monkeypatch, evaluations)
+
+    step = result["steps"]["evaluation"]
+    assert (step["paired"], step["unpaired"]) == (7, 3)
+    printed = capsys.readouterr().out
+    assert "Evaluated 10 answers: context=80.0, baseline=60.0, delta=+20.0" in printed
+    assert "over the 7 scored in both arms; 3 left out" in printed

From f54995036ad801d73e520dbd46a4ef6618f7b1ff Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Thu, 29 Oct 2026 11:42:02 -0700
Subject: [PATCH 16/53] fix(benchmark): resolve the library id through the
 --registry that was loaded

benchmark run and batch loaded the registry for the entry, but the pipeline
looked the key up in the bundled products.yaml. A custom entry without a
context7_id then reached Context7 as a bare key and retrieved nothing, and a
custom key that the bundled registry also has took the bundled id.

Thread: PRRT_kwDOVBg7g86xgwou
Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/dokimi/commands/benchmark.py           |  6 +-
 .../tests/test_benchmark_registry_wiring.py   | 84 +++++++++++++++++++
 2 files changed, 89 insertions(+), 1 deletion(-)
 create mode 100644 dokimi/tests/test_benchmark_registry_wiring.py

diff --git a/dokimi/dokimi/commands/benchmark.py b/dokimi/dokimi/commands/benchmark.py
index 43315db86..e74508c7d 100644
--- a/dokimi/dokimi/commands/benchmark.py
+++ b/dokimi/dokimi/commands/benchmark.py
@@ -27,7 +27,7 @@
 from dokimi.commands.library import _load_registry
 
 
-def _run_single_library(entry, output_dir: str, model: str, provider: str, judge_model: str, judge_provider: str = DEFAULT_JUDGE_PROVIDER, doc_source_override=None, max_tokens_per_question: int = DEFAULT_MAX_TOKENS_PER_QUESTION, force_regen: bool = False, concurrency: int = DEFAULT_CONCURRENCY, questions_from=None, questions_root=None, judge_temperature=None, judge_placeholder=False) -> dict:
+def _run_single_library(entry, output_dir: str, model: str, provider: str, judge_model: str, judge_provider: str = DEFAULT_JUDGE_PROVIDER, doc_source_override=None, max_tokens_per_question: int = DEFAULT_MAX_TOKENS_PER_QUESTION, force_regen: bool = False, concurrency: int = DEFAULT_CONCURRENCY, questions_from=None, questions_root=None, judge_temperature=None, judge_placeholder=False, registry=None) -> dict:
     """Run full evaluation pipeline for one ProductEntry. Returns result dict."""
     from dokimi.orchestrator import EvaluationPipeline
     from pathlib import Path as _Path
@@ -66,6 +66,7 @@ def _run_single_library(entry, output_dir: str, model: str, provider: str, judge
         force_regen=force_regen,
         questions_from=questions_from,
         product_key=entry.key,
+        registry=registry,
         questions_root=_Path(questions_root) if questions_root else None,
     )
     result = pipeline.run(concurrency=concurrency)
@@ -251,6 +252,7 @@ def cmd_benchmark_run(args: argparse.Namespace) -> None:
             questions_root=questions_root,
             judge_temperature=getattr(args, "judge_temperature", None),
             judge_placeholder=getattr(args, "judge_placeholder", False),
+            registry=registry,
         )
     else:
         print(f"\n🔁 Multi-run mode: {n_runs} evaluation passes")
@@ -275,6 +277,7 @@ def cmd_benchmark_run(args: argparse.Namespace) -> None:
                 questions_root=questions_root,
                 judge_temperature=getattr(args, "judge_temperature", None),
                 judge_placeholder=getattr(args, "judge_placeholder", False),
+                registry=registry,
             )
             try:
                 eval_summary = r["result"]["steps"]["evaluation"]["summary"]
@@ -329,6 +332,7 @@ def cmd_benchmark_batch(args: argparse.Namespace) -> None:
                 force_regen=getattr(args, "force_regen", False),
                 concurrency=getattr(args, "concurrency", DEFAULT_CONCURRENCY),
                 questions_root=getattr(args, "questions_root", None),
+                registry=registry,
             )
             results.append(r)
         except Exception as exc:
diff --git a/dokimi/tests/test_benchmark_registry_wiring.py b/dokimi/tests/test_benchmark_registry_wiring.py
new file mode 100644
index 000000000..e7fef9f52
--- /dev/null
+++ b/dokimi/tests/test_benchmark_registry_wiring.py
@@ -0,0 +1,84 @@
+# Copyright (C) 2026 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+"""``benchmark run --registry`` resolves the library id through that registry."""
+
+import pytest
+
+from dokimi import orchestrator
+from dokimi.commands import benchmark
+from dokimi.orchestrator.pipeline import EvaluationPipeline
+from tests.test_answering_stack_parity import _subparser
+
+CUSTOM = """\
+products:
+  my_lib:
+    name: My Lib
+    description: A library only this registry knows
+    doc_sources: [context7]
+  onetbb:
+    name: Custom TBB
+    description: A key that the bundled registry also has, with no id here
+    doc_sources: [context7]
+"""
+
+
+class _DeclaredIdClient:
+    """A doc source like Context7: a name that is not a declared id is not resolved."""
+
+    requires_declared_library_id = True
+
+    def resolve_library_id(self, name):
+        return name
+
+
+class _Spy:
+    """Stands in for EvaluationPipeline: keeps what ``benchmark run`` hands it."""
+
+    built: list = []
+
+    def __init__(self, **kwargs):
+        self.kwargs = kwargs
+        _Spy.built.append(kwargs)
+
+    def run(self, concurrency):
+        return {"steps": {}}
+
+
+@pytest.fixture
+def spy(monkeypatch):
+    _Spy.built = []
+    monkeypatch.setattr(orchestrator, "EvaluationPipeline", _Spy)
+    return _Spy
+
+
+def _run(tmp_path, library, *extra):
+    registry = tmp_path / "my.yaml"
+    registry.write_text(CUSTOM, encoding="utf-8")
+    args = _subparser("benchmark", "run").parse_args(
+        ["--library", library, "--registry", str(registry), "--output-dir", str(tmp_path / "out"), *extra]
+    )
+    benchmark.cmd_benchmark_run(args)
+
+
+@pytest.mark.parametrize("library", ["my_lib", "onetbb"])
+def test_a_custom_entry_without_an_id_is_not_looked_up_in_the_bundled_registry(spy, tmp_path, library):
+    _run(tmp_path, library)
+
+    (kwargs,) = spy.built
+    real = EvaluationPipeline(**kwargs)
+    with pytest.raises(ValueError, match="declares no 'context7_id'"):
+        real._library_id(_DeclaredIdClient())
+
+
+def test_multi_run_and_batch_hand_the_registry_over_too(spy, tmp_path):
+    _run(tmp_path, "my_lib", "--multi-run", "2")
+    registry = tmp_path / "my.yaml"
+    args = _subparser("benchmark", "batch").parse_args(
+        ["--libraries", "my_lib", "--registry", str(registry), "--output-dir", str(tmp_path / "batch")]
+    )
+    benchmark.cmd_benchmark_batch(args)
+
+    assert len(spy.built) == 3
+    for kwargs in spy.built:
+        assert "my_lib" in kwargs["registry"]

From 9e1410666bcc81a739e666685f988f0de6fa46d9 Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Thu, 29 Oct 2026 11:42:41 -0700
Subject: [PATCH 17/53] test(pipeline): pin the explicit id and the
 registry-key lookup of _library_id

Both mutants named in the review (explicit_id=None, product instead of
product_key) now fail a test.

Thread: PRRT_kwDOVBg7g86xiJYO
Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/tests/test_pipeline_library_id.py | 62 ++++++++++++++++++++++++
 1 file changed, 62 insertions(+)
 create mode 100644 dokimi/tests/test_pipeline_library_id.py

diff --git a/dokimi/tests/test_pipeline_library_id.py b/dokimi/tests/test_pipeline_library_id.py
new file mode 100644
index 000000000..6b1719866
--- /dev/null
+++ b/dokimi/tests/test_pipeline_library_id.py
@@ -0,0 +1,62 @@
+# Copyright (C) 2026 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+"""``EvaluationPipeline._library_id``: which doc-source library a run queries."""
+
+import pytest
+
+from dokimi.orchestrator.pipeline import EvaluationPipeline
+from dokimi.registry import ProductRegistry
+
+REGISTRY = """\
+products:
+  my_lib:
+    name: My Lib
+    description: The registry key differs from the display name
+    context7_id: org/declared
+    doc_sources: [context7]
+"""
+
+
+class _RecordingClient:
+    """Records what the pipeline asks it to resolve; a bare name resolves to a marker."""
+
+    requires_declared_library_id = True
+
+    def __init__(self):
+        self.asked = []
+
+    def resolve_library_id(self, name):
+        self.asked.append(name)
+        return f"client:{name}"
+
+
+@pytest.fixture
+def registry(tmp_path):
+    path = tmp_path / "products.yaml"
+    path.write_text(REGISTRY, encoding="utf-8")
+    return ProductRegistry(path=path)
+
+
+def _pipeline(tmp_path, registry, **kwargs):
+    return EvaluationPipeline(
+        product="My Lib", description="d", product_key="my_lib", output_dir=tmp_path / "run", registry=registry, **kwargs
+    )
+
+
+def test_an_explicit_id_wins_over_the_registry_entry(tmp_path, registry):
+    client = _RecordingClient()
+
+    library_id = _pipeline(tmp_path, registry, context7_id="org/explicit")._library_id(client)
+
+    assert library_id == "org/explicit"
+    assert client.asked == []
+
+
+def test_the_registry_key_not_the_display_name_finds_the_declared_id(tmp_path, registry):
+    client = _RecordingClient()
+
+    library_id = _pipeline(tmp_path, registry)._library_id(client)
+
+    assert library_id == "org/declared"
+    assert client.asked == []

From 0f0aaa465b476248d4d9d40deb946bd2e7f9ac48 Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Thu, 29 Oct 2026 11:43:15 -0700
Subject: [PATCH 18/53] fix(judge): stamp whether the placeholder was judged in
 the eval output

--judge-placeholder gives different numbers on the same answers under the same
method_version. The eval output now carries judge_placeholder (declared in
eval.v1), so two files of one version that are not comparable can be told apart.

Thread: PRRT_kwDOVBg7g86xgwoz
Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/dokimi/eval/judge.py                  |  1 +
 dokimi/dokimi/schemas/eval.v1.json           |  4 ++++
 dokimi/tests/test_answering_stack_p_step2.py | 14 ++++++++++++++
 3 files changed, 19 insertions(+)

diff --git a/dokimi/dokimi/eval/judge.py b/dokimi/dokimi/eval/judge.py
index dfad323a0..7dd52bf41 100644
--- a/dokimi/dokimi/eval/judge.py
+++ b/dokimi/dokimi/eval/judge.py
@@ -652,6 +652,7 @@ def _build_evaluation_output(self, evaluations: List[Dict[str, Any]]) -> Dict[st
         output: Dict[str, Any] = {
             "evaluated_at": self._get_timestamp(),
             "method_version": ANSWERING_METHOD_VERSION,
+            "judge_placeholder": self.judge_placeholder,
             "judge_model": self.model,
             "judge_provider": self.provider,
             "judge_temperature": self.temperature,
diff --git a/dokimi/dokimi/schemas/eval.v1.json b/dokimi/dokimi/schemas/eval.v1.json
index ebbb0f479..b3b609d79 100644
--- a/dokimi/dokimi/schemas/eval.v1.json
+++ b/dokimi/dokimi/schemas/eval.v1.json
@@ -89,6 +89,10 @@
       "type": "integer",
       "description": "Answering method version (defaults.ANSWERING_METHOD_VERSION); absent on artifacts from before it was stamped."
     },
+    "judge_placeholder": {
+      "type": "boolean",
+      "description": "True when the judge also scored a context answer that is the no-retrieval placeholder (--judge-placeholder), which the default leaves unjudged; absent on artifacts from before it was stamped. Two eval files with the same method_version but a different value here are not comparable."
+    },
     "judge_temperature": {
       "description": "Sampling temperature the judge ran at; null when left to the provider default."
     },
diff --git a/dokimi/tests/test_answering_stack_p_step2.py b/dokimi/tests/test_answering_stack_p_step2.py
index ac33ffe23..981e8fac2 100644
--- a/dokimi/tests/test_answering_stack_p_step2.py
+++ b/dokimi/tests/test_answering_stack_p_step2.py
@@ -76,3 +76,17 @@ def test_eval_output_is_stamped_with_the_method_version(monkeypatch):
     monkeypatch.setenv("OPENAI_API_KEY", "test-key-not-used")
     out = Judge(model="gpt-x", provider="openai")._build_evaluation_output([])
     assert out["method_version"] == ANSWERING_METHOD_VERSION == 2
+
+
+def test_eval_output_says_whether_the_placeholder_was_judged(monkeypatch):
+    """The two modes give different numbers on the same answers under one method_version."""
+    from tests.artifact_fixtures import artifact_fixture
+    from dokimi.eval.judge import Judge
+
+    monkeypatch.setenv("OPENAI_API_KEY", "test-key-not-used")
+    default = Judge(model="gpt-x", provider="openai")._build_evaluation_output([])
+    opted_in = Judge(model="gpt-x", provider="openai", judge_placeholder=True)._build_evaluation_output([])
+
+    assert default["judge_placeholder"] is False
+    assert opted_in["judge_placeholder"] is True
+    artifact_fixture("eval", opted_in)  # eval.v1 declares the key

From ea052d8056b45d727ba3667e8775159b23538ea6 Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Thu, 29 Oct 2026 11:43:56 -0700
Subject: [PATCH 19/53] fix(gate): show the retrieval rate when no question is
 scored in both arms

With every question an unjudged miss the gate returned after min_questions
with "0 questions evaluated", pointing away from the cause. The retrieval
check is now a helper and also runs on that path. The 0-hit case has its
test again, and the 2-hit test is named for what it checks.

Thread: PRRT_kwDOVBg7g86xgwo7
Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/dokimi/gate/trust_gate.py | 110 +++++++++++++++++--------------
 dokimi/tests/test_trust_gate.py  |  11 ++++
 2 files changed, 71 insertions(+), 50 deletions(-)

diff --git a/dokimi/dokimi/gate/trust_gate.py b/dokimi/dokimi/gate/trust_gate.py
index e98618a68..de23115ec 100644
--- a/dokimi/dokimi/gate/trust_gate.py
+++ b/dokimi/dokimi/gate/trust_gate.py
@@ -15,7 +15,7 @@
 from dataclasses import dataclass, field
 from typing import Any, Dict, Iterable, List, Optional
 
-from dokimi.records import BASELINE, CONTEXT, RecordLike, records
+from dokimi.records import BASELINE, CONTEXT, Record, RecordLike, records
 
 
 # ── Thresholds (can be overridden via spec/config) ────────────────────────────
@@ -94,6 +94,60 @@ def status(self) -> str:
         return "✅ PASS"
 
 
+def _retrieval_check(every: List[Record], cfg: Dict[str, Any]) -> TrustCheck:
+    """The retrieval hit rate over every evaluation, ``measured=False`` when it cannot be read.
+
+    A question "hit" retrieval when at least one document reached the
+    context-arm prompt. The judge stamps that per evaluation as
+    ``retrieval_hit`` (from the answer's retrieved_docs / doc_source); eval
+    artifacts judged before that carry no retrieval outcome at all, and the
+    check reports "not measured" for them rather than a pass. So does a mix of
+    the two below ``min_retrieval_coverage``: the rate would describe a subset.
+    Over every evaluation, not only the both-arms-scored ones: a question with no
+    document is not judged by default, and it still records that it missed.
+    """
+    measured = [h for r in every if (h := r.retrieval_hit) is not None]
+    n_all = len(every)
+    if measured and len(measured) / n_all < cfg["min_retrieval_coverage"]:
+        return TrustCheck(
+            name="retrieval_hit_rate",
+            passed=True,
+            value=None,
+            threshold=cfg["min_retrieval_hit_rate"],
+            message=(
+                f"Not measured: only {len(measured)} of {n_all} evaluations record a retrieval "
+                f"outcome (retrieval_hit), under the {cfg['min_retrieval_coverage']:.0%} coverage "
+                f"floor; {sum(measured)} of those {len(measured)} hit"
+            ),
+            severity="warn",
+            measured=False,
+        )
+    if measured:
+        retrieval_hits = sum(measured)
+        hit_rate = retrieval_hits / len(measured)
+        coverage = "" if len(measured) == n_all else f"; {n_all - len(measured)} unrecorded"
+        return TrustCheck(
+            name="retrieval_hit_rate",
+            passed=hit_rate >= cfg["min_retrieval_hit_rate"],
+            value=round(hit_rate, 3),
+            threshold=cfg["min_retrieval_hit_rate"],
+            message=(
+                f"Retrieval hit rate = {hit_rate:.1%} ({retrieval_hits}/{len(measured)}{coverage}) "
+                f"(min {cfg['min_retrieval_hit_rate']:.0%})"
+            ),
+            severity="warn",
+        )
+    return TrustCheck(
+        name="retrieval_hit_rate",
+        passed=True,
+        value=None,
+        threshold=cfg["min_retrieval_hit_rate"],
+        message="Not measured: the evaluations record no retrieval outcome (retrieval_hit)",
+        severity="warn",
+        measured=False,
+    )
+
+
 def evaluate_trust(
     evaluations: Iterable[RecordLike],
     thresholds: Optional[Dict[str, Any]] = None,
@@ -135,6 +189,10 @@ def evaluate_trust(
         severity="fail",
     ))
     if n == 0:
+        # Nothing is scored in both arms (every question an unjudged miss, say): the
+        # retrieval rate is still known, and it is the cause, so it goes beside the count.
+        if any(r.retrieval_hit is not None for r in every):
+            checks.append(_retrieval_check(every, cfg))
         return TrustVerdict(checks=checks)  # can't compute anything else
 
     # ── 2. Zero-score fraction ─────────────────────────────────────────────
@@ -210,55 +268,7 @@ def evaluate_trust(
         ))
 
     # ── 7. Retrieval hit rate ─────────────────────────────────────────────
-    # A question "hit" retrieval when at least one document reached the
-    # context-arm prompt. The judge stamps that per evaluation as
-    # ``retrieval_hit`` (from the answer's retrieved_docs / doc_source); eval
-    # artifacts judged before that carry no retrieval outcome at all, and the
-    # check reports "not measured" for them rather than a pass. So does a mix of
-    # the two below ``min_retrieval_coverage``: the rate would describe a subset.
-    # Over every evaluation, not only the both-arms-scored ones: a question with no
-    # document is not judged by default, and it still records that it missed.
-    measured = [h for r in every if (h := r.retrieval_hit) is not None]
-    n_all = len(every)
-    if measured and len(measured) / n_all < cfg["min_retrieval_coverage"]:
-        checks.append(TrustCheck(
-            name="retrieval_hit_rate",
-            passed=True,
-            value=None,
-            threshold=cfg["min_retrieval_hit_rate"],
-            message=(
-                f"Not measured: only {len(measured)} of {n_all} evaluations record a retrieval "
-                f"outcome (retrieval_hit), under the {cfg['min_retrieval_coverage']:.0%} coverage "
-                f"floor; {sum(measured)} of those {len(measured)} hit"
-            ),
-            severity="warn",
-            measured=False,
-        ))
-    elif measured:
-        retrieval_hits = sum(measured)
-        hit_rate = retrieval_hits / len(measured)
-        coverage = "" if len(measured) == n_all else f"; {n_all - len(measured)} unrecorded"
-        checks.append(TrustCheck(
-            name="retrieval_hit_rate",
-            passed=hit_rate >= cfg["min_retrieval_hit_rate"],
-            value=round(hit_rate, 3),
-            threshold=cfg["min_retrieval_hit_rate"],
-            message=(
-                f"Retrieval hit rate = {hit_rate:.1%} ({retrieval_hits}/{len(measured)}{coverage}) "
-                f"(min {cfg['min_retrieval_hit_rate']:.0%})"
-            ),
-            severity="warn",
-        ))
-    else:
-        checks.append(TrustCheck(
-            name="retrieval_hit_rate",
-            passed=True,
-            value=None,
-            threshold=cfg["min_retrieval_hit_rate"],
-            message="Not measured: the evaluations record no retrieval outcome (retrieval_hit)",
-            severity="warn",
-            measured=False,
-        ))
+    checks.append(_retrieval_check(every, cfg))
 
     # ── 8. Multi-run variance (optional) ─────────────────────────────────
     multirun_std = None
diff --git a/dokimi/tests/test_trust_gate.py b/dokimi/tests/test_trust_gate.py
index 174fb2e63..dbcf6f81e 100644
--- a/dokimi/tests/test_trust_gate.py
+++ b/dokimi/tests/test_trust_gate.py
@@ -182,6 +182,17 @@ def _judged(n_hit, n_miss):
         ]
 
     def test_fails_when_retrieval_never_returned_a_doc(self):
+        evals = self._judged(n_hit=0, n_miss=14)
+        # Every question is an unjudged miss: nothing is scored in both arms, the run is not
+        # trusted, and the message names the cause (0 of 14 hit) beside the question count.
+        verdict = evaluate_trust(evals)
+        assert not verdict.is_trusted
+        assert [c.name for c in verdict.failures] == ["min_questions"]
+        check = next(c for c in verdict.checks if c.name == "retrieval_hit_rate")
+        assert not check.passed and check.measured and check.value == 0.0
+        assert "(0/14)" in check.message
+
+    def test_fails_when_retrieval_rarely_returned_a_doc(self):
         evals = self._judged(n_hit=2, n_miss=12)
         # The placeholder is not judged by default: only the two hits carry a with-docs score,
         # and the twelve misses still count against the rate.

From 6bda45662f39a021bd6f94b71440d04bb4bd405b Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Thu, 29 Oct 2026 11:44:31 -0700
Subject: [PATCH 20/53] test(gate): pin the three conditions of the
 unjudged-miss term

A legacy row, a hit that was not scored and a miss without a baseline must
leave zero_score_fraction unchanged; each mutant of the filter named in the
reviews now fails. The judge stub branch that only a judged miss reached is
gone: the default leaves misses unjudged.

Thread: PRRT_kwDOVBg7g86xiJYM
Thread: PRRT_kwDOVBg7g86xlsnp
Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 .../tests/test_trust_gate_unjudged_misses.py  | 37 ++++++++++++++++---
 1 file changed, 32 insertions(+), 5 deletions(-)

diff --git a/dokimi/tests/test_trust_gate_unjudged_misses.py b/dokimi/tests/test_trust_gate_unjudged_misses.py
index 4cb7118a3..95911c13b 100644
--- a/dokimi/tests/test_trust_gate_unjudged_misses.py
+++ b/dokimi/tests/test_trust_gate_unjudged_misses.py
@@ -10,6 +10,8 @@
 
 from unittest.mock import MagicMock
 
+import pytest
+
 from dokimi.eval.diagnoser import EMPTY_RETRIEVAL, INSUFFICIENT_DATA, summarise_diagnoses
 from dokimi.eval.judge import Judge
 from dokimi.gate.trust_gate import evaluate_trust
@@ -18,10 +20,6 @@
     '{"correctness": 60, "completeness": 60, "specificity": 60, '
     '"code_quality": 60, "actionability": 60, "aggregate": 0}'
 ))
-ZERO = MagicMock(content=(
-    '{"correctness": 0, "completeness": 0, "specificity": 0, '
-    '"code_quality": 0, "actionability": 0, "aggregate": 0}'
-))
 HIT = {
     "answer": "use parallel_for",
     "doc_source": "context7",
@@ -41,7 +39,7 @@ def _rows(n_hit, n_miss):
     judge._accumulate_usage = lambda usage: None
     judge.temperature = judge.seed = None
     judge.llm = MagicMock()
-    judge.llm.invoke.side_effect = lambda msgs, *a, **k: ZERO if "FALLBACK" in str(msgs) else GOOD
+    judge.llm.invoke.return_value = GOOD
     rows = []
     for i in range(n_hit + n_miss):
         rows.append(judge._evaluate_answer_pair("L", {
@@ -107,3 +105,32 @@ def test_unjudged_misses_stay_empty_retrievals_in_the_failure_analysis():
     summary = summarise_diagnoses(_rows(10, 4))
     assert summary["counts"][EMPTY_RETRIEVAL] == 4
     assert summary["counts"][INSUFFICIENT_DATA] == 0
+
+
+def _row(question_id, context, baseline, retrieval_hit):
+    """An eval record as the judge writes it: a null condition is one that was not scored."""
+    return {
+        "question_id": question_id,
+        "question_text": "q?",
+        "with_docs": None if context is None else {"aggregate": context},
+        "without_docs": None if baseline is None else {"aggregate": baseline},
+        "delta": None if context is None or baseline is None else context - baseline,
+        "retrieval_hit": retrieval_hit,
+    }
+
+
+@pytest.mark.parametrize("extra", [
+    # no context score, and no retrieval outcome recorded: an older artifact, or a judge error
+    # on an answer whose outcome was never stamped
+    _row("legacy", None, 60, None),
+    # a document reached the prompt, but the context answer was not scored (judge error)
+    _row("judge-error", None, 60, True),
+    # a miss with no baseline score: it is not a question both arms were meant to score
+    _row("no-baseline", None, None, False),
+], ids=["legacy-row", "hit-not-scored", "miss-without-baseline"])
+def test_only_a_recorded_miss_with_a_baseline_counts_as_unusable(extra):
+    rows = [_row(f"q{i}", 70, 60, True) for i in range(10)]
+
+    check = _zero_check(evaluate_trust(rows + [extra]))
+
+    assert (check.value, check.message.split(" ")[0]) == (0.0, "0/10")

From 30212241e604a4f700eb0b8626c56c74ef29701a Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Thu, 29 Oct 2026 11:46:30 -0700
Subject: [PATCH 21/53] test(cli): drive the real handlers for
 --judge-temperature and --judge-placeholder

benchmark run (single and multi-run) is driven through cmd_benchmark_run with
a faked pipeline; evaluate and eval score run end to end against a mocked
judge transport. Each removed call-site kwarg named in the reviews now fails a
test. The old test named for the handler called _run_single_library directly.

Thread: PRRT_kwDOVBg7g86xiJYT
Thread: PRRT_kwDOVBg7g86xlsUD
Thread: PRRT_kwDOVBg7g86xlsb0
Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/tests/test_answering_stack_p_step2.py  | 63 +++++++-----
 .../test_judge_placeholder_entry_points.py    | 96 +++++++++++++++++++
 2 files changed, 134 insertions(+), 25 deletions(-)
 create mode 100644 dokimi/tests/test_judge_placeholder_entry_points.py

diff --git a/dokimi/tests/test_answering_stack_p_step2.py b/dokimi/tests/test_answering_stack_p_step2.py
index 981e8fac2..85b60a8d9 100644
--- a/dokimi/tests/test_answering_stack_p_step2.py
+++ b/dokimi/tests/test_answering_stack_p_step2.py
@@ -8,44 +8,57 @@
 parses to, and where it lands.
 """
 
-from types import SimpleNamespace
+import pytest
 
 from tests.test_answering_stack_parity import _subparser
 
 
-def test_benchmark_run_judge_temperature_reaches_the_pipeline(monkeypatch, tmp_path):
+class _FakePipeline:
+    """Stands in for EvaluationPipeline: keeps the kwargs ``benchmark run`` builds it with."""
+
+    built: list = []
+
+    def __init__(self, **kwargs):
+        _FakePipeline.built.append(kwargs)
+
+    def run(self, concurrency):
+        return {"steps": {}}
+
+
+@pytest.mark.parametrize("extra, runs", [([], 1), (["--multi-run", "2"], 2)], ids=["single-run", "multi-run"])
+def test_benchmark_run_hands_the_judge_flags_to_the_pipeline(monkeypatch, tmp_path, extra, runs):
+    """The real handler runs; only the pipeline is faked. Both flags reach it in every mode."""
     from dokimi.commands import benchmark
     from dokimi import orchestrator
 
+    _FakePipeline.built = []
+    monkeypatch.setattr(orchestrator, "EvaluationPipeline", _FakePipeline)
     args = _subparser("benchmark", "run").parse_args(
-        ["--library", "onetbb", "--judge-temperature", "0.2"]
+        ["--library", "onetbb", "--output-dir", str(tmp_path), "--judge-temperature", "0.3",
+         "--judge-placeholder", *extra]
     )
-    assert args.judge_temperature == 0.2
 
-    seen = {}
+    benchmark.cmd_benchmark_run(args)
 
-    class FakePipeline:
-        def __init__(self, **kwargs):
-            seen.update(kwargs)
+    assert len(_FakePipeline.built) == runs
+    for kwargs in _FakePipeline.built:
+        assert kwargs["judge_temperature"] == 0.3
+        assert kwargs["judge_placeholder"] is True
 
-        def run(self, concurrency):
-            return {"steps": {}}
 
-    monkeypatch.setattr(orchestrator, "EvaluationPipeline", FakePipeline)
-    entry = SimpleNamespace(
-        name="oneTBB", key="onetbb", repo=None, description="d",
-        doc_sources=["context7"], context7_id="uxlfoundation/oneTBB",
-    )
-    benchmark._run_single_library(
-        entry,
-        output_dir=str(tmp_path),
-        model="m",
-        provider="openai",
-        judge_model="jm",
-        judge_provider="openai",
-        judge_temperature=args.judge_temperature,
-    )
-    assert seen["judge_temperature"] == 0.2
+def test_benchmark_run_leaves_the_judge_flags_off_when_not_given(monkeypatch, tmp_path):
+    from dokimi.commands import benchmark
+    from dokimi import orchestrator
+
+    _FakePipeline.built = []
+    monkeypatch.setattr(orchestrator, "EvaluationPipeline", _FakePipeline)
+    args = _subparser("benchmark", "run").parse_args(["--library", "onetbb", "--output-dir", str(tmp_path)])
+
+    benchmark.cmd_benchmark_run(args)
+
+    (kwargs,) = _FakePipeline.built
+    assert kwargs["judge_temperature"] is None
+    assert kwargs["judge_placeholder"] is False
 
 
 def test_benchmark_run_judge_temperature_defaults_to_provider_default():
diff --git a/dokimi/tests/test_judge_placeholder_entry_points.py b/dokimi/tests/test_judge_placeholder_entry_points.py
new file mode 100644
index 000000000..31f69d931
--- /dev/null
+++ b/dokimi/tests/test_judge_placeholder_entry_points.py
@@ -0,0 +1,96 @@
+# Copyright (C) 2026 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+"""--judge-placeholder reaches the judge of every entry point that takes it."""
+
+import json
+from types import SimpleNamespace
+
+import pytest
+
+from dokimi.cli import build_parser
+
+GOOD = ('{"reasoning":"r","correctness":80,"completeness":80,"specificity":80,'
+        '"code_quality":80,"actionability":80,"aggregate":80}')
+
+#: A question whose retrieval missed: the with-docs answer is the no-retrieval placeholder.
+ANSWERS = [{"question_id": "Q1", "question_text": "q", "question": "q",
+            "with_docs": {"answer": "[FALLBACK: No relevant documentation found]",
+                          "doc_source": "fallback_none", "retrieved_docs": []},
+            "without_docs": {"answer": "without docs"}}]
+
+JUDGE = ["--judge-provider", "amazon-bedrock", "--judge-model", "us.anthropic.x"]
+
+
+@pytest.fixture
+def requests(monkeypatch):
+    import litellm
+
+    sent = []
+
+    def completion(**kw):
+        sent.append(kw)
+        return SimpleNamespace(choices=[SimpleNamespace(message=SimpleNamespace(content=GOOD))],
+                               usage=None)
+
+    monkeypatch.setattr(litellm, "completion", completion)
+    return sent
+
+
+def _run(argv):
+    args = build_parser().parse_args(argv)
+    args.func(args)
+
+
+def _eval_score(tmp_path, *flag):
+    answers = tmp_path / "answers.json"
+    answers.write_text(json.dumps(ANSWERS), encoding="utf-8")
+    out = tmp_path / "eval.json"
+    _run(["eval", "score", "--product", "L", "--answers", str(answers), "--output", str(out),
+          "--concurrency", "1", *JUDGE, *flag])
+    return json.loads(out.read_text(encoding="utf-8"))
+
+
+def _evaluate(tmp_path, monkeypatch, *flag):
+    from dokimi.orchestrator import EvaluationPipeline
+
+    def run(self, **kw):
+        self._evaluate_answers(ANSWERS)
+        raise RuntimeError("stop after the judge step")
+
+    monkeypatch.setattr(EvaluationPipeline, "run", run)
+    with pytest.raises(SystemExit):
+        _run(["evaluate", "--product", "L", "--description", "d", "--output-dir", str(tmp_path), *JUDGE, *flag])
+    return json.loads((tmp_path / "eval" / "L.json").read_text(encoding="utf-8"))
+
+
+def test_eval_score_leaves_the_placeholder_unjudged_by_default(tmp_path, requests):
+    out = _eval_score(tmp_path)
+
+    assert len(requests) == 1  # the baseline answer only
+    assert out["evaluations"][0]["with_docs"] is None
+    assert out["judge_placeholder"] is False
+
+
+def test_eval_score_judge_placeholder_judges_it_and_says_so(tmp_path, requests):
+    out = _eval_score(tmp_path, "--judge-placeholder")
+
+    assert len(requests) == 2
+    assert out["evaluations"][0]["with_docs"]["aggregate"] == 80
+    assert out["judge_placeholder"] is True
+
+
+def test_evaluate_leaves_the_placeholder_unjudged_by_default(tmp_path, requests, monkeypatch):
+    out = _evaluate(tmp_path, monkeypatch)
+
+    assert len(requests) == 1
+    assert out["evaluations"][0]["with_docs"] is None
+    assert out["judge_placeholder"] is False
+
+
+def test_evaluate_judge_placeholder_judges_it_and_says_so(tmp_path, requests, monkeypatch):
+    out = _evaluate(tmp_path, monkeypatch, "--judge-placeholder")
+
+    assert len(requests) == 2
+    assert out["evaluations"][0]["with_docs"]["aggregate"] == 80
+    assert out["judge_placeholder"] is True

From 3ef4eae53f71cb93607db26515a60f8ac3485bfb Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Thu, 29 Oct 2026 11:46:55 -0700
Subject: [PATCH 22/53] docs(changelog): name the flags and commands that
 exist; say old scores differ

benchmark run has no --top-k and there is no orchestrate evaluate: the command
is dokimi evaluate. The fragment now says what each command asks for, that
answers generate changes too, that old scores are not comparable, and covers
the fixes of this round.

Thread: PRRT_kwDOVBg7g86xgwo3
Thread: PRRT_kwDOVBg7g86xlsKV
Thread: PRR_kwDOVBg7g88AAAABU4c24A
Thread: PRR_kwDOVBg7g88AAAABU5LNng
Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 .../20261028_120000_napetrov_stack-p-step2.md | 33 +++++++++++--------
 1 file changed, 20 insertions(+), 13 deletions(-)

diff --git a/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md b/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md
index 5fc6d7893..b626ec4f7 100644
--- a/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md
+++ b/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md
@@ -1,18 +1,24 @@
 ### Changed
 
-- **Stack P scores move (#228 step 2)** — `benchmark run` and `orchestrate evaluate` now
-  answer with the `--top-k` they were given (retrieval asked for `max_results` 5 and the
-  client default before), take the judge temperature from `--judge-temperature` (it used to
-  be dropped), and resolve the doc library id through `products.yaml` `context7_id` before the
-  client's own resolver. A question with no document (no hit, or the reranker raised) is no
-  longer judged on its `[FALLBACK: ...]` placeholder by default, so it drops out of the paired
-  statistics; the record and its `retrieval_hit` stay. Pass `--judge-placeholder` to judge the
-  placeholder again, which restores the old numbers. `eval score` takes the same flag.
+- **Stack P scores move (#228 step 2)** — the context arm now asks the doc source for the
+  `top_k` docs the command was given: `benchmark run` asks for 3 (`DEFAULT_RETRIEVAL_TOP_K`; it
+  has no `--top-k`), `dokimi evaluate` and `answers generate` for their `--top-k` (default 3).
+  Before, they left `max_results` at the client default of 5. `benchmark run --judge-temperature`
+  now reaches the judge (it used to be dropped), and the doc library id resolves through the
+  registry's `context7_id` before the client's own resolver, in the `--registry` that was
+  loaded. A question with no document (no hit, or the reranker raised) is no longer judged on
+  its `[FALLBACK: ...]` placeholder by default, so it drops out of the paired statistics; the
+  record and its `retrieval_hit` stay, its diagnosis stays `empty_retrieval`, and the run
+  summary says how many questions it left out. Pass `--judge-placeholder` to judge the
+  placeholder again, which restores the old numbers. `evaluate` and `eval score` take the same
+  flag. Scores from before this change are not comparable with new ones.
 - **Trust gate** — `retrieval_hit_rate` counts every evaluation's recorded retrieval outcome,
-  not only the ones scored in both arms, so unjudged misses still count against the rate. The
-  thresholds are unchanged.
+  not only the ones scored in both arms, so unjudged misses still count against the rate, and a
+  judge-error row keeps the outcome of its answer. The rate is shown even when no question is
+  scored in both arms. The thresholds are unchanged.
 - **Stamped artifacts** — the answers and eval artifacts carry `method_version` (2). Older
-  artifacts have none and read as version 1.
+  artifacts have none and read as version 1. An eval artifact also carries `judge_placeholder`,
+  so two files of one version that differ in it are told apart.
 
 ### Fixed
 
@@ -20,8 +26,9 @@
   source; the context arm's retrieval request is now the one the arms stack makes.
 - **`benchmark run --judge-temperature`** reaches the judge pipeline.
 - **Library id from the registry** — `EvaluationPipeline` resolves the library id through
-  `resolve_library_id_for_product` (explicit id, then `context7_id`, then the client). A
-  registered product without `context7_id` now fails for a client that needs a declared id.
+  `resolve_library_id_for_product` (explicit id, then `context7_id`, then the client), in the
+  registry that `benchmark run` and `benchmark batch` loaded from `--registry`. A registered
+  product without `context7_id` now fails for a client that needs a declared id.
 
 ### Removed
 

From 87a1a0a4b27f705d5deaca457c54235b58eb5c93 Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Thu, 29 Oct 2026 11:47:04 -0700
Subject: [PATCH 23/53] test(answerer): drop the ChatOpenAI patch the module no
 longer uses

dokimi.eval.answerer neither imports nor uses ChatOpenAI; the patch with
create=True and its mock did nothing and suggested the init builds a client.

Thread: IC_kwDOVBg7g88AAAABg14M_w
Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/tests/test_eval_answerer.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/dokimi/tests/test_eval_answerer.py b/dokimi/tests/test_eval_answerer.py
index a514d8d2a..75c9767fb 100644
--- a/dokimi/tests/test_eval_answerer.py
+++ b/dokimi/tests/test_eval_answerer.py
@@ -84,9 +84,7 @@ def test_init_accepts_supported_provider(self, provider):  # test-quality: ok mu
             Answerer(model="m", provider=provider)
 
     def test_init_unsupported_provider(self):
-        mock_cls = Mock()
-        with patch('dokimi.eval.answerer.ChatOpenAI', mock_cls, create=True), \
-             patch('dokimi.eval.answerer.LANGCHAIN_AVAILABLE', True):
+        with patch('dokimi.eval.answerer.LANGCHAIN_AVAILABLE', True):
             with pytest.raises(ValueError) as exc:
                 Answerer(provider="unsupported")
             assert "Unsupported provider" in str(exc.value)

From 24a75ca4fd291ed8c5194e389e6cbc9b95e837ba Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Thu, 29 Oct 2026 11:50:27 -0700
Subject: [PATCH 24/53] chore(pipeline): drop the records imports that the
 merge with main left unused

main's paired_summary (#250) replaced the BASELINE/CONTEXT/records reads in
run(); the merge kept this branch's import line, which ruff F401 rejects.

Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/dokimi/orchestrator/pipeline.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/dokimi/dokimi/orchestrator/pipeline.py b/dokimi/dokimi/orchestrator/pipeline.py
index a66e678b4..1dcf3e59c 100644
--- a/dokimi/dokimi/orchestrator/pipeline.py
+++ b/dokimi/dokimi/orchestrator/pipeline.py
@@ -12,7 +12,6 @@
 from dokimi.defaults import DEFAULT_ANSWER_MODEL, DEFAULT_ANSWER_PROVIDER, DEFAULT_JUDGE_MODEL, DEFAULT_JUDGE_PROVIDER, DEFAULT_RETRIEVAL_TOP_K
 from dokimi.eval.stats import paired_summary
 from dokimi.mcp.factory import create_doc_source_client, resolve_library_id_for_product
-from dokimi.records import BASELINE, CONTEXT, records as read_records
 
 logger = logging.getLogger(__name__)
 

From 284b24059d1278d25c121a392bd787dfd5d18fcc Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Thu, 29 Oct 2026 11:55:05 -0700
Subject: [PATCH 25/53] test(pipeline): pin the library id the question step
 hands the topic extractor

Codecov flagged the one patch line in _generate_questions that no test reached.
The test asserts the extractor gets the registry's declared id; reverting the
line to the client's name lookup fails it.

Thread: IC_kwDOVBg7g88AAAABgqSv4w
Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/tests/test_pipeline_library_id.py | 28 ++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/dokimi/tests/test_pipeline_library_id.py b/dokimi/tests/test_pipeline_library_id.py
index 6b1719866..3747c8682 100644
--- a/dokimi/tests/test_pipeline_library_id.py
+++ b/dokimi/tests/test_pipeline_library_id.py
@@ -60,3 +60,31 @@ def test_the_registry_key_not_the_display_name_finds_the_declared_id(tmp_path, r
 
     assert library_id == "org/declared"
     assert client.asked == []
+
+
+def test_question_generation_extracts_topics_for_the_declared_library(tmp_path, registry, monkeypatch):
+    """The question step asks the extractor for the registry's id, not for whatever the
+    client resolves the display name to: the arms stack and this step must agree."""
+    import dokimi.questions as questions
+
+    client = _RecordingClient()
+    seen = {}
+
+    class _StopHere(Exception):
+        pass
+
+    class _Extractor:
+        def __init__(self, **kwargs):
+            pass
+
+        def extract_topics(self, library_id, **kwargs):
+            seen["library_id"] = library_id
+            raise _StopHere
+
+    monkeypatch.setattr("dokimi.orchestrator.pipeline.create_doc_source_client", lambda source: client)
+    monkeypatch.setattr(questions, "RagasSeedExtractor", _Extractor)
+
+    with pytest.raises(_StopHere):
+        _pipeline(tmp_path, registry)._generate_questions({})
+
+    assert seen["library_id"] == "org/declared"

From 82d8f930f2a66b201e71dfd4f97c09526f736bca Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Thu, 29 Oct 2026 18:01:56 -0700
Subject: [PATCH 26/53] test: pin the branches the #274 review nits left
 untested

Three nits from the review of #274, one commit:

- test_diagnoser: an unjudged `none` retrieval miss is an empty retrieval,
  and an errored placeholder answer stays insufficient_data. Deleting the
  `not error` guard now fails a test.
- test_pipeline_library_id: a product key that is not in the registry is
  resolved by the client under the key. Passing the display name instead
  now fails a test.
- test_pipeline_summary: an all-paired run does not print "left out".
  Printing the suffix unconditionally now fails a test.

No source change: each new test was checked against the mutation the nit
describes (red), then the unchanged tree (green).

Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/tests/test_diagnoser.py           | 22 +++++++++++++++++++---
 dokimi/tests/test_pipeline_library_id.py | 13 +++++++++++++
 dokimi/tests/test_pipeline_summary.py    | 10 ++++++++++
 3 files changed, 42 insertions(+), 3 deletions(-)

diff --git a/dokimi/tests/test_diagnoser.py b/dokimi/tests/test_diagnoser.py
index e0d3362e5..4ba649503 100644
--- a/dokimi/tests/test_diagnoser.py
+++ b/dokimi/tests/test_diagnoser.py
@@ -18,13 +18,15 @@
 )
 
 
-def _make_answer(doc_source="context7", retrieved_docs=None):
+def _make_answer(doc_source="context7", retrieved_docs=None, error=None):
     """The context-arm answer of an answers.v1 record, as the judge hands it to diagnose()."""
+    with_docs = {"answer": "an answer", "doc_source": doc_source, "retrieved_docs": retrieved_docs or []}
+    if error:
+        with_docs["error"] = error
     data = artifact_fixture("answers", {"answers": [{
         "question_id": "q1",
         "question_text": "How?",
-        "with_docs": {"answer": "an answer", "doc_source": doc_source,
-                      "retrieved_docs": retrieved_docs or []},
+        "with_docs": with_docs,
         "without_docs": {"answer": "some answer"},
     }]})
     return records.read(data).records[0].answer(CONTEXT)
@@ -61,6 +63,20 @@ def test_unjudged_retrieval_miss_is_an_empty_retrieval_not_insufficient_data():
     assert result["evidence"]["doc_source"] == "fallback_none"
 
 
+def test_unjudged_none_source_is_an_empty_retrieval():
+    answer = _make_answer(doc_source="none", retrieved_docs=[])
+    result = diagnose(answer, None)
+    assert result["label"] == EMPTY_RETRIEVAL
+    assert result["evidence"]["doc_source"] == "none"
+
+
+def test_errored_placeholder_answer_stays_insufficient_data():
+    """An errored answer is missing, not a retrieval miss, whatever its doc source."""
+    answer = _make_answer(doc_source="fallback_none", retrieved_docs=[], error="timeout")
+    result = diagnose(answer, None)
+    assert result["label"] == INSUFFICIENT_DATA
+
+
 def test_empty_retrieval_no_mcp_client():
     answer = _make_answer(doc_source="none", retrieved_docs=[])
     result = diagnose(answer, -2.0)
diff --git a/dokimi/tests/test_pipeline_library_id.py b/dokimi/tests/test_pipeline_library_id.py
index 3747c8682..cf1364a55 100644
--- a/dokimi/tests/test_pipeline_library_id.py
+++ b/dokimi/tests/test_pipeline_library_id.py
@@ -62,6 +62,19 @@ def test_the_registry_key_not_the_display_name_finds_the_declared_id(tmp_path, r
     assert client.asked == []
 
 
+def test_an_unregistered_product_asks_the_client_for_its_key(tmp_path, registry):
+    """A key that is not in the registry goes to the client, by its key and not its display name."""
+    client = _RecordingClient()
+    pipeline = EvaluationPipeline(
+        product="Other Lib", description="d", product_key="other_lib", output_dir=tmp_path / "run", registry=registry
+    )
+
+    library_id = pipeline._library_id(client)
+
+    assert library_id == "client:other_lib"
+    assert client.asked == ["other_lib"]
+
+
 def test_question_generation_extracts_topics_for_the_declared_library(tmp_path, registry, monkeypatch):
     """The question step asks the extractor for the registry's id, not for whatever the
     client resolves the display name to: the arms stack and this step must agree."""
diff --git a/dokimi/tests/test_pipeline_summary.py b/dokimi/tests/test_pipeline_summary.py
index e7dcd57ec..a2c38ef07 100644
--- a/dokimi/tests/test_pipeline_summary.py
+++ b/dokimi/tests/test_pipeline_summary.py
@@ -85,3 +85,13 @@ def test_summary_says_how_many_questions_it_left_out(tmp_path, monkeypatch, caps
     printed = capsys.readouterr().out
     assert "Evaluated 10 answers: context=80.0, baseline=60.0, delta=+20.0" in printed
     assert "over the 7 scored in both arms; 3 left out" in printed
+
+
+def test_summary_does_not_say_left_out_when_every_question_is_paired(tmp_path, monkeypatch, capsys):
+    result = _run(tmp_path, monkeypatch, [_evaluation("q1", 80, 60), _evaluation("q2", 70, 50)])
+
+    step = result["steps"]["evaluation"]
+    assert (step["paired"], step["unpaired"]) == (2, 0)
+    printed = capsys.readouterr().out
+    assert "Evaluated 2 answers: context=75.0, baseline=55.0, delta=+20.0" in printed
+    assert "left out" not in printed

From 0cf25ef4ca9f83bd9b02f07777d0c36b7db41d65 Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Thu, 29 Oct 2026 19:26:50 -0700
Subject: [PATCH 27/53] docs(changelog): say benchmark run --judge-temperature
 is a new flag

The flag did not exist before this PR: benchmark run rejected it as an
unknown argument. The entry moves it from Changed and Fixed to Added, so
the Fixed section no longer repeats the claim.

Thread: PRRT_kwDOVBg7g86xtZgo
Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 .../20261028_120000_napetrov_stack-p-step2.md      | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md b/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md
index b626ec4f7..0ed9a2f26 100644
--- a/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md
+++ b/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md
@@ -1,12 +1,17 @@
+### Added
+
+- **`benchmark run --judge-temperature`** — a new flag: it sets the judge's sampling
+  temperature and is recorded as `judge_temperature` in the eval output. Before this change the
+  flag did not exist, and `benchmark run` rejected it as an unknown argument.
+
 ### Changed
 
 - **Stack P scores move (#228 step 2)** — the context arm now asks the doc source for the
   `top_k` docs the command was given: `benchmark run` asks for 3 (`DEFAULT_RETRIEVAL_TOP_K`; it
   has no `--top-k`), `dokimi evaluate` and `answers generate` for their `--top-k` (default 3).
-  Before, they left `max_results` at the client default of 5. `benchmark run --judge-temperature`
-  now reaches the judge (it used to be dropped), and the doc library id resolves through the
-  registry's `context7_id` before the client's own resolver, in the `--registry` that was
-  loaded. A question with no document (no hit, or the reranker raised) is no longer judged on
+  Before, they left `max_results` at the client default of 5. The doc library id resolves
+  through the registry's `context7_id` before the client's own resolver, in the `--registry`
+  that was loaded. A question with no document (no hit, or the reranker raised) is no longer judged on
   its `[FALLBACK: ...]` placeholder by default, so it drops out of the paired statistics; the
   record and its `retrieval_hit` stay, its diagnosis stays `empty_retrieval`, and the run
   summary says how many questions it left out. Pass `--judge-placeholder` to judge the
@@ -24,7 +29,6 @@
 
 - **Stack P no longer ignores `top_k`** — `Answerer` passes `max_results=top_k` to the doc
   source; the context arm's retrieval request is now the one the arms stack makes.
-- **`benchmark run --judge-temperature`** reaches the judge pipeline.
 - **Library id from the registry** — `EvaluationPipeline` resolves the library id through
   `resolve_library_id_for_product` (explicit id, then `context7_id`, then the client), in the
   registry that `benchmark run` and `benchmark batch` loaded from `--registry`. A registered

From 40308f6ae71c9f77c8eb7e449668d8dfdc7bf4c8 Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Thu, 29 Oct 2026 19:40:24 -0700
Subject: [PATCH 28/53] docs(changelog): say a judge-error row on a retrieval
 miss counts in zero_score_fraction

A judge-error row whose retrieval missed now counts as unusable in the
zero-score check; one on a retrieval hit is still left out. The fragment
did not say so, and the check is fail-severity.

Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md b/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md
index 0ed9a2f26..386d8a236 100644
--- a/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md
+++ b/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md
@@ -21,6 +21,9 @@
   not only the ones scored in both arms, so unjudged misses still count against the rate, and a
   judge-error row keeps the outcome of its answer. The rate is shown even when no question is
   scored in both arms. The thresholds are unchanged.
+- **Zero-score check** — a judge-error row on a retrieval miss now counts as unusable in
+  `zero_score_fraction` (a judge error on a retrieval hit is still left out), so a run can fail
+  that fail-severity check where intel/main passed it.
 - **Stamped artifacts** — the answers and eval artifacts carry `method_version` (2). Older
   artifacts have none and read as version 1. An eval artifact also carries `judge_placeholder`,
   so two files of one version that differ in it are told apart.

From 6731a112a7de346837d2bafeff001bfde25600a0 Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Thu, 29 Oct 2026 19:56:56 -0700
Subject: [PATCH 29/53] docs: keep the comment blocks this PR adds to two lines

Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/dokimi/defaults.py                   | 5 ++---
 dokimi/dokimi/gate/trust_gate.py            | 5 +----
 dokimi/tests/test_answering_stack_parity.py | 5 ++---
 3 files changed, 5 insertions(+), 10 deletions(-)

diff --git a/dokimi/dokimi/defaults.py b/dokimi/dokimi/defaults.py
index a7a6c85a5..f1b6fe264 100644
--- a/dokimi/dokimi/defaults.py
+++ b/dokimi/dokimi/defaults.py
@@ -91,9 +91,8 @@ class _JudgeModelForProvider(str):
 # Decision (2026-03-04): one fixed semantic retrieval config.
 DEFAULT_RETRIEVAL_TOP_K = 3
 
-# Version of the answering and judging method, stamped in the answers and eval artifacts.
-# Bump it when a change moves scores. Nothing reads it yet: a later compare is meant to refuse
-# runs of different versions. An artifact without the stamp predates it (#228 step 2); treat it as version 1.
+# Answering/judging method version, stamped into answers and eval artifacts; bump it when a change moves scores.
+# A later compare is meant to refuse mixed versions; an unstamped artifact predates it (#228 step 2) and is v1.
 ANSWERING_METHOD_VERSION = 2
 DEFAULT_MAX_TOKENS_PER_QUESTION = 4000
 DEFAULT_CONCURRENCY = 5
diff --git a/dokimi/dokimi/gate/trust_gate.py b/dokimi/dokimi/gate/trust_gate.py
index de23115ec..1ddb8f970 100644
--- a/dokimi/dokimi/gate/trust_gate.py
+++ b/dokimi/dokimi/gate/trust_gate.py
@@ -196,10 +196,7 @@ def evaluate_trust(
         return TrustVerdict(checks=checks)  # can't compute anything else
 
     # ── 2. Zero-score fraction ─────────────────────────────────────────────
-    # A retrieval miss is not judged by default (the no-document placeholder is not a
-    # with-docs answer). It still counts here as unusable: the zero-score check
-    # has always counted those questions, judged at zero. Leaving them out would
-    # let a run with many misses pass this fail-severity check.
+    # Misses count here as unusable (judged at zero, as this check always did), or a run with many misses would pass.
     unjudged_misses = [
         r for r in every
         if r.score(CONTEXT) is None and r.score(BASELINE) is not None
diff --git a/dokimi/tests/test_answering_stack_parity.py b/dokimi/tests/test_answering_stack_parity.py
index 744804a41..ca1035220 100644
--- a/dokimi/tests/test_answering_stack_parity.py
+++ b/dokimi/tests/test_answering_stack_parity.py
@@ -141,9 +141,8 @@
     # what the judge is handed as ground truth: the q-gt chunk on both stacks, None on the rest
     ("shared", "judge.ground_truth.q-gt", (GROUND_TRUTH_CHUNK,), (GROUND_TRUTH_CHUNK,)),
     ("shared", "judge.ground_truth.others", (None,), (None,)),
-    # the per-question deltas that feed D9's statistics (P: stored "delta"; A: "docs" vs "baseline").
-    # q-empty and q-rerank have no P delta (no with-docs judgement, D2/D3); A's q-empty is 0 and
-    # its q-rerank falls back to the raw docs (20).
+    # Per-question deltas feeding D9's statistics. q-empty and q-rerank have no P delta (no with-docs judgement, D2/D3);
+    # A's q-empty is 0 and its q-rerank falls back to the raw docs (20).
     (
         "D9",
         "delta.per_question",

From 2e85321a193fb0eb7ad3861e640555e9f0b0d89b Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Thu, 29 Oct 2026 20:08:51 -0700
Subject: [PATCH 30/53] fix(gate): a judge error on a retrieval miss counts as
 unusable

The zero-score check counted an unjudged retrieval miss, but a judge error
on a miss has no scores either and was left out. The error row is still a
miss, so it is counted; a judge error on a hit stays out, as before. Told
apart by the record's error field, not by its score shape.

Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/dokimi/gate/trust_gate.py              |  6 +--
 .../tests/test_trust_gate_unjudged_misses.py  | 50 +++++++++++++++++++
 2 files changed, 53 insertions(+), 3 deletions(-)

diff --git a/dokimi/dokimi/gate/trust_gate.py b/dokimi/dokimi/gate/trust_gate.py
index 1ddb8f970..1398f0c8c 100644
--- a/dokimi/dokimi/gate/trust_gate.py
+++ b/dokimi/dokimi/gate/trust_gate.py
@@ -196,11 +196,11 @@ def evaluate_trust(
         return TrustVerdict(checks=checks)  # can't compute anything else
 
     # ── 2. Zero-score fraction ─────────────────────────────────────────────
-    # Misses count here as unusable (judged at zero, as this check always did), or a run with many misses would pass.
+    # Misses count as unusable, judge-error misses too (by their error field): else many misses would pass.
     unjudged_misses = [
         r for r in every
-        if r.score(CONTEXT) is None and r.score(BASELINE) is not None
-        and r.retrieval_hit is False
+        if r.score(CONTEXT) is None and r.retrieval_hit is False
+        and (r.score(BASELINE) is not None or r.error is not None)
     ]
     zero_with = sum(1 for v in with_scores if v == 0) + len(unjudged_misses)
     unusable_n = n + len(unjudged_misses)
diff --git a/dokimi/tests/test_trust_gate_unjudged_misses.py b/dokimi/tests/test_trust_gate_unjudged_misses.py
index 95911c13b..3dbaf1bcb 100644
--- a/dokimi/tests/test_trust_gate_unjudged_misses.py
+++ b/dokimi/tests/test_trust_gate_unjudged_misses.py
@@ -134,3 +134,53 @@ def test_only_a_recorded_miss_with_a_baseline_counts_as_unusable(extra):
     check = _zero_check(evaluate_trust(rows + [extra]))
 
     assert (check.value, check.message.split(" ")[0]) == (0.0, "0/10")
+
+
+def _judged_with_failures(n_hit, n_miss, failing):
+    """Judge the answers through ``evaluate_answers``; each question id in ``failing`` raises in the judge."""
+    judge = Judge(model="gpt-x", provider="openai")
+    judge.llm = MagicMock()
+    judge.llm.invoke.side_effect = lambda msgs, *a, **k: GOOD
+    real = judge._evaluate_answer_pair
+
+    def flaky(library_name, item):
+        if item.question_id in failing:
+            raise RuntimeError("judge down")
+        return real(library_name, item)
+
+    judge._evaluate_answer_pair = flaky
+    answers = [
+        {"question_id": f"h{i}", "question_text": "q?",
+         "with_docs": dict(HIT), "without_docs": {"answer": "baseline"}}
+        for i in range(n_hit)
+    ] + [
+        {"question_id": f"m{i}", "question_text": "q?",
+         "with_docs": dict(MISS), "without_docs": {"answer": "baseline"}}
+        for i in range(n_miss)
+    ]
+    return judge.evaluate_answers("L", answers)
+
+
+def test_a_judge_error_on_a_retrieval_miss_counts_as_unusable(monkeypatch):
+    """The judge raises on a miss: its error row has no scores, but it is still an unusable question."""
+    monkeypatch.setenv("OPENAI_API_KEY", "test-key-not-used")
+    rows = _judged_with_failures(3, 1, failing={"m0"})
+    error_row = next(r for r in rows if r["question_id"] == "m0")
+    assert "error" in error_row and error_row["retrieval_hit"] is False
+
+    check = _zero_check(evaluate_trust(rows))
+
+    assert (check.value, check.passed) == (0.25, False)
+    assert check.message.split(" ")[0] == "1/4"
+
+
+def test_a_judge_error_on_a_retrieval_hit_stays_out_of_the_zero_check(monkeypatch):
+    monkeypatch.setenv("OPENAI_API_KEY", "test-key-not-used")
+    rows = _judged_with_failures(3, 0, failing={"h0"})
+    error_row = next(r for r in rows if r["question_id"] == "h0")
+    assert "error" in error_row and error_row["retrieval_hit"] is True
+
+    check = _zero_check(evaluate_trust(rows))
+
+    assert (check.value, check.passed) == (0.0, True)
+    assert check.message.split(" ")[0] == "0/2"

From a9a320b5618ccf54427c6d460ddd0fbc39558caf Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Thu, 29 Oct 2026 20:08:51 -0700
Subject: [PATCH 31/53] fix(answerer): a missing top_k asks the doc source for
 the default

Answerer passed max_results=self.top_k to the doc source, so a None top_k
reached the client as max_results=None, which returns every chunk. Make the
Answerer default and an explicit None both DEFAULT_RETRIEVAL_TOP_K (3). The
answers command already resolves --top-k to that default before it gets here;
this closes the direct-construction path.

Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/dokimi/eval/answerer.py     |  9 +++++----
 dokimi/tests/test_eval_answerer.py | 14 +++++++++++++-
 2 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/dokimi/dokimi/eval/answerer.py b/dokimi/dokimi/eval/answerer.py
index 74e6f0252..ebe8ce6a3 100644
--- a/dokimi/dokimi/eval/answerer.py
+++ b/dokimi/dokimi/eval/answerer.py
@@ -17,7 +17,7 @@
 from dokimi.artifacts import save_artifact
 from dokimi.defaults import ANSWERING_METHOD_VERSION
 from dokimi.llm import llm_call_with_usage, LANGCHAIN_AVAILABLE
-from dokimi.defaults import DEFAULT_ANSWER_MODEL, DEFAULT_ANSWER_PROVIDER
+from dokimi.defaults import DEFAULT_ANSWER_MODEL, DEFAULT_ANSWER_PROVIDER, DEFAULT_RETRIEVAL_TOP_K
 from dokimi.metrics.usage import UsageRecord, to_record
 
 from .reranker import SimpleReranker, SentenceTransformerReranker, SENTENCE_TRANSFORMERS_AVAILABLE
@@ -70,7 +70,7 @@ def __init__(
         model: str = DEFAULT_ANSWER_MODEL,
         provider: str = DEFAULT_ANSWER_PROVIDER,
         api_key: Optional[str] = None,
-        top_k: int = 5,
+        top_k: Optional[int] = None,
         rerank_threshold: float = 0.3,
         debug_retrieval: bool = False
     ):
@@ -80,7 +80,7 @@ def __init__(
             model: LLM model name
             provider: "openai" or "anthropic"
             api_key: Optional API key
-            top_k: Number of docs to retrieve before reranking
+            top_k: Number of docs to retrieve before reranking (default DEFAULT_RETRIEVAL_TOP_K)
             rerank_threshold: Minimum relevance score (0-1) to keep docs
             debug_retrieval: If True, include detailed retrieval metadata in output
         """
@@ -102,7 +102,8 @@ def __init__(
                 f"Unsupported provider: '{provider}'. "
                 "Use 'openai', 'anthropic', 'google', 'openrouter', or 'amazon-bedrock'."
             )
-        self.top_k = top_k
+        # None means the default: the doc source must never get max_results=None (it would return every chunk).
+        self.top_k = DEFAULT_RETRIEVAL_TOP_K if top_k is None else top_k
         self.debug_retrieval = debug_retrieval
         
         # Initialize reranker (prefer semantic, fallback to lexical)
diff --git a/dokimi/tests/test_eval_answerer.py b/dokimi/tests/test_eval_answerer.py
index 75c9767fb..7d5004d78 100644
--- a/dokimi/tests/test_eval_answerer.py
+++ b/dokimi/tests/test_eval_answerer.py
@@ -18,6 +18,7 @@
         sys.modules[mod_name] = mock_mod
 
 from dokimi.eval.answerer import Answerer, ANSWER_PROMPT_WITH_DOCS, ANSWER_PROMPT_WITHOUT_DOCS
+from dokimi.defaults import DEFAULT_RETRIEVAL_TOP_K
 
 # Canonical return value for llm_call_with_usage mock
 _LLM_MOCK_RETURN = ("This is a test answer from LLM.", {
@@ -196,12 +197,23 @@ def test_calls_mcp_client(self, mock_mcp_client):
 
             mock_mcp_client.get_library_docs.assert_called_once_with(
                 library_id="uxlfoundation/oneTBB", query="parallel_for",
-                max_results=5, max_tokens=4000,
+                max_results=DEFAULT_RETRIEVAL_TOP_K, max_tokens=4000,
             )
             assert [d["content"] for d in docs] == [
                 mock_mcp_client.get_library_docs.return_value[0]["content"]
             ]
 
+    def test_no_top_k_asks_the_doc_source_for_the_default(self, mock_mcp_client):
+        """`answers generate` without --top-k reaches here as top_k=None: the doc source gets the default, not None."""
+        with patch('dokimi.eval.answerer.LANGCHAIN_AVAILABLE', True):
+            answerer = Answerer(mcp_client=mock_mcp_client, top_k=None)
+            answerer._retrieve_docs("uxlfoundation/oneTBB", "parallel_for", 4000)
+
+            mock_mcp_client.get_library_docs.assert_called_once_with(
+                library_id="uxlfoundation/oneTBB", query="parallel_for",
+                max_results=3, max_tokens=4000,
+            )
+
     def test_error_handling(self, mock_mcp_client):
         mock_mcp_client.get_library_docs.side_effect = Exception("Network error")
 

From 5fae382faa7cd3e85ee0e90e234ba91a99c6d63a Mon Sep 17 00:00:00 2001
From: Anton Volkov <antonwolfy@gmail.com>
Date: Fri, 30 Oct 2026 09:36:36 +0000
Subject: [PATCH 32/53] docs(pipeline): document registry and judge_placeholder
 in Args

Signed-off-by: Anton Volkov <antonwolfy@gmail.com>
---
 dokimi/dokimi/orchestrator/pipeline.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/dokimi/dokimi/orchestrator/pipeline.py b/dokimi/dokimi/orchestrator/pipeline.py
index 1dcf3e59c..de80927ba 100644
--- a/dokimi/dokimi/orchestrator/pipeline.py
+++ b/dokimi/dokimi/orchestrator/pipeline.py
@@ -137,6 +137,10 @@ def __init__(
                 skipped; when it is absent a warning is logged and questions are
                 generated. Without this argument (and without
                 ``custom_questions_path``) the run has no golden set.
+            registry: ProductRegistry used to find the declared context7_id
+                (default: the bundled products.yaml)
+            judge_placeholder: Also judge a with-docs answer that is the
+                no-retrieval placeholder (default: recorded, not scored)
         """
         if not repo and not description:
             raise ValueError(

From 6dc1f4232c1a6da1346d82ca6545c16bf3ae16bc Mon Sep 17 00:00:00 2001
From: Anton Volkov <antonwolfy@gmail.com>
Date: Fri, 30 Oct 2026 09:37:01 +0000
Subject: [PATCH 33/53] docs(changelog): limit what --judge-placeholder
 restores

Signed-off-by: Anton Volkov <antonwolfy@gmail.com>
---
 dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md b/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md
index 386d8a236..c64a25838 100644
--- a/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md
+++ b/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md
@@ -15,8 +15,9 @@
   its `[FALLBACK: ...]` placeholder by default, so it drops out of the paired statistics; the
   record and its `retrieval_hit` stay, its diagnosis stays `empty_retrieval`, and the run
   summary says how many questions it left out. Pass `--judge-placeholder` to judge the
-  placeholder again, which restores the old numbers. `evaluate` and `eval score` take the same
-  flag. Scores from before this change are not comparable with new ones.
+  placeholder again (this restores only that part of the old method). `benchmark run`,
+  `evaluate` and `eval score` take the same flag; `benchmark batch` has none and never judges
+  the placeholder. Scores from before this change are not comparable with new ones.
 - **Trust gate** — `retrieval_hit_rate` counts every evaluation's recorded retrieval outcome,
   not only the ones scored in both arms, so unjudged misses still count against the rate, and a
   judge-error row keeps the outcome of its answer. The rate is shown even when no question is

From 805aa58a6ab3a36605a0258e3615ebc9352d94af Mon Sep 17 00:00:00 2001
From: Anton Volkov <antonwolfy@gmail.com>
Date: Fri, 30 Oct 2026 09:37:10 +0000
Subject: [PATCH 34/53] docs(methodology): zero-score bullet counts judge-error
 misses

Signed-off-by: Anton Volkov <antonwolfy@gmail.com>
---
 dokimi/docs/methodology.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dokimi/docs/methodology.md b/dokimi/docs/methodology.md
index cc1e9785f..85fe035c2 100644
--- a/dokimi/docs/methodology.md
+++ b/dokimi/docs/methodology.md
@@ -200,7 +200,7 @@ RAGAS meta-evaluation can be run after answer generation to add faithfulness and
 Every serious run should answer "Can we trust this run?" before interpreting deltas. The trust gate checks:
 
 - minimum evaluated question count (both arms scored; a question with no document is not judged by default, so it is not counted here),
-- zero-score fraction (over the both-arm-scored questions, plus the questions whose retrieval missed: those count as unusable, scored 0 or no document),
+- zero-score fraction (over the both-arm-scored questions, plus the questions whose retrieval missed and were left unjudged or hit a judge error: those count as unusable, scored 0 or no document),
 - minimum `with_docs` average (same subset),
 - average delta (same subset),
 - score coefficient of variation (same subset),

From 9284f52df2e0f8fd86d2b7c39651918345cb20e9 Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Thu, 29 Oct 2026 22:41:06 -0700
Subject: [PATCH 35/53] style(answerer): one import from dokimi.defaults

Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/dokimi/eval/answerer.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/dokimi/dokimi/eval/answerer.py b/dokimi/dokimi/eval/answerer.py
index ebe8ce6a3..dbf3c0599 100644
--- a/dokimi/dokimi/eval/answerer.py
+++ b/dokimi/dokimi/eval/answerer.py
@@ -15,9 +15,13 @@
 
 from dokimi.utils import question_id
 from dokimi.artifacts import save_artifact
-from dokimi.defaults import ANSWERING_METHOD_VERSION
+from dokimi.defaults import (
+    ANSWERING_METHOD_VERSION,
+    DEFAULT_ANSWER_MODEL,
+    DEFAULT_ANSWER_PROVIDER,
+    DEFAULT_RETRIEVAL_TOP_K,
+)
 from dokimi.llm import llm_call_with_usage, LANGCHAIN_AVAILABLE
-from dokimi.defaults import DEFAULT_ANSWER_MODEL, DEFAULT_ANSWER_PROVIDER, DEFAULT_RETRIEVAL_TOP_K
 from dokimi.metrics.usage import UsageRecord, to_record
 
 from .reranker import SimpleReranker, SentenceTransformerReranker, SENTENCE_TRANSFORMERS_AVAILABLE

From caaa3da3ddab381b30db7a2ef252f8d3aa204bac Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Fri, 30 Oct 2026 08:40:16 -0700
Subject: [PATCH 36/53] test(trust-gate): pin the scored-miss clause of the
 unjudged-miss filter

A miss the judge scored with a 0 context score is already a zero; a
mutant that drops the score-is-None clause counts it twice and passes
the existing tests. The new case asserts 1/11, and fails on that mutant.

Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/tests/test_trust_gate_unjudged_misses.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/dokimi/tests/test_trust_gate_unjudged_misses.py b/dokimi/tests/test_trust_gate_unjudged_misses.py
index 3dbaf1bcb..0b2053282 100644
--- a/dokimi/tests/test_trust_gate_unjudged_misses.py
+++ b/dokimi/tests/test_trust_gate_unjudged_misses.py
@@ -136,6 +136,16 @@ def test_only_a_recorded_miss_with_a_baseline_counts_as_unusable(extra):
     assert (check.value, check.message.split(" ")[0]) == (0.0, "0/10")
 
 
+def test_a_scored_miss_counts_once_in_the_zero_check():
+    # A miss the judge scored (--judge-placeholder) with a 0 context score is a zero already; it
+    # must not also count as an unjudged miss, or the numerator and the denominator both grow.
+    rows = [_row(f"q{i}", 70, 60, True) for i in range(10)] + [_row("scored-miss", 0, 60, False)]
+
+    check = _zero_check(evaluate_trust(rows))
+
+    assert (check.value, check.message.split(" ")[0]) == (round(1 / 11, 3), "1/11")
+
+
 def _judged_with_failures(n_hit, n_miss, failing):
     """Judge the answers through ``evaluate_answers``; each question id in ``failing`` raises in the judge."""
     judge = Judge(model="gpt-x", provider="openai")

From 291dee0e007909644896e748c12df7892c3d3311 Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Fri, 30 Oct 2026 08:40:16 -0700
Subject: [PATCH 37/53] test(answerer): pin a non-default top_k so an override
 cannot be ignored

The explicit top_k case used 3, which is also the default. With the
constructor ignoring the caller, the suite still passed. The case now
uses 7 and expects max_results=7; the mutant fails it.

Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/tests/test_eval_answerer.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dokimi/tests/test_eval_answerer.py b/dokimi/tests/test_eval_answerer.py
index 7d5004d78..5c9e2824a 100644
--- a/dokimi/tests/test_eval_answerer.py
+++ b/dokimi/tests/test_eval_answerer.py
@@ -182,12 +182,12 @@ def test_token_usage_summary_in_output(self, sample_questions, mock_mcp_client,
 class TestRetrieveDocs:
     def test_asks_the_doc_source_for_top_k_docs(self, mock_mcp_client):
         with patch('dokimi.eval.answerer.LANGCHAIN_AVAILABLE', True):
-            answerer = Answerer(mcp_client=mock_mcp_client, top_k=3)
+            answerer = Answerer(mcp_client=mock_mcp_client, top_k=7)
             answerer._retrieve_docs("uxlfoundation/oneTBB", "parallel_for", 4000)
 
             mock_mcp_client.get_library_docs.assert_called_once_with(
                 library_id="uxlfoundation/oneTBB", query="parallel_for",
-                max_results=3, max_tokens=4000,
+                max_results=7, max_tokens=4000,
             )
 
     def test_calls_mcp_client(self, mock_mcp_client):

From 633065f747d74ab4f5bcb460115741a09d70a751 Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Fri, 30 Oct 2026 08:40:16 -0700
Subject: [PATCH 38/53] docs(methodology): name the both-arm subset in the
 three metric bullets

The zero-score fraction uses an expanded subset; the minimum with_docs
average, the average delta and the coefficient of variation do not.

Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/docs/methodology.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/dokimi/docs/methodology.md b/dokimi/docs/methodology.md
index 85fe035c2..3f34ff28c 100644
--- a/dokimi/docs/methodology.md
+++ b/dokimi/docs/methodology.md
@@ -201,9 +201,9 @@ Every serious run should answer "Can we trust this run?" before interpreting del
 
 - minimum evaluated question count (both arms scored; a question with no document is not judged by default, so it is not counted here),
 - zero-score fraction (over the both-arm-scored questions, plus the questions whose retrieval missed and were left unjudged or hit a judge error: those count as unusable, scored 0 or no document),
-- minimum `with_docs` average (same subset),
-- average delta (same subset),
-- score coefficient of variation (same subset),
+- minimum `with_docs` average (both-arm-scored questions only),
+- average delta (both-arm-scored questions only),
+- score coefficient of variation (both-arm-scored questions only),
 - inter-rater agreement when panel data exists,
 - retrieval hit rate — the share of questions where at least one document reached the `with_docs` prompt, from the per-evaluation `retrieval_hit` the judge and panel record; eval files judged before that field existed show it as not measured, and so do files where fewer than 80% of evaluations record it (`min_retrieval_coverage`), since the rate would then describe a subset,
 - multi-run variance when `--multi-run N` is used.

From d84bb56d779921c4de864da9fa7e4e242db6c82e Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Fri, 30 Oct 2026 08:48:24 -0700
Subject: [PATCH 39/53] test(cli): allowlist --judge-placeholder, which this
 stack adds

No registry concept covers the placeholder judge yet, so the spelling
gets a reasoned allowlist row, as the contract test asks.

Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/tests/data/cli_flags_allowlist.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/dokimi/tests/data/cli_flags_allowlist.txt b/dokimi/tests/data/cli_flags_allowlist.txt
index 6e9ed384d..a24e75064 100644
--- a/dokimi/tests/data/cli_flags_allowlist.txt
+++ b/dokimi/tests/data/cli_flags_allowlist.txt
@@ -86,6 +86,7 @@
 --judge-configs
 --judge-model
 --judge-panel
+--judge-placeholder  # placeholder judge (#274) adds it: no registry concept yet; a F1b migration names it
 --judge-provider
 --judge-temperature
 --judges

From aabe058577bb2a7f8cfa83ed3b064737babd9369 Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Fri, 30 Oct 2026 11:18:02 -0700
Subject: [PATCH 40/53] docs(cli): regenerate the CLI snapshot after the merge
 of main and this stack's flags

Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/docs/reference/cli-interface.json | 64 ++++++++++++++++++++++++
 1 file changed, 64 insertions(+)

diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json
index 7f68d6a09..e278ff9c2 100644
--- a/dokimi/docs/reference/cli-interface.json
+++ b/dokimi/docs/reference/cli-interface.json
@@ -2250,6 +2250,22 @@
               "required": false,
               "type": null
             },
+            {
+              "action": "_StoreTrueAction",
+              "choices": null,
+              "const": true,
+              "default": false,
+              "group_key": null,
+              "group_required": null,
+              "hidden": false,
+              "is_flag": true,
+              "key": "--judge-placeholder",
+              "kind": "option",
+              "nargs": 0,
+              "position": null,
+              "required": false,
+              "type": null
+            },
             {
               "action": "_StoreAction",
               "choices": [
@@ -2273,6 +2289,22 @@
               "required": false,
               "type": null
             },
+            {
+              "action": "_StoreAction",
+              "choices": null,
+              "const": null,
+              "default": null,
+              "group_key": null,
+              "group_required": null,
+              "hidden": false,
+              "is_flag": false,
+              "key": "--judge-temperature",
+              "kind": "option",
+              "nargs": null,
+              "position": null,
+              "required": false,
+              "type": "float"
+            },
             {
               "action": "_StoreAction",
               "choices": null,
@@ -5630,6 +5662,22 @@
               "required": false,
               "type": null
             },
+            {
+              "action": "_StoreTrueAction",
+              "choices": null,
+              "const": true,
+              "default": false,
+              "group_key": null,
+              "group_required": null,
+              "hidden": false,
+              "is_flag": true,
+              "key": "--judge-placeholder",
+              "kind": "option",
+              "nargs": 0,
+              "position": null,
+              "required": false,
+              "type": null
+            },
             {
               "action": "_StoreAction",
               "choices": [
@@ -5839,6 +5887,22 @@
           "required": false,
           "type": null
         },
+        {
+          "action": "_StoreTrueAction",
+          "choices": null,
+          "const": true,
+          "default": false,
+          "group_key": null,
+          "group_required": null,
+          "hidden": false,
+          "is_flag": true,
+          "key": "--judge-placeholder",
+          "kind": "option",
+          "nargs": 0,
+          "position": null,
+          "required": false,
+          "type": null
+        },
         {
           "action": "_StoreAction",
           "choices": [

From a3cb4b37d69d236035d93723f145fbfa92e601a0 Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Fri, 30 Oct 2026 14:01:52 -0700
Subject: [PATCH 41/53] fix(eval): eval score stamps the method version of the
 answers it judges

An answers file with no method_version is legacy (version 1). Judging it used to stamp the eval artifact with the current version, so legacy answers re-judged today read as version 2 next to answers generated under the new method. The Judge now takes the answers' version; answers generated in process keep the current one.

Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/dokimi/commands/evaluate.py            |  3 +
 dokimi/dokimi/defaults.py                     |  2 +
 dokimi/dokimi/eval/judge.py                   | 11 +++-
 .../tests/test_eval_score_method_version.py   | 60 +++++++++++++++++++
 4 files changed, 74 insertions(+), 2 deletions(-)
 create mode 100644 dokimi/tests/test_eval_score_method_version.py

diff --git a/dokimi/dokimi/commands/evaluate.py b/dokimi/dokimi/commands/evaluate.py
index 62bfeeb0d..f413fd30d 100644
--- a/dokimi/dokimi/commands/evaluate.py
+++ b/dokimi/dokimi/commands/evaluate.py
@@ -17,6 +17,7 @@
     DEFAULT_JUDGE_PROVIDER,
     JUDGE_MODEL_FOR_PROVIDER,
     JUDGE_MODEL_HELP,
+    LEGACY_ANSWERING_METHOD_VERSION,
 )
 from dokimi.eval.judge import JUDGE_PROVIDERS
 from dokimi.eval.judge_validity import DEFAULT_MIN_ACCURACY, NoUsablePairs
@@ -204,6 +205,8 @@ def cmd_eval_score(args: argparse.Namespace) -> None:
         provider=args.judge_provider,
         temperature=args.judge_temperature,
         judge_placeholder=getattr(args, "judge_placeholder", False),
+        # The eval says which answering method made the answers it judges; an unstamped file is legacy.
+        answers_method_version=run.meta.get("method_version") or LEGACY_ANSWERING_METHOD_VERSION,
         run_metadata={
             "question_model": run.meta.get("question_model", "unknown"),
             "question_provider": run.meta.get("question_provider", "unknown"),
diff --git a/dokimi/dokimi/defaults.py b/dokimi/dokimi/defaults.py
index f1b6fe264..07aa2a2a0 100644
--- a/dokimi/dokimi/defaults.py
+++ b/dokimi/dokimi/defaults.py
@@ -94,6 +94,8 @@ class _JudgeModelForProvider(str):
 # Answering/judging method version, stamped into answers and eval artifacts; bump it when a change moves scores.
 # A later compare is meant to refuse mixed versions; an unstamped artifact predates it (#228 step 2) and is v1.
 ANSWERING_METHOD_VERSION = 2
+#: The version of an artifact that carries no stamp (written before ANSWERING_METHOD_VERSION existed).
+LEGACY_ANSWERING_METHOD_VERSION = 1
 DEFAULT_MAX_TOKENS_PER_QUESTION = 4000
 DEFAULT_CONCURRENCY = 5
 DEFAULT_RESULTS_SUFFIX = "_final"
diff --git a/dokimi/dokimi/eval/judge.py b/dokimi/dokimi/eval/judge.py
index 7dd52bf41..961722f0c 100644
--- a/dokimi/dokimi/eval/judge.py
+++ b/dokimi/dokimi/eval/judge.py
@@ -198,7 +198,9 @@ class Judge:
 
     #: Class default, so a Judge built without __init__ (tests build one with __new__) keeps the default.
     judge_placeholder: bool = False
-    
+    #: Same for the stamp: the eval artifact carries the method version of the answers it judged.
+    answers_method_version: int = ANSWERING_METHOD_VERSION
+
     def __init__(
         self,
         model: str = DEFAULT_JUDGE_MODEL,
@@ -209,6 +211,7 @@ def __init__(
         seed: Optional[int] = None,
         prompt_template: Optional[str] = None,
         judge_placeholder: bool = False,
+        answers_method_version: int = ANSWERING_METHOD_VERSION,
     ):
         """
         Args:
@@ -225,6 +228,9 @@ def __init__(
             judge_placeholder: Judge a context answer that is the no-retrieval
                 placeholder ("[FALLBACK: ...]") too. Off by default: a question
                 with no document reaching the prompt is not scored as an answer.
+            answers_method_version: The method version of the answers being judged,
+                stamped into the eval artifact. Answers read from an artifact with no
+                stamp are version 1; answers generated in this process are the current version.
         """
         if not LANGCHAIN_AVAILABLE:
             raise ImportError(
@@ -241,6 +247,7 @@ def __init__(
         self.run_metadata = dict(run_metadata) if run_metadata else {}
         self.prompt_template = prompt_template
         self.judge_placeholder = judge_placeholder
+        self.answers_method_version = answers_method_version
         # Accumulate judge-side token/cost telemetry across all judge calls
         # (Phase-C.0 covered answering; the judge path was untracked).
         self._usage_lock = threading.Lock()
@@ -651,7 +658,7 @@ def _build_evaluation_output(self, evaluations: List[Dict[str, Any]]) -> Dict[st
         """Build the serialisable output structure for a set of evaluations."""
         output: Dict[str, Any] = {
             "evaluated_at": self._get_timestamp(),
-            "method_version": ANSWERING_METHOD_VERSION,
+            "method_version": self.answers_method_version,
             "judge_placeholder": self.judge_placeholder,
             "judge_model": self.model,
             "judge_provider": self.provider,
diff --git a/dokimi/tests/test_eval_score_method_version.py b/dokimi/tests/test_eval_score_method_version.py
new file mode 100644
index 000000000..04545e3c6
--- /dev/null
+++ b/dokimi/tests/test_eval_score_method_version.py
@@ -0,0 +1,60 @@
+# Copyright (C) 2026 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+"""``eval score`` stamps the eval artifact with the method version of the answers it judged."""
+
+import json
+from types import SimpleNamespace
+
+import pytest
+
+from dokimi.cli import build_parser
+
+GOOD = ('{"reasoning":"r","correctness":80,"completeness":80,"specificity":80,'
+        '"code_quality":80,"actionability":80,"aggregate":80}')
+
+ANSWERS = [{"question_id": "Q1", "question_text": "q", "question": "q",
+            "with_docs": {"answer": "an answer", "doc_source": "x", "retrieved_docs": ["d"]},
+            "without_docs": {"answer": "without docs"}}]
+
+JUDGE = ["--judge-provider", "amazon-bedrock", "--judge-model", "us.anthropic.x"]
+
+
+@pytest.fixture
+def judge_calls(monkeypatch):
+    import litellm
+
+    sent = []
+
+    def completion(**kw):
+        sent.append(kw)
+        return SimpleNamespace(choices=[SimpleNamespace(message=SimpleNamespace(content=GOOD))],
+                               usage=None)
+
+    monkeypatch.setattr(litellm, "completion", completion)
+    return sent
+
+
+def _eval_score(tmp_path, answers_doc):
+    answers = tmp_path / "answers.json"
+    answers.write_text(json.dumps(answers_doc), encoding="utf-8")
+    out = tmp_path / "eval.json"
+    args = build_parser().parse_args(["eval", "score", "--product", "L", "--answers", str(answers),
+                                      "--output", str(out), "--concurrency", "1", *JUDGE])
+    args.func(args)
+    return json.loads(out.read_text(encoding="utf-8"))
+
+
+def test_unstamped_answers_are_legacy_so_the_eval_says_version_1(tmp_path, judge_calls):
+    """Answers written before the stamp are the legacy method; re-judging them today does not make them version 2."""
+    out = _eval_score(tmp_path, ANSWERS)
+
+    assert len(judge_calls) == 2  # both arms judged: the stamp does not change what is judged
+    assert out["method_version"] == 1
+
+
+@pytest.mark.parametrize("version", [1, 2])
+def test_stamped_answers_keep_their_method_version(tmp_path, judge_calls, version):
+    out = _eval_score(tmp_path, {"method_version": version, "answers": ANSWERS})
+
+    assert out["method_version"] == version

From ffd6247b46afe8f0b2b4ea9f984d72322875956b Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Fri, 30 Oct 2026 14:01:52 -0700
Subject: [PATCH 42/53] fix(pipeline): an mcp source's ;id= outranks the
 registry's context7_id

resolve_library_id_for_product checks the registry before the client, so a registered product replaced the id named in an mcp: descriptor. The pipeline now passes the descriptor's id as the explicit id, after the --context7-id option and before the registry, as it did before the registry lookup.

Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/dokimi/orchestrator/pipeline.py   |  9 ++++++---
 dokimi/tests/test_pipeline_library_id.py | 21 +++++++++++++++++++++
 2 files changed, 27 insertions(+), 3 deletions(-)

diff --git a/dokimi/dokimi/orchestrator/pipeline.py b/dokimi/dokimi/orchestrator/pipeline.py
index de80927ba..9d21c7f41 100644
--- a/dokimi/dokimi/orchestrator/pipeline.py
+++ b/dokimi/dokimi/orchestrator/pipeline.py
@@ -458,12 +458,15 @@ def _discover_personas(self) -> Dict[str, Any]:
         return personas
     
     def _library_id(self, mcp_client) -> str:
-        """The doc-source library id: explicit id, then the registry's ``context7_id``, then the client.
+        """The doc-source library id: explicit id, then the ``;id=`` of an ``mcp:`` source, then the
+        registry's ``context7_id``, then the client.
 
-        The same order the arms stack uses (``commands/arms.py``, ``_resolve_doc_library_id``).
+        The caller's own choices (the explicit id, the source's descriptor) go before the registry, as
+        they did before the pipeline used the registry lookup; ``eval/arms_run.py`` resolves registry-first.
         """
+        descriptor_id = getattr(mcp_client, "default_library_id", None)
         return resolve_library_id_for_product(
-            self.product_key, mcp_client, explicit_id=self.context7_id, registry=self.registry,
+            self.product_key, mcp_client, explicit_id=self.context7_id or descriptor_id, registry=self.registry,
         )
 
     def _generate_questions(self, personas: Dict[str, Any]) -> List[Dict[str, Any]]:
diff --git a/dokimi/tests/test_pipeline_library_id.py b/dokimi/tests/test_pipeline_library_id.py
index cf1364a55..854130bfa 100644
--- a/dokimi/tests/test_pipeline_library_id.py
+++ b/dokimi/tests/test_pipeline_library_id.py
@@ -53,6 +53,27 @@ def test_an_explicit_id_wins_over_the_registry_entry(tmp_path, registry):
     assert client.asked == []
 
 
+def test_an_mcp_descriptor_id_wins_over_the_registry_entry(tmp_path, registry):
+    """``mcp:...;id=X`` names the library for that source; the registry's id does not replace it."""
+    from dokimi.mcp.factory import create_doc_source_client
+
+    client = create_doc_source_client("mcp:http=https://example.com/mcp;id=org/custom")
+
+    library_id = _pipeline(tmp_path, registry)._library_id(client)
+
+    assert library_id == "org/custom"
+
+
+def test_an_explicit_id_wins_over_the_mcp_descriptor_id(tmp_path, registry):
+    from dokimi.mcp.factory import create_doc_source_client
+
+    client = create_doc_source_client("mcp:http=https://example.com/mcp;id=org/custom")
+
+    library_id = _pipeline(tmp_path, registry, context7_id="org/explicit")._library_id(client)
+
+    assert library_id == "org/explicit"
+
+
 def test_the_registry_key_not_the_display_name_finds_the_declared_id(tmp_path, registry):
     client = _RecordingClient()
 

From b92f6c2cdfefb40d487a396bad656083cdf2e9c7 Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Fri, 30 Oct 2026 14:01:52 -0700
Subject: [PATCH 43/53] docs(changelog,methodology): say what the stamp and the
 zero-score bullet cover

Legacy artifacts are version 1 but readers do not normalize that yet; the zero-score bullet names the case where no question is scored in both arms.

Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 .../20261028_120000_napetrov_stack-p-step2.md        | 12 ++++++++----
 dokimi/docs/methodology.md                           |  2 +-
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md b/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md
index c64a25838..78bc15912 100644
--- a/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md
+++ b/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md
@@ -25,16 +25,20 @@
 - **Zero-score check** — a judge-error row on a retrieval miss now counts as unusable in
   `zero_score_fraction` (a judge error on a retrieval hit is still left out), so a run can fail
   that fail-severity check where intel/main passed it.
-- **Stamped artifacts** — the answers and eval artifacts carry `method_version` (2). Older
-  artifacts have none and read as version 1. An eval artifact also carries `judge_placeholder`,
-  so two files of one version that differ in it are told apart.
+- **Stamped artifacts** — the answers and eval artifacts carry `method_version` (2). Artifacts
+  from before this change have no stamp and represent the legacy method (version 1); readers do
+  not yet normalize or enforce the version, so it does not yet stop a comparison of mixed methods.
+  `eval score` stamps its eval with the version of the answers file it judges, so legacy answers
+  re-judged today stay version 1. An eval artifact also carries `judge_placeholder`, so two files
+  of one version that differ in it are told apart.
 
 ### Fixed
 
 - **Stack P no longer ignores `top_k`** — `Answerer` passes `max_results=top_k` to the doc
   source; the context arm's retrieval request is now the one the arms stack makes.
 - **Library id from the registry** — `EvaluationPipeline` resolves the library id through
-  `resolve_library_id_for_product` (explicit id, then `context7_id`, then the client), in the
+  `resolve_library_id_for_product` (explicit id, then an `mcp:` source's `;id=`, then `context7_id`,
+  then the client), in the
   registry that `benchmark run` and `benchmark batch` loaded from `--registry`. A registered
   product without `context7_id` now fails for a client that needs a declared id.
 
diff --git a/dokimi/docs/methodology.md b/dokimi/docs/methodology.md
index 3f34ff28c..a5a43b9cb 100644
--- a/dokimi/docs/methodology.md
+++ b/dokimi/docs/methodology.md
@@ -200,7 +200,7 @@ RAGAS meta-evaluation can be run after answer generation to add faithfulness and
 Every serious run should answer "Can we trust this run?" before interpreting deltas. The trust gate checks:
 
 - minimum evaluated question count (both arms scored; a question with no document is not judged by default, so it is not counted here),
-- zero-score fraction (over the both-arm-scored questions, plus the questions whose retrieval missed and were left unjudged or hit a judge error: those count as unusable, scored 0 or no document),
+- zero-score fraction (computed when at least one question is scored in both arms; over those paired questions plus retrieval misses with a scored baseline or a judge error, which count as unusable),
 - minimum `with_docs` average (both-arm-scored questions only),
 - average delta (both-arm-scored questions only),
 - score coefficient of variation (both-arm-scored questions only),

From 21b305489ba5ee7a96040661b98b61ab73ff25d6 Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Fri, 30 Oct 2026 17:59:29 -0700
Subject: [PATCH 44/53] fix(pipeline): eval judges answers with the method
 version of their artifact

The pipeline built its Judge without answers_method_version, so answers
read from a version 1 artifact, or from an unstamped legacy file, were
stamped as version 2. The version now comes from the artifact it read;
fresh in-memory answers keep the current version.

Thread: PRRT_kwDOVBg7g86yJ-x3
Comment: IC_kwDOVBg7g88AAAABhDPhBg
Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/dokimi/orchestrator/pipeline.py        | 13 +++++--
 .../tests/test_eval_score_method_version.py   | 39 +++++++++++++++++++
 2 files changed, 49 insertions(+), 3 deletions(-)

diff --git a/dokimi/dokimi/orchestrator/pipeline.py b/dokimi/dokimi/orchestrator/pipeline.py
index 9d21c7f41..548519bd6 100644
--- a/dokimi/dokimi/orchestrator/pipeline.py
+++ b/dokimi/dokimi/orchestrator/pipeline.py
@@ -9,7 +9,7 @@
 from typing import List, Dict, Any, Optional
 from pathlib import Path
 
-from dokimi.defaults import DEFAULT_ANSWER_MODEL, DEFAULT_ANSWER_PROVIDER, DEFAULT_JUDGE_MODEL, DEFAULT_JUDGE_PROVIDER, DEFAULT_RETRIEVAL_TOP_K
+from dokimi.defaults import (ANSWERING_METHOD_VERSION, LEGACY_ANSWERING_METHOD_VERSION, DEFAULT_ANSWER_MODEL, DEFAULT_ANSWER_PROVIDER, DEFAULT_JUDGE_MODEL, DEFAULT_JUDGE_PROVIDER, DEFAULT_RETRIEVAL_TOP_K)
 from dokimi.eval.stats import paired_summary
 from dokimi.mcp.factory import create_doc_source_client, resolve_library_id_for_product
 
@@ -680,9 +680,15 @@ def _evaluate_answers(self, answers: List[Dict[str, Any]] | Dict[str, Any],
         # Use in-memory answers from the previous step (avoid stale disk read);
         # fall back to the answers file when none were passed.
         if answers:
-            answers_list = list(read(answers, source="answers", section="answers").records)
+            run = read(answers, source="answers", section="answers")
         else:
-            answers_list = list(load(self.answers_path, section="answers").records)
+            run = load(self.answers_path, section="answers")
+        answers_list = list(run.records)
+        # Fresh in-memory answers are this process's method; an artifact says its own, unstamped is legacy.
+        if answers and isinstance(answers, list):
+            answers_method_version = ANSWERING_METHOD_VERSION
+        else:
+            answers_method_version = run.meta.get("method_version") or LEGACY_ANSWERING_METHOD_VERSION
         
         # Evaluate
         judge = Judge(
@@ -690,6 +696,7 @@ def _evaluate_answers(self, answers: List[Dict[str, Any]] | Dict[str, Any],
             provider=self.judge_provider,
             temperature=self.judge_temperature,
             judge_placeholder=self.judge_placeholder,
+            answers_method_version=answers_method_version,
             run_metadata={
                 "question_model": self.model,
                 "question_provider": self.provider,
diff --git a/dokimi/tests/test_eval_score_method_version.py b/dokimi/tests/test_eval_score_method_version.py
index 04545e3c6..4649341c9 100644
--- a/dokimi/tests/test_eval_score_method_version.py
+++ b/dokimi/tests/test_eval_score_method_version.py
@@ -58,3 +58,42 @@ def test_stamped_answers_keep_their_method_version(tmp_path, judge_calls, versio
     out = _eval_score(tmp_path, {"method_version": version, "answers": ANSWERS})
 
     assert out["method_version"] == version
+
+
+def _pipeline_judge_version(monkeypatch, tmp_path, answers, answers_file=None):
+    """The method version the pipeline's judge is built with, for these answers."""
+    from dokimi.eval.judge import Judge
+    from dokimi.orchestrator.pipeline import EvaluationPipeline
+
+    monkeypatch.setenv("OPENAI_API_KEY", "test-key-not-used")
+    seen = []
+
+    def evaluate(self, product, records, concurrency=5):
+        seen.append(self.answers_method_version)
+        return []
+
+    monkeypatch.setattr(Judge, "evaluate_answers", evaluate)
+    monkeypatch.setattr(Judge, "save_evaluations", lambda self, evaluations, path: None)
+    p = EvaluationPipeline.__new__(EvaluationPipeline)  # the attributes _evaluate_answers reads
+    p.product, p.model, p.provider = "L", "gpt-x", "openai"
+    p.judge_model, p.judge_provider, p.judge_temperature, p.judge_placeholder = "gpt-x", "openai", None, False
+    p.answers_path, p.eval_path = answers_file, tmp_path / "eval.json"
+    p._evaluate_answers(answers, concurrency=1)
+    return seen[0]
+
+
+def test_pipeline_judges_a_wrapped_version_1_artifact_as_version_1(monkeypatch, tmp_path):
+    assert _pipeline_judge_version(monkeypatch, tmp_path, {"method_version": 1, "answers": ANSWERS}) == 1
+
+
+def test_pipeline_judges_an_unstamped_answers_file_as_legacy_version_1(monkeypatch, tmp_path):
+    answers_file = tmp_path / "answers.json"
+    answers_file.write_text(json.dumps({"answers": ANSWERS}), encoding="utf-8")
+
+    assert _pipeline_judge_version(monkeypatch, tmp_path, [], answers_file) == 1
+
+
+def test_pipeline_judges_fresh_in_memory_answers_as_the_current_version(monkeypatch, tmp_path):
+    from dokimi.defaults import ANSWERING_METHOD_VERSION
+
+    assert _pipeline_judge_version(monkeypatch, tmp_path, ANSWERS) == ANSWERING_METHOD_VERSION

From f1854ee14e61aca53bd583eabec3d21d2d7ffc1d Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Fri, 30 Oct 2026 17:59:29 -0700
Subject: [PATCH 45/53] test(trust-gate): expect the zero-score wording the
 gate now uses

Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/tests/test_trust_gate.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dokimi/tests/test_trust_gate.py b/dokimi/tests/test_trust_gate.py
index 7d0bf118a..cc17ad1b4 100644
--- a/dokimi/tests/test_trust_gate.py
+++ b/dokimi/tests/test_trust_gate.py
@@ -445,7 +445,7 @@ def test_failures_and_warnings_render_apart(self):
             "| Check | Status | Value | Threshold | Notes |",
             "|-------|--------|-------|-----------|-------|",
             "| min_questions | ❌ | 3 | 10 | 3 questions evaluated (min 10) |",
-            "| zero_score_fraction | ❌ | 0.333 | 0.2 | 1/3 questions scored 0 in the context arm (33.3%) |",
+            "| zero_score_fraction | ❌ | 0.333 | 0.2 | 1/3 questions scored 0 or had no document in the context arm (33.3%) |",
             "| min_with_docs_avg | ✅ | 33.3 | 30.0 | Context-arm avg = 33.3 (min 30.0) |",
             "| min_delta | ✅ | 13.3 | -5.0 | Avg delta (context arm minus baseline) = +13.3 (min -5.0) |",
             "| score_cv | ⚠️ | 0.917 | 0.6 | Score CV = 0.92 std=30.6 (max CV 0.6) |",
@@ -455,7 +455,7 @@ def test_failures_and_warnings_render_apart(self):
             "",
             "**Blocking issues:**",
             "- ❌ **min_questions**: 3 questions evaluated (min 10)",
-            "- ❌ **zero_score_fraction**: 1/3 questions scored 0 in the context arm (33.3%)",
+            "- ❌ **zero_score_fraction**: 1/3 questions scored 0 or had no document in the context arm (33.3%)",
             "",
             "**Warnings:**",
             "- ⚠️ **score_cv**: Score CV = 0.92 std=30.6 (max CV 0.6)",

From a104154aca6414dec3bdc5d4e90cb401af1c383c Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Fri, 30 Oct 2026 22:05:08 -0700
Subject: [PATCH 46/53] Address review: a judged placeholder with no baseline
 is insufficient data

With judge_placeholder the context answer is scored, so a None delta
comes from a missing baseline. diagnose() classed any no-retrieval
context answer with no delta as an empty retrieval; it now does so only
when the context was left unjudged.

Thread: PRRT_kwDOVBg7g86yMEiW
Thread: IC_kwDOVBg7g88AAAABhDPhBg
Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/dokimi/eval/diagnoser.py | 7 ++++++-
 dokimi/dokimi/eval/judge.py     | 2 +-
 dokimi/tests/test_diagnoser.py  | 7 +++++++
 3 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/dokimi/dokimi/eval/diagnoser.py b/dokimi/dokimi/eval/diagnoser.py
index 4bad7a466..af1e8e0b1 100644
--- a/dokimi/dokimi/eval/diagnoser.py
+++ b/dokimi/dokimi/eval/diagnoser.py
@@ -74,6 +74,7 @@ def diagnose(
     context_answer: Optional[Answer],
     delta: Optional[float],
     low_relevance_cutoff: float = LOW_RELEVANCE_CUTOFF,
+    context_scored: bool = False,
 ) -> Dict[str, Any]:
     """
     Classify the retrieval outcome for a single question.
@@ -87,6 +88,9 @@ def diagnose(
         Context-arm score minus baseline score, ``None`` when either is missing.
     low_relevance_cutoff:
         Relevance score below which retrieved docs are treated as off-topic.
+    context_scored:
+        True when the judge scored the context answer (``judge_placeholder``), so a
+        ``None`` delta comes from a missing baseline, not from an unjudged miss.
 
     Returns
     -------
@@ -98,7 +102,8 @@ def diagnose(
     if delta is None:
         # A retrieval miss is left unjudged, so it has no delta; it is still an
         # empty retrieval, not a missing or errored answer.
-        if context_answer is not None and not context_answer.error and context_answer.doc_source in NO_RETRIEVAL_SOURCES:
+        unjudged = not context_scored
+        if unjudged and context_answer is not None and not context_answer.error and context_answer.doc_source in NO_RETRIEVAL_SOURCES:
             return _result(EMPTY_RETRIEVAL, {"doc_source": context_answer.doc_source, "retrieved_count": 0})
         return _result(INSUFFICIENT_DATA, {})
 
diff --git a/dokimi/dokimi/eval/judge.py b/dokimi/dokimi/eval/judge.py
index 051299982..03de110c2 100644
--- a/dokimi/dokimi/eval/judge.py
+++ b/dokimi/dokimi/eval/judge.py
@@ -435,7 +435,7 @@ def _evaluate_answer_pair(
         }
 
         # Classify retrieval outcome (why docs helped or didn't)
-        result["diagnosis"] = diagnose(context_answer, delta)
+        result["diagnosis"] = diagnose(context_answer, delta, context_scored=with_docs_eval is not None)
         # Carried onto the eval record because the trust gate reads eval.v1, not
         # the answers it was judged from.
         result["retrieval_hit"] = answer.retrieval_hit
diff --git a/dokimi/tests/test_diagnoser.py b/dokimi/tests/test_diagnoser.py
index 4ba649503..2931d49e1 100644
--- a/dokimi/tests/test_diagnoser.py
+++ b/dokimi/tests/test_diagnoser.py
@@ -70,6 +70,13 @@ def test_unjudged_none_source_is_an_empty_retrieval():
     assert result["evidence"]["doc_source"] == "none"
 
 
+def test_judged_placeholder_with_no_baseline_is_insufficient_data():
+    """With judge_placeholder the context is scored; a missing baseline is the reason there is no delta."""
+    answer = _make_answer(doc_source="fallback_none", retrieved_docs=[])
+    result = diagnose(answer, None, context_scored=True)
+    assert result["label"] == INSUFFICIENT_DATA
+
+
 def test_errored_placeholder_answer_stays_insufficient_data():
     """An errored answer is missing, not a retrieval miss, whatever its doc source."""
     answer = _make_answer(doc_source="fallback_none", retrieved_docs=[], error="timeout")

From d33a8950b7fa50b20281c5642a95edb8d8f6f8a2 Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Fri, 30 Oct 2026 22:05:37 -0700
Subject: [PATCH 47/53] Address review summaries: points already fixed on this
 branch

Each review below raised points that its inline threads cover; the fixes
are in the commits that answer those threads (the gate clauses are pinned
by test_trust_gate.py and test_trust_gate_unjudged_misses.py).

Thread: PRR_kwDOVBg7g88AAAABU4c24A: already fixed in the inline threads' commits
Thread: PRR_kwDOVBg7g88AAAABU5LNng: already fixed in the inline threads' commits
Thread: PRR_kwDOVBg7g88AAAABVF59Bw: already fixed in the inline threads' commits
Thread: PRR_kwDOVBg7g88AAAABVGGmvw: already fixed in 5fae382, 6dc1f42 and 805aa58
Thread: PRR_kwDOVBg7g88AAAABVHLzTg: already fixed in the inline threads' commits
Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>

From 58c49b2db730006d19bc6c9d64be003862d0bc47 Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Fri, 30 Oct 2026 22:18:16 -0700
Subject: [PATCH 48/53] Refuse a CLI judge's temperature in benchmark run
 before any answer

The judge refuses a temperature for claude-cli, but only after
benchmark run had generated every answer. benchmark run and batch
now run the same check first, exit 2, as dokimi run does.

Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/dokimi/commands/benchmark.py | 13 +++++++++++++
 dokimi/tests/test_llm_cli.py        | 15 +++++++++++++++
 2 files changed, 28 insertions(+)

diff --git a/dokimi/dokimi/commands/benchmark.py b/dokimi/dokimi/commands/benchmark.py
index c69ce150b..2bce0174e 100644
--- a/dokimi/dokimi/commands/benchmark.py
+++ b/dokimi/dokimi/commands/benchmark.py
@@ -27,6 +27,17 @@
 )
 from dokimi.commands.library import _load_registry
 from dokimi.exit_codes import EXIT_USAGE
+from dokimi.llm_cli import SamplingError, check_sampling
+
+
+def _refuse_judge_sampling(args: argparse.Namespace) -> None:
+    """Exit 2 before any answer is paid for when a CLI judge is given a temperature (the judge would refuse later)."""
+    try:
+        check_sampling(getattr(args, "judge_provider", DEFAULT_JUDGE_PROVIDER),
+                       getattr(args, "judge_temperature", None), None)
+    except SamplingError as exc:
+        print(f"Error: {exc}", file=sys.stderr)
+        sys.exit(EXIT_USAGE)
 
 
 def _run_single_library(entry, output_dir: str, model: str, provider: str, judge_model: str, judge_provider: str = DEFAULT_JUDGE_PROVIDER, doc_source_override=None, max_tokens_per_question: int = DEFAULT_MAX_TOKENS_PER_QUESTION, force_regen: bool = False, concurrency: int = DEFAULT_CONCURRENCY, questions_from=None, questions_root=None, judge_temperature=None, judge_placeholder=False, registry=None) -> dict:
@@ -304,6 +315,7 @@ def refuse(message: str) -> None:
 def cmd_benchmark_run(args: argparse.Namespace) -> None:
     """Run full evaluation pipeline for a single registered library."""
     import statistics as _stats
+    _refuse_judge_sampling(args)
     registry = _load_registry(args)
     try:
         entry = registry.get(args.library)
@@ -384,6 +396,7 @@ def cmd_benchmark_run(args: argparse.Namespace) -> None:
 
 def cmd_benchmark_batch(args: argparse.Namespace) -> None:
     """Run evaluation pipeline for multiple registered libraries."""
+    _refuse_judge_sampling(args)
     registry = _load_registry(args)
 
     if args.all_libraries or not args.libraries:
diff --git a/dokimi/tests/test_llm_cli.py b/dokimi/tests/test_llm_cli.py
index c70b40ad6..8986ff57f 100644
--- a/dokimi/tests/test_llm_cli.py
+++ b/dokimi/tests/test_llm_cli.py
@@ -831,6 +831,21 @@ def test_evaluate_with_a_cli_judge_given_a_temperature_exits_2_like_the_other_ju
     assert "Traceback" not in err
 
 
+def test_benchmark_run_with_a_cli_judge_given_a_temperature_refuses_before_any_answer(monkeypatch, capsys):
+    """The refusal comes before the pipeline is built, so no answer is paid for."""
+    from dokimi import cli, orchestrator
+
+    built = []
+    monkeypatch.setattr(orchestrator, "EvaluationPipeline", lambda **kwargs: built.append(kwargs))
+    monkeypatch.setattr("sys.argv", ["dokimi", "benchmark", "run", "--library", "onetbb",
+                                     "--judge-provider", "claude-cli", "--judge-temperature", "0"])
+    with pytest.raises(SystemExit) as exc:
+        cli.main()
+    err = capsys.readouterr().err
+    assert exc.value.code == 2 and err.startswith("Error: claude-cli takes no temperature")
+    assert built == []
+
+
 def test_the_main_model_falls_back_to_most_output_when_the_requested_family_is_unknown():
     usage = {"helper-a": {"outputTokens": 5}, "helper-b": {"outputTokens": 500}}
     assert llm_cli._main_model(usage, "claude-fable-5-1") == "helper-b"

From 609a4342c25ed966242e14423e049ed7cabeca42 Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Fri, 30 Oct 2026 22:18:16 -0700
Subject: [PATCH 49/53] Reword the unjudged-miss comment in diagnose()

Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/dokimi/eval/diagnoser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dokimi/dokimi/eval/diagnoser.py b/dokimi/dokimi/eval/diagnoser.py
index af1e8e0b1..1283554f0 100644
--- a/dokimi/dokimi/eval/diagnoser.py
+++ b/dokimi/dokimi/eval/diagnoser.py
@@ -100,7 +100,7 @@ def diagnose(
         - ``evidence``: dict of signals used to reach the diagnosis
     """
     if delta is None:
-        # A retrieval miss is left unjudged, so it has no delta; it is still an
+        # A retrieval miss the judge did not score has no delta; it is still an
         # empty retrieval, not a missing or errored answer.
         unjudged = not context_scored
         if unjudged and context_answer is not None and not context_answer.error and context_answer.doc_source in NO_RETRIEVAL_SOURCES:

From a6a74c46185df09b46c73427c0712bba267d91e6 Mon Sep 17 00:00:00 2001
From: Alexander Rybkin <alexander.rybkin@intel.com>
Date: Fri, 30 Oct 2026 22:24:21 -0700
Subject: [PATCH 50/53] fix(gate): keep stored judge-error rows out of the
 retrieval coverage count

Eval files written before error rows carried retrieval_hit turned a measured
hit rate into "Not measured" (and a failing rate into a pass) once the
denominator became every evaluation. Such a row now counts as neither
measured nor unrecorded, as before; new error rows still count.

Signed-off-by: Alexander Rybkin <alexander.rybkin@intel.com>
---
 .../20261028_120000_napetrov_stack-p-step2.md            | 3 ++-
 dokimi/dokimi/gate/trust_gate.py                         | 3 ++-
 dokimi/tests/test_trust_gate.py                          | 9 +++++++++
 3 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md b/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md
index 78bc15912..8aac22f30 100644
--- a/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md
+++ b/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md
@@ -21,7 +21,8 @@
 - **Trust gate** — `retrieval_hit_rate` counts every evaluation's recorded retrieval outcome,
   not only the ones scored in both arms, so unjudged misses still count against the rate, and a
   judge-error row keeps the outcome of its answer. The rate is shown even when no question is
-  scored in both arms. The thresholds are unchanged.
+  scored in both arms. A judge-error row of an older eval file, which has no `retrieval_hit`,
+  is left out of that count as before. The thresholds are unchanged.
 - **Zero-score check** — a judge-error row on a retrieval miss now counts as unusable in
   `zero_score_fraction` (a judge error on a retrieval hit is still left out), so a run can fail
   that fail-severity check where intel/main passed it.
diff --git a/dokimi/dokimi/gate/trust_gate.py b/dokimi/dokimi/gate/trust_gate.py
index 1398f0c8c..72b8ad912 100644
--- a/dokimi/dokimi/gate/trust_gate.py
+++ b/dokimi/dokimi/gate/trust_gate.py
@@ -107,7 +107,8 @@ def _retrieval_check(every: List[Record], cfg: Dict[str, Any]) -> TrustCheck:
     document is not judged by default, and it still records that it missed.
     """
     measured = [h for r in every if (h := r.retrieval_hit) is not None]
-    n_all = len(every)
+    # A legacy judge-error row has no retrieval_hit: left out of the count, as before.
+    n_all = sum(1 for r in every if not (r.error is not None and r.retrieval_hit is None))
     if measured and len(measured) / n_all < cfg["min_retrieval_coverage"]:
         return TrustCheck(
             name="retrieval_hit_rate",
diff --git a/dokimi/tests/test_trust_gate.py b/dokimi/tests/test_trust_gate.py
index cc17ad1b4..81382eae9 100644
--- a/dokimi/tests/test_trust_gate.py
+++ b/dokimi/tests/test_trust_gate.py
@@ -366,6 +366,15 @@ def test_coverage_below_the_default_floor_is_not_measured(self):
         assert not _check(v, "retrieval_hit_rate").measured
         assert v.not_measured == ["retrieval_hit_rate"]
 
+    def test_legacy_judge_error_rows_leave_a_stored_rate_measured(self):
+        # A judge-error row as eval score wrote it before error rows carried retrieval_hit.
+        legacy_error = {"question_id": "q", "error": "judge failed", "with_docs": None,
+                        "without_docs": None, "delta": None}
+        evals = ([_row(70, 50, hit=True)] * 5 + [_row(70, 50, hit=False)] * 10
+                 + [legacy_error] * 5)
+        check = _check(evaluate_trust(evals), "retrieval_hit_rate")
+        assert (check.measured, check.value, check.passed) == (True, 0.333, False)
+
     def test_a_zero_baseline_counts_as_zero(self):
         check = _check(evaluate_trust(_evals(10, with_score=40, without_score=0)), "min_delta")
         assert check.value == 40.0

From 3d7f8c5805849231ea7db16c7ffcd04cc3e204bd Mon Sep 17 00:00:00 2001
From: Alexander Rybkin <alexander.rybkin@intel.com>
Date: Fri, 30 Oct 2026 22:25:52 -0700
Subject: [PATCH 51/53] fix(answers): refuse a top_k below 1 now that it
 reaches the doc source

top_k is sent as max_results, so answers generate --top-k 0 (or a config
retrieval.top_k of 0, "3" or true) made every answer an unjudged placeholder
and still exited 0. The flag takes positive_int, as dokimi evaluate does, and
a bad config value exits 2 before any question is answered.

Signed-off-by: Alexander Rybkin <alexander.rybkin@intel.com>
---
 .../20261028_120000_napetrov_stack-p-step2.md |  2 ++
 dokimi/docs/reference/cli-interface.json      |  2 +-
 dokimi/dokimi/commands/answers.py             |  6 ++++-
 dokimi/tests/test_answers_command.py          | 25 +++++++++++++++++++
 4 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md b/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md
index 8aac22f30..3e4866885 100644
--- a/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md
+++ b/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md
@@ -37,6 +37,8 @@
 
 - **Stack P no longer ignores `top_k`** — `Answerer` passes `max_results=top_k` to the doc
   source; the context arm's retrieval request is now the one the arms stack makes.
+  `answers generate --top-k` takes a positive integer, as `dokimi evaluate` does, and a config
+  `retrieval.top_k` that is not one exits 2.
 - **Library id from the registry** — `EvaluationPipeline` resolves the library id through
   `resolve_library_id_for_product` (explicit id, then an `mcp:` source's `;id=`, then `context7_id`,
   then the client), in the
diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json
index 17e1ff039..70d10c11f 100644
--- a/dokimi/docs/reference/cli-interface.json
+++ b/dokimi/docs/reference/cli-interface.json
@@ -344,7 +344,7 @@
               "nargs": null,
               "position": null,
               "required": false,
-              "type": "int"
+              "type": "dokimi.cli.build_parser.<locals>.positive_int#faf57db5ff9f"
             }
           ],
           "path": "dokimi answers generate",
diff --git a/dokimi/dokimi/commands/answers.py b/dokimi/dokimi/commands/answers.py
index ad2601620..ecb1401f0 100644
--- a/dokimi/dokimi/commands/answers.py
+++ b/dokimi/dokimi/commands/answers.py
@@ -85,6 +85,10 @@ def cmd_answers_generate(args: argparse.Namespace) -> None:
     _retrieval_cfg = config.get("retrieval") or {}
     if args.top_k is None:
         args.top_k = _retrieval_cfg.get("top_k", DEFAULT_RETRIEVAL_TOP_K)
+        if isinstance(args.top_k, bool) or not isinstance(args.top_k, int) or args.top_k < 1:
+            print(f"Error: retrieval.top_k in the config must be a positive integer, "
+                  f"got {args.top_k!r}", file=sys.stderr)
+            sys.exit(EXIT_USAGE)
 
     questions = _load_questions(args.questions)
     print(f"Loaded {len(questions)} questions from {args.questions}")
@@ -162,7 +166,7 @@ def register(sub, positive_int) -> None:
     gen_a_p.add_argument("--model", default=DEFAULT_ANSWER_MODEL, help="LLM model for answering")
     gen_a_p.add_argument("--provider", default=DEFAULT_ANSWER_PROVIDER, choices=ANSWER_PROVIDERS)
     gen_a_p.add_argument("--max-tokens", type=int, default=DEFAULT_MAX_TOKENS_PER_QUESTION, help="Max tokens to retrieve per question")
-    gen_a_p.add_argument("--top-k", type=int, default=None, help=f"Number of docs to retrieve before reranking (default from config/products.yaml retrieval.top_k or {DEFAULT_RETRIEVAL_TOP_K})")
+    gen_a_p.add_argument("--top-k", type=positive_int, default=None, help=f"Number of docs to retrieve before reranking (default from config/products.yaml retrieval.top_k or {DEFAULT_RETRIEVAL_TOP_K})")
     gen_a_p.add_argument("--rerank-threshold", type=float, default=0.3, help="Min relevance score (0-1) to keep docs")
     gen_a_p.add_argument("--debug-retrieval", action="store_true", help="Include retrieval metadata in output")
     gen_a_p.add_argument("--concurrency", type=positive_int, default=DEFAULT_CONCURRENCY, help="Parallel API calls (default: 5)")
diff --git a/dokimi/tests/test_answers_command.py b/dokimi/tests/test_answers_command.py
index 1eb720361..e806e470f 100644
--- a/dokimi/tests/test_answers_command.py
+++ b/dokimi/tests/test_answers_command.py
@@ -308,3 +308,28 @@ def test_generate_unreadable_questions_exits_2_naming_the_path(tmp_path, monkeyp
         answers.cmd_answers_generate(_args(questions=str(path)))
     assert exc.value.code == 2
     assert f"Error: cannot read {path}" in capsys.readouterr().err
+
+
+@pytest.mark.parametrize("value", ["0", "-1", '"3"', "true"])
+def test_a_config_top_k_that_is_not_a_positive_int_exits_2(tmp_path, monkeypatch, capsys, value):
+    # top_k reaches the doc source as max_results: 0 would make every answer a placeholder.
+    path = tmp_path / "products.yaml"
+    path.write_text(f"retrieval:\n  top_k: {value}\n", encoding="utf-8")
+    _point_config(monkeypatch, path)
+    monkeypatch.setattr(dokimi.questions.artifact, "load_questions",
+                        lambda path: pytest.fail("questions loaded after a bad top_k"))
+    with pytest.raises(SystemExit) as exc:
+        answers.cmd_answers_generate(_args())
+    assert exc.value.code == 2
+    assert "retrieval.top_k in the config must be a positive integer" in capsys.readouterr().err
+
+
+@pytest.mark.parametrize("value", ["0", "-1"])
+def test_top_k_flag_below_one_is_refused(capsys, value):
+    from dokimi.cli import build_parser
+
+    with pytest.raises(SystemExit) as exc:
+        build_parser().parse_args(["answers", "generate", "--product", "L",
+                                   "--questions", "q.json", "--top-k", value])
+    assert exc.value.code == 2
+    assert "--top-k" in capsys.readouterr().err

From f34aa62e5f62c458bbb1dd9da1932724d480a6a4 Mon Sep 17 00:00:00 2001
From: Alexander Rybkin <alexander.rybkin@intel.com>
Date: Fri, 30 Oct 2026 22:27:27 -0700
Subject: [PATCH 52/53] fix(eval): refuse a malformed answers method_version
 before judging

eval score and the pipeline copied the answers' method_version into the eval
unchecked: "2", true or 3.5 failed eval validation after judge calls were
paid for and left no file, 0 became 1 and -7 went through. A stamp that is
not a positive int now exits 2 (eval score) before any judge call.

Signed-off-by: Alexander Rybkin <alexander.rybkin@intel.com>
---
 .../20261028_120000_napetrov_stack-p-step2.md |  3 ++-
 dokimi/dokimi/commands/evaluate.py            |  9 +++++--
 dokimi/dokimi/defaults.py                     | 13 ++++++++++
 dokimi/dokimi/orchestrator/pipeline.py        |  5 ++--
 .../tests/test_eval_score_method_version.py   | 24 +++++++++++++++++++
 5 files changed, 49 insertions(+), 5 deletions(-)

diff --git a/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md b/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md
index 3e4866885..9f014eb3c 100644
--- a/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md
+++ b/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md
@@ -31,7 +31,8 @@
   not yet normalize or enforce the version, so it does not yet stop a comparison of mixed methods.
   `eval score` stamps its eval with the version of the answers file it judges, so legacy answers
   re-judged today stay version 1. An eval artifact also carries `judge_placeholder`, so two files
-  of one version that differ in it are told apart.
+  of one version that differ in it are told apart. A `method_version` that is not a positive
+  integer is refused before any judge call (`eval score` exits 2).
 
 ### Fixed
 
diff --git a/dokimi/dokimi/commands/evaluate.py b/dokimi/dokimi/commands/evaluate.py
index 48accf2f5..fbc68bd11 100644
--- a/dokimi/dokimi/commands/evaluate.py
+++ b/dokimi/dokimi/commands/evaluate.py
@@ -18,7 +18,7 @@
     DEFAULT_JUDGE_PROVIDER,
     JUDGE_MODEL_FOR_PROVIDER,
     JUDGE_MODEL_HELP,
-    LEGACY_ANSWERING_METHOD_VERSION,
+    answers_method_version,
 )
 from dokimi.eval.judge import JUDGE_PROVIDERS
 from dokimi.eval.judge_validity import DEFAULT_MIN_ACCURACY, NoUsablePairs
@@ -202,6 +202,11 @@ def cmd_eval_score(args: argparse.Namespace) -> None:
             context="eval-score",
         )
 
+    try:
+        method_version = answers_method_version(run.meta)
+    except ValueError as exc:  # refused before any judge call is paid for
+        print(f"Error: {args.answers}: {exc}", file=sys.stderr)
+        sys.exit(EXIT_USAGE)
     print(f"Evaluating with judge: {args.judge_provider}/{args.judge_model}")
 
     # Evaluate
@@ -211,7 +216,7 @@ def cmd_eval_score(args: argparse.Namespace) -> None:
         temperature=args.judge_temperature,
         judge_placeholder=getattr(args, "judge_placeholder", False),
         # The eval says which answering method made the answers it judges; an unstamped file is legacy.
-        answers_method_version=run.meta.get("method_version") or LEGACY_ANSWERING_METHOD_VERSION,
+        answers_method_version=method_version,
         run_metadata={
             "question_model": run.meta.get("question_model", "unknown"),
             "question_provider": run.meta.get("question_provider", "unknown"),
diff --git a/dokimi/dokimi/defaults.py b/dokimi/dokimi/defaults.py
index f583e9a02..b0b0d26c2 100644
--- a/dokimi/dokimi/defaults.py
+++ b/dokimi/dokimi/defaults.py
@@ -20,6 +20,7 @@
 """
 
 import os
+from typing import Any, Mapping
 
 DEFAULT_ANSWER_PROVIDER = "openai"
 DEFAULT_ANSWER_MODEL = "gpt-6.1-sol"
@@ -97,6 +98,18 @@ class _JudgeModelForProvider(str):
 ANSWERING_METHOD_VERSION = 2
 #: The version of an artifact that carries no stamp (written before ANSWERING_METHOD_VERSION existed).
 LEGACY_ANSWERING_METHOD_VERSION = 1
+
+
+def answers_method_version(meta: Mapping[str, Any]) -> int:
+    """The method_version an answers artifact carries; absent or null is legacy, else a positive int."""
+    value = meta.get("method_version")
+    if value is None:
+        return LEGACY_ANSWERING_METHOD_VERSION
+    if isinstance(value, bool) or not isinstance(value, int) or value < 1:
+        raise ValueError(f"method_version must be a positive integer, got {value!r}")
+    return value
+
+
 DEFAULT_MAX_TOKENS_PER_QUESTION = 4000
 DEFAULT_CONCURRENCY = 5
 DEFAULT_RESULTS_SUFFIX = "_final"
diff --git a/dokimi/dokimi/orchestrator/pipeline.py b/dokimi/dokimi/orchestrator/pipeline.py
index 548519bd6..69d541418 100644
--- a/dokimi/dokimi/orchestrator/pipeline.py
+++ b/dokimi/dokimi/orchestrator/pipeline.py
@@ -9,7 +9,8 @@
 from typing import List, Dict, Any, Optional
 from pathlib import Path
 
-from dokimi.defaults import (ANSWERING_METHOD_VERSION, LEGACY_ANSWERING_METHOD_VERSION, DEFAULT_ANSWER_MODEL, DEFAULT_ANSWER_PROVIDER, DEFAULT_JUDGE_MODEL, DEFAULT_JUDGE_PROVIDER, DEFAULT_RETRIEVAL_TOP_K)
+from dokimi.defaults import (ANSWERING_METHOD_VERSION, DEFAULT_ANSWER_MODEL, DEFAULT_ANSWER_PROVIDER, DEFAULT_JUDGE_MODEL, DEFAULT_JUDGE_PROVIDER, DEFAULT_RETRIEVAL_TOP_K,
+                             answers_method_version as answers_method_version_of)
 from dokimi.eval.stats import paired_summary
 from dokimi.mcp.factory import create_doc_source_client, resolve_library_id_for_product
 
@@ -688,7 +689,7 @@ def _evaluate_answers(self, answers: List[Dict[str, Any]] | Dict[str, Any],
         if answers and isinstance(answers, list):
             answers_method_version = ANSWERING_METHOD_VERSION
         else:
-            answers_method_version = run.meta.get("method_version") or LEGACY_ANSWERING_METHOD_VERSION
+            answers_method_version = answers_method_version_of(run.meta)
         
         # Evaluate
         judge = Judge(
diff --git a/dokimi/tests/test_eval_score_method_version.py b/dokimi/tests/test_eval_score_method_version.py
index 4649341c9..9c8c30a7c 100644
--- a/dokimi/tests/test_eval_score_method_version.py
+++ b/dokimi/tests/test_eval_score_method_version.py
@@ -97,3 +97,27 @@ def test_pipeline_judges_fresh_in_memory_answers_as_the_current_version(monkeypa
     from dokimi.defaults import ANSWERING_METHOD_VERSION
 
     assert _pipeline_judge_version(monkeypatch, tmp_path, ANSWERS) == ANSWERING_METHOD_VERSION
+
+
+@pytest.mark.parametrize("bad", ["2", True, 3.5, [1], 0, -7])
+def test_a_malformed_stamp_exits_2_before_any_judge_call(tmp_path, judge_calls, capsys, bad):
+    """main ignored the field; writing it unchecked failed eval validation after paid calls."""
+    with pytest.raises(SystemExit) as exc:
+        _eval_score(tmp_path, {"method_version": bad, "answers": ANSWERS})
+
+    assert exc.value.code == 2
+    assert judge_calls == []
+    assert not (tmp_path / "eval.json").exists()
+    assert "method_version must be a positive integer" in capsys.readouterr().err
+
+
+def test_a_null_stamp_is_legacy(tmp_path, judge_calls):
+    assert _eval_score(tmp_path, {"method_version": None, "answers": ANSWERS})["method_version"] == 1
+
+
+def test_pipeline_refuses_an_answers_file_with_a_malformed_stamp(monkeypatch, tmp_path):
+    answers_file = tmp_path / "answers.json"
+    answers_file.write_text(json.dumps({"method_version": "2", "answers": ANSWERS}), encoding="utf-8")
+
+    with pytest.raises(ValueError, match="method_version must be a positive integer"):
+        _pipeline_judge_version(monkeypatch, tmp_path, [], answers_file)

From b945e44f045455759bf4edbfc56b3f5897774a66 Mon Sep 17 00:00:00 2001
From: Alexander Rybkin <alexander.rybkin@intel.com>
Date: Fri, 30 Oct 2026 22:29:01 -0700
Subject: [PATCH 53/53] fix(pipeline): key the topics cache on the doc source
 too

The library id now comes from the registry for local:, url: and okf:
sources as well, and .cache/topics was keyed on that id alone, so a second
doc set of one registered product silently got the first set's topics.

Signed-off-by: Alexander Rybkin <alexander.rybkin@intel.com>
---
 .../20261028_120000_napetrov_stack-p-step2.md |  2 ++
 dokimi/dokimi/orchestrator/pipeline.py        |  5 +++-
 dokimi/tests/test_pipeline_library_id.py      | 27 +++++++++++++++++++
 3 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md b/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md
index 9f014eb3c..7a26ff298 100644
--- a/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md
+++ b/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md
@@ -45,6 +45,8 @@
   then the client), in the
   registry that `benchmark run` and `benchmark batch` loaded from `--registry`. A registered
   product without `context7_id` now fails for a client that needs a declared id.
+  The question step's topics cache (`.cache/topics`) is keyed on the doc source as well, so
+  two doc sets of one product (two `local:` dirs, say) no longer share their topics.
 
 ### Removed
 
diff --git a/dokimi/dokimi/orchestrator/pipeline.py b/dokimi/dokimi/orchestrator/pipeline.py
index 69d541418..2523c225c 100644
--- a/dokimi/dokimi/orchestrator/pipeline.py
+++ b/dokimi/dokimi/orchestrator/pipeline.py
@@ -478,7 +478,10 @@ def _generate_questions(self, personas: Dict[str, Any]) -> List[Dict[str, Any]]:
         mcp_client = create_doc_source_client(self.doc_source)
         library_id = self._library_id(mcp_client)
 
-        extractor = RagasSeedExtractor(mcp_client=mcp_client, cache_dir=Path(".cache/topics"))
+        # The id is the registry's for every source, so the topics cache is also keyed on the source.
+        source_key = hashlib.sha256(self.doc_source.encode("utf-8")).hexdigest()[:16]
+        extractor = RagasSeedExtractor(mcp_client=mcp_client,
+                                       cache_dir=Path(".cache/topics") / source_key)
         topics = extractor.extract_topics(
             library_id=library_id,
             library_name=self.product,
diff --git a/dokimi/tests/test_pipeline_library_id.py b/dokimi/tests/test_pipeline_library_id.py
index 854130bfa..64d07be0e 100644
--- a/dokimi/tests/test_pipeline_library_id.py
+++ b/dokimi/tests/test_pipeline_library_id.py
@@ -122,3 +122,30 @@ def extract_topics(self, library_id, **kwargs):
         _pipeline(tmp_path, registry)._generate_questions({})
 
     assert seen["library_id"] == "org/declared"
+
+
+def test_two_local_doc_sets_of_one_product_get_their_own_topics(tmp_path, registry, monkeypatch):
+    """Both runs resolve the registry's id, so a topics cache keyed on the id alone gave the
+    second doc set the first one's topics."""
+    import dokimi.questions as questions
+
+    seen = []
+
+    class _Generator:
+        def __init__(self, **kwargs):
+            pass
+
+        def generate_questions(self, topics, **kwargs):
+            seen.append(sorted(topics))
+            return []
+
+    monkeypatch.setattr(questions, "QuestionGenerator", _Generator)
+    monkeypatch.chdir(tmp_path)
+    docs = {"docsA": "`parallel_for` `task_arena`", "docsB": "`flow_graph` `concurrent_queue`"}
+    for name, words in docs.items():
+        (tmp_path / name).mkdir()
+        (tmp_path / name / "api.md").write_text(f"My Lib API overview {words}\n", encoding="utf-8")
+        pipeline = _pipeline(tmp_path, registry, doc_source=f"local:{tmp_path / name}")
+        pipeline._generate_questions({"personas": []})
+
+    assert seen == [["parallel_for", "task_arena"], ["concurrent_queue", "flow_graph"]]