diff --git a/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md b/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md new file mode 100644 index 00000000..7a26ff29 --- /dev/null +++ b/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md @@ -0,0 +1,54 @@ +### Added + +- **`benchmark run --judge-temperature`** — a new flag: it sets the judge's sampling + temperature and is recorded as `judge_temperature` in the eval output. Before this change the + flag did not exist, and `benchmark run` rejected it as an unknown argument. + +### Changed + +- **Stack P scores move (#228 step 2)** — the context arm now asks the doc source for the + `top_k` docs the command was given: `benchmark run` asks for 3 (`DEFAULT_RETRIEVAL_TOP_K`; it + has no `--top-k`), `dokimi evaluate` and `answers generate` for their `--top-k` (default 3). + Before, they left `max_results` at the client default of 5. The doc library id resolves + through the registry's `context7_id` before the client's own resolver, in the `--registry` + that was loaded. A question with no document (no hit, or the reranker raised) is no longer judged on + its `[FALLBACK: ...]` placeholder by default, so it drops out of the paired statistics; the + record and its `retrieval_hit` stay, its diagnosis stays `empty_retrieval`, and the run + summary says how many questions it left out. Pass `--judge-placeholder` to judge the + placeholder again (this restores only that part of the old method). `benchmark run`, + `evaluate` and `eval score` take the same flag; `benchmark batch` has none and never judges + the placeholder. Scores from before this change are not comparable with new ones. +- **Trust gate** — `retrieval_hit_rate` counts every evaluation's recorded retrieval outcome, + not only the ones scored in both arms, so unjudged misses still count against the rate, and a + judge-error row keeps the outcome of its answer. The rate is shown even when no question is + scored in both arms. A judge-error row of an older eval file, which has no `retrieval_hit`, + is left out of that count as before. The thresholds are unchanged. +- **Zero-score check** — a judge-error row on a retrieval miss now counts as unusable in + `zero_score_fraction` (a judge error on a retrieval hit is still left out), so a run can fail + that fail-severity check where intel/main passed it. +- **Stamped artifacts** — the answers and eval artifacts carry `method_version` (2). Artifacts + from before this change have no stamp and represent the legacy method (version 1); readers do + not yet normalize or enforce the version, so it does not yet stop a comparison of mixed methods. + `eval score` stamps its eval with the version of the answers file it judges, so legacy answers + re-judged today stay version 1. An eval artifact also carries `judge_placeholder`, so two files + of one version that differ in it are told apart. A `method_version` that is not a positive + integer is refused before any judge call (`eval score` exits 2). + +### Fixed + +- **Stack P no longer ignores `top_k`** — `Answerer` passes `max_results=top_k` to the doc + source; the context arm's retrieval request is now the one the arms stack makes. + `answers generate --top-k` takes a positive integer, as `dokimi evaluate` does, and a config + `retrieval.top_k` that is not one exits 2. +- **Library id from the registry** — `EvaluationPipeline` resolves the library id through + `resolve_library_id_for_product` (explicit id, then an `mcp:` source's `;id=`, then `context7_id`, + then the client), in the + registry that `benchmark run` and `benchmark batch` loaded from `--registry`. A registered + product without `context7_id` now fails for a client that needs a declared id. + The question step's topics cache (`.cache/topics`) is keyed on the doc source as well, so + two doc sets of one product (two `local:` dirs, say) no longer share their topics. + +### Removed + +- Dead code in the answering path: the unused `Answerer.llm` client and the unreachable + "no documents" branch of `_generate_with_docs`. diff --git a/dokimi/docs/methodology.md b/dokimi/docs/methodology.md index 952f3ca5..a5a43b9c 100644 --- a/dokimi/docs/methodology.md +++ b/dokimi/docs/methodology.md @@ -199,11 +199,11 @@ RAGAS meta-evaluation can be run after answer generation to add faithfulness and Every serious run should answer "Can we trust this run?" before interpreting deltas. The trust gate checks: -- minimum evaluated question count, -- zero-score fraction, -- minimum `with_docs` average, -- average delta, -- score coefficient of variation, +- minimum evaluated question count (both arms scored; a question with no document is not judged by default, so it is not counted here), +- zero-score fraction (computed when at least one question is scored in both arms; over those paired questions plus retrieval misses with a scored baseline or a judge error, which count as unusable), +- minimum `with_docs` average (both-arm-scored questions only), +- average delta (both-arm-scored questions only), +- score coefficient of variation (both-arm-scored questions only), - inter-rater agreement when panel data exists, - retrieval hit rate — the share of questions where at least one document reached the `with_docs` prompt, from the per-evaluation `retrieval_hit` the judge and panel record; eval files judged before that field existed show it as not measured, and so do files where fewer than 80% of evaluations record it (`min_retrieval_coverage`), since the rate would then describe a subset, - multi-run variance when `--multi-run N` is used. diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index c9eb437d..70d10c11 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -344,7 +344,7 @@ "nargs": null, "position": null, "required": false, - "type": "int" + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" } ], "path": "dokimi answers generate", @@ -2299,6 +2299,22 @@ "required": false, "type": null }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--judge-placeholder", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, { "action": "_StoreAction", "choices": [ @@ -2323,6 +2339,22 @@ "required": false, "type": null }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--judge-temperature", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "float" + }, { "action": "_StoreAction", "choices": null, @@ -5805,6 +5837,22 @@ "required": false, "type": null }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--judge-placeholder", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, { "action": "_StoreAction", "choices": [ @@ -6015,6 +6063,22 @@ "required": false, "type": null }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--judge-placeholder", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, { "action": "_StoreAction", "choices": [ diff --git a/dokimi/dokimi/commands/answers.py b/dokimi/dokimi/commands/answers.py index ad260162..ecb1401f 100644 --- a/dokimi/dokimi/commands/answers.py +++ b/dokimi/dokimi/commands/answers.py @@ -85,6 +85,10 @@ def cmd_answers_generate(args: argparse.Namespace) -> None: _retrieval_cfg = config.get("retrieval") or {} if args.top_k is None: args.top_k = _retrieval_cfg.get("top_k", DEFAULT_RETRIEVAL_TOP_K) + if isinstance(args.top_k, bool) or not isinstance(args.top_k, int) or args.top_k < 1: + print(f"Error: retrieval.top_k in the config must be a positive integer, " + f"got {args.top_k!r}", file=sys.stderr) + sys.exit(EXIT_USAGE) questions = _load_questions(args.questions) print(f"Loaded {len(questions)} questions from {args.questions}") @@ -162,7 +166,7 @@ def register(sub, positive_int) -> None: gen_a_p.add_argument("--model", default=DEFAULT_ANSWER_MODEL, help="LLM model for answering") gen_a_p.add_argument("--provider", default=DEFAULT_ANSWER_PROVIDER, choices=ANSWER_PROVIDERS) gen_a_p.add_argument("--max-tokens", type=int, default=DEFAULT_MAX_TOKENS_PER_QUESTION, help="Max tokens to retrieve per question") - gen_a_p.add_argument("--top-k", type=int, default=None, help=f"Number of docs to retrieve before reranking (default from config/products.yaml retrieval.top_k or {DEFAULT_RETRIEVAL_TOP_K})") + gen_a_p.add_argument("--top-k", type=positive_int, default=None, help=f"Number of docs to retrieve before reranking (default from config/products.yaml retrieval.top_k or {DEFAULT_RETRIEVAL_TOP_K})") gen_a_p.add_argument("--rerank-threshold", type=float, default=0.3, help="Min relevance score (0-1) to keep docs") gen_a_p.add_argument("--debug-retrieval", action="store_true", help="Include retrieval metadata in output") gen_a_p.add_argument("--concurrency", type=positive_int, default=DEFAULT_CONCURRENCY, help="Parallel API calls (default: 5)") diff --git a/dokimi/dokimi/commands/benchmark.py b/dokimi/dokimi/commands/benchmark.py index 44b1c888..2bce0174 100644 --- a/dokimi/dokimi/commands/benchmark.py +++ b/dokimi/dokimi/commands/benchmark.py @@ -27,9 +27,20 @@ ) from dokimi.commands.library import _load_registry from dokimi.exit_codes import EXIT_USAGE +from dokimi.llm_cli import SamplingError, check_sampling -def _run_single_library(entry, output_dir: str, model: str, provider: str, judge_model: str, judge_provider: str = DEFAULT_JUDGE_PROVIDER, doc_source_override=None, max_tokens_per_question: int = DEFAULT_MAX_TOKENS_PER_QUESTION, force_regen: bool = False, concurrency: int = DEFAULT_CONCURRENCY, questions_from=None, questions_root=None) -> dict: +def _refuse_judge_sampling(args: argparse.Namespace) -> None: + """Exit 2 before any answer is paid for when a CLI judge is given a temperature (the judge would refuse later).""" + try: + check_sampling(getattr(args, "judge_provider", DEFAULT_JUDGE_PROVIDER), + getattr(args, "judge_temperature", None), None) + except SamplingError as exc: + print(f"Error: {exc}", file=sys.stderr) + sys.exit(EXIT_USAGE) + + +def _run_single_library(entry, output_dir: str, model: str, provider: str, judge_model: str, judge_provider: str = DEFAULT_JUDGE_PROVIDER, doc_source_override=None, max_tokens_per_question: int = DEFAULT_MAX_TOKENS_PER_QUESTION, force_regen: bool = False, concurrency: int = DEFAULT_CONCURRENCY, questions_from=None, questions_root=None, judge_temperature=None, judge_placeholder=False, registry=None) -> dict: """Run full evaluation pipeline for one ProductEntry. Returns result dict.""" from dokimi.orchestrator import EvaluationPipeline from pathlib import Path as _Path @@ -60,12 +71,15 @@ def _run_single_library(entry, output_dir: str, model: str, provider: str, judge provider=provider, judge_model=judge_model, judge_provider=judge_provider, + judge_temperature=judge_temperature, + judge_placeholder=judge_placeholder, context7_id=entry.context7_id, doc_source=doc_source, max_tokens_per_question=max_tokens_per_question, force_regen=force_regen, questions_from=questions_from, product_key=entry.key, + registry=registry, questions_root=_Path(questions_root) if questions_root else None, ) result = pipeline.run(concurrency=concurrency) @@ -301,6 +315,7 @@ def refuse(message: str) -> None: def cmd_benchmark_run(args: argparse.Namespace) -> None: """Run full evaluation pipeline for a single registered library.""" import statistics as _stats + _refuse_judge_sampling(args) registry = _load_registry(args) try: entry = registry.get(args.library) @@ -334,6 +349,9 @@ def cmd_benchmark_run(args: argparse.Namespace) -> None: concurrency=concurrency, questions_from=questions_from, questions_root=questions_root, + judge_temperature=getattr(args, "judge_temperature", None), + judge_placeholder=getattr(args, "judge_placeholder", False), + registry=registry, ) else: print(f"\n🔁 Multi-run mode: {n_runs} evaluation passes") @@ -356,6 +374,9 @@ def cmd_benchmark_run(args: argparse.Namespace) -> None: concurrency=concurrency, questions_from=qfrom, questions_root=questions_root, + judge_temperature=getattr(args, "judge_temperature", None), + judge_placeholder=getattr(args, "judge_placeholder", False), + registry=registry, ) try: eval_summary = r["result"]["steps"]["evaluation"]["summary"] @@ -375,6 +396,7 @@ def cmd_benchmark_run(args: argparse.Namespace) -> None: def cmd_benchmark_batch(args: argparse.Namespace) -> None: """Run evaluation pipeline for multiple registered libraries.""" + _refuse_judge_sampling(args) registry = _load_registry(args) if args.all_libraries or not args.libraries: @@ -410,6 +432,7 @@ def cmd_benchmark_batch(args: argparse.Namespace) -> None: force_regen=getattr(args, "force_regen", False), concurrency=getattr(args, "concurrency", DEFAULT_CONCURRENCY), questions_root=getattr(args, "questions_root", None), + registry=registry, ) results.append(r) except Exception as exc: @@ -456,6 +479,10 @@ def register(sub, positive_int) -> None: help=JUDGE_MODEL_HELP) bench_run_p.add_argument("--judge-provider", default=DEFAULT_JUDGE_PROVIDER, dest="judge_provider", choices=JUDGE_PROVIDERS) + bench_run_p.add_argument("--judge-temperature", type=float, default=None, dest="judge_temperature", + help="Judge sampling temperature, recorded as judge_temperature in the eval output (default: provider default)") + bench_run_p.add_argument("--judge-placeholder", action="store_true", dest="judge_placeholder", + help="Judge the no-retrieval placeholder answers too (default: they are not scored)") bench_run_p.add_argument("--registry", default=None, help="Path to custom products.yaml") bench_run_p.add_argument("--max-tokens", type=positive_int, default=DEFAULT_MAX_TOKENS_PER_QUESTION, dest="max_tokens", help="Max tokens to retrieve per question from doc source (default: 4000)") diff --git a/dokimi/dokimi/commands/evaluate.py b/dokimi/dokimi/commands/evaluate.py index 18f1da85..dba67e5f 100644 --- a/dokimi/dokimi/commands/evaluate.py +++ b/dokimi/dokimi/commands/evaluate.py @@ -18,6 +18,7 @@ DEFAULT_JUDGE_PROVIDER, JUDGE_MODEL_FOR_PROVIDER, JUDGE_MODEL_HELP, + answers_method_version, ) from dokimi.eval.judge import JUDGE_PROVIDERS from dokimi.eval.judge_validity import DEFAULT_MIN_ACCURACY, NoUsablePairs @@ -198,6 +199,11 @@ def cmd_eval_score(args: argparse.Namespace) -> None: context="eval-score", ) + try: + method_version = answers_method_version(run.meta) + except ValueError as exc: # refused before any judge call is paid for + print(f"Error: {args.answers}: {exc}", file=sys.stderr) + sys.exit(EXIT_USAGE) print(f"Evaluating with judge: {args.judge_provider}/{args.judge_model}") # Evaluate @@ -205,6 +211,9 @@ def cmd_eval_score(args: argparse.Namespace) -> None: model=args.judge_model, provider=args.judge_provider, temperature=args.judge_temperature, + judge_placeholder=getattr(args, "judge_placeholder", False), + # The eval says which answering method made the answers it judges; an unstamped file is legacy. + answers_method_version=method_version, run_metadata={ "question_model": run.meta.get("question_model", "unknown"), "question_provider": run.meta.get("question_provider", "unknown"), @@ -480,6 +489,8 @@ def register(sub, positive_int) -> None: score_p.add_argument("--concurrency", type=positive_int, default=DEFAULT_CONCURRENCY, help="Parallel judge calls (default: 5)") score_p.add_argument("--judge-temperature", type=float, default=None, dest="judge_temperature", help="Judge sampling temperature, recorded as judge_temperature in the output") + score_p.add_argument("--judge-placeholder", action="store_true", dest="judge_placeholder", + help="Judge the no-retrieval placeholder answers too (default: they are not scored)") score_p.set_defaults(func=cmd_eval_score) # eval ragas diff --git a/dokimi/dokimi/commands/orchestrate.py b/dokimi/dokimi/commands/orchestrate.py index 32eed2c0..f550333d 100644 --- a/dokimi/dokimi/commands/orchestrate.py +++ b/dokimi/dokimi/commands/orchestrate.py @@ -68,6 +68,7 @@ def cmd_evaluate(args: argparse.Namespace) -> None: judge_model=args.judge_model, judge_provider=args.judge_provider, judge_temperature=args.judge_temperature, + judge_placeholder=getattr(args, "judge_placeholder", False), personas_count=args.personas_count, questions_per_topic=args.questions_per_topic, # Only pass top_k when explicitly set; otherwise let pipeline use config default @@ -130,6 +131,8 @@ def register(sub, positive_int) -> None: eval_p.add_argument("--judge-provider", default=DEFAULT_JUDGE_PROVIDER, choices=JUDGE_PROVIDERS) eval_p.add_argument("--judge-temperature", type=float, default=None, dest="judge_temperature", help="Judge sampling temperature, recorded as judge_temperature in the eval output") + eval_p.add_argument("--judge-placeholder", action="store_true", dest="judge_placeholder", + help="Judge the no-retrieval placeholder answers too (default: they are not scored)") eval_p.add_argument("--personas-count", type=positive_int, default=5, help="Target number of personas") eval_p.add_argument("--questions-per-topic", type=positive_int, default=2, help="Questions per topic per persona") eval_p.add_argument("--top-k", type=positive_int, default=None, help="Docs to retrieve before reranking (default from config/products.yaml retrieval.top_k)") diff --git a/dokimi/dokimi/defaults.py b/dokimi/dokimi/defaults.py index 10edc9e2..b0b0d26c 100644 --- a/dokimi/dokimi/defaults.py +++ b/dokimi/dokimi/defaults.py @@ -20,6 +20,7 @@ """ import os +from typing import Any, Mapping DEFAULT_ANSWER_PROVIDER = "openai" DEFAULT_ANSWER_MODEL = "gpt-6.1-sol" @@ -91,6 +92,24 @@ class _JudgeModelForProvider(str): # Decision (2026-03-04): one fixed semantic retrieval config. DEFAULT_RETRIEVAL_TOP_K = 3 + +# Answering/judging method version, stamped into answers and eval artifacts; bump it when a change moves scores. +# A later compare is meant to refuse mixed versions; an unstamped artifact predates it (#228 step 2) and is v1. +ANSWERING_METHOD_VERSION = 2 +#: The version of an artifact that carries no stamp (written before ANSWERING_METHOD_VERSION existed). +LEGACY_ANSWERING_METHOD_VERSION = 1 + + +def answers_method_version(meta: Mapping[str, Any]) -> int: + """The method_version an answers artifact carries; absent or null is legacy, else a positive int.""" + value = meta.get("method_version") + if value is None: + return LEGACY_ANSWERING_METHOD_VERSION + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise ValueError(f"method_version must be a positive integer, got {value!r}") + return value + + DEFAULT_MAX_TOKENS_PER_QUESTION = 4000 DEFAULT_CONCURRENCY = 5 DEFAULT_RESULTS_SUFFIX = "_final" diff --git a/dokimi/dokimi/eval/answerer.py b/dokimi/dokimi/eval/answerer.py index 5e2dca75..a80f4f03 100644 --- a/dokimi/dokimi/eval/answerer.py +++ b/dokimi/dokimi/eval/answerer.py @@ -15,10 +15,14 @@ from dokimi.utils import question_id from dokimi.artifacts import save_artifact -from dokimi.llm import ChatModel +from dokimi.defaults import ( + ANSWERING_METHOD_VERSION, + DEFAULT_ANSWER_MODEL, + DEFAULT_ANSWER_PROVIDER, + DEFAULT_RETRIEVAL_TOP_K, +) +from dokimi.llm import llm_call_with_usage, LANGCHAIN_AVAILABLE from dokimi.llm_cli import BACKENDS as CLI_BACKENDS -from dokimi.llm import llm_call_with_usage, ChatOpenAI, ChatAnthropic, LANGCHAIN_AVAILABLE -from dokimi.defaults import DEFAULT_ANSWER_MODEL, DEFAULT_ANSWER_PROVIDER from dokimi.metrics.usage import UsageRecord, to_record from .reranker import SimpleReranker, SentenceTransformerReranker, SENTENCE_TRANSFORMERS_AVAILABLE @@ -71,7 +75,7 @@ def __init__( model: str = DEFAULT_ANSWER_MODEL, provider: str = DEFAULT_ANSWER_PROVIDER, api_key: Optional[str] = None, - top_k: int = 5, + top_k: Optional[int] = None, rerank_threshold: float = 0.3, debug_retrieval: bool = False ): @@ -81,7 +85,7 @@ def __init__( model: LLM model name provider: "openai" or "anthropic" api_key: Optional API key - top_k: Number of docs to retrieve before reranking + top_k: Number of docs to retrieve before reranking (default DEFAULT_RETRIEVAL_TOP_K) rerank_threshold: Minimum relevance score (0-1) to keep docs debug_retrieval: If True, include detailed retrieval metadata in output """ @@ -96,20 +100,15 @@ def __init__( self.provider = provider self.api_key = api_key - self.llm: Optional[ChatModel] - if provider == "openai": - self.llm = ChatOpenAI(model=model, api_key=api_key) - elif provider == "anthropic": - self.llm = ChatAnthropic(model=model, api_key=api_key) - elif provider in ("google", "gemini", "google-vertex", "openrouter", - "amazon-bedrock", "vertex_ai", *CLI_BACKENDS): - self.llm = None # llm_call_with_usage handles these natively - else: + # The answer call goes through llm_call_with_usage, which picks the provider's client itself. + if provider not in ("openai", "anthropic", "google", "gemini", "google-vertex", + "openrouter", "amazon-bedrock", "vertex_ai", *CLI_BACKENDS): raise ValueError( f"Unsupported provider: '{provider}'. " "Use 'openai', 'anthropic', 'google', 'openrouter', 'amazon-bedrock' or 'claude-cli'." ) - self.top_k = top_k + # None means the default: the doc source must never get max_results=None (it would return every chunk). + self.top_k = DEFAULT_RETRIEVAL_TOP_K if top_k is None else top_k self.debug_retrieval = debug_retrieval # Initialize reranker (prefer semantic, fallback to lexical) @@ -316,12 +315,11 @@ def _retrieve_docs( } try: - # Retrieve top-k docs - # Note: Context7 client returns single doc, so we call it once for now - # TODO: Update Context7 client to support top_k natively + # Retrieve top-k docs, the same request the arms stack sends (max_results=top_k) raw_docs = self.mcp_client.get_library_docs( library_id=library_id, query=question, + max_results=self.top_k, max_tokens=max_tokens ) metadata["raw_count"] = len(raw_docs) @@ -359,24 +357,7 @@ def _generate_with_docs( retrieval_metadata: Optional[Dict[str, Any]] = None, verbatim: bool = False, ) -> Dict[str, Any]: - """Generate context-arm answer.""" - if not docs: - # No LLM call is made — emit a zero-cost/zero-token metrics block so - # this row carries the same telemetry shape as answered rows. - empty = UsageRecord(model=self.model, provider=self.provider, - cost_usd=0.0, n_calls=0) - token_usage = empty.as_token_usage_dict() - token_usage["context_chars"] = 0 - return { - "answer": "[No documentation retrieved]", - "retrieved_docs": [], - "model": self.model, - "doc_source": "none", - "retrieval_metadata": retrieval_metadata if self.debug_retrieval else None, - "token_usage": token_usage, - "metrics": empty.as_metrics_dict(answer_chars=0), - } - + """Generate context-arm answer. Called only with a non-empty *docs* (see ``_generate_answer_pair``).""" # ~4k tokens at ~4 chars/token; keeps prompt within most models' context window MAX_DOC_CHARS = 15_000 @@ -459,6 +440,7 @@ def _build_output(self, answers: List[Dict[str, Any]]) -> Dict[str, Any]: return { "generated_at": self._get_timestamp(), + "method_version": ANSWERING_METHOD_VERSION, "model": self.model, "provider": self.provider, "total_questions": len(answers), diff --git a/dokimi/dokimi/eval/diagnoser.py b/dokimi/dokimi/eval/diagnoser.py index 15e194e6..1283554f 100644 --- a/dokimi/dokimi/eval/diagnoser.py +++ b/dokimi/dokimi/eval/diagnoser.py @@ -30,12 +30,13 @@ added noise rather than signal. insufficient_data - Delta could not be computed (one or both answers missing). + Delta could not be computed (one or both answers missing or errored). A + retrieval miss that was left unjudged is ``empty_retrieval``, not this. """ from typing import Any, Dict, Iterable, List, Mapping, Optional -from dokimi.records import Answer, RecordLike, records +from dokimi.records import NO_RETRIEVAL_SOURCES, Answer, RecordLike, records # Keyword-overlap score below which retrieved docs are considered off-topic LOW_RELEVANCE_CUTOFF = 0.3 @@ -73,6 +74,7 @@ def diagnose( context_answer: Optional[Answer], delta: Optional[float], low_relevance_cutoff: float = LOW_RELEVANCE_CUTOFF, + context_scored: bool = False, ) -> Dict[str, Any]: """ Classify the retrieval outcome for a single question. @@ -86,6 +88,9 @@ def diagnose( Context-arm score minus baseline score, ``None`` when either is missing. low_relevance_cutoff: Relevance score below which retrieved docs are treated as off-topic. + context_scored: + True when the judge scored the context answer (``judge_placeholder``), so a + ``None`` delta comes from a missing baseline, not from an unjudged miss. Returns ------- @@ -95,6 +100,11 @@ def diagnose( - ``evidence``: dict of signals used to reach the diagnosis """ if delta is None: + # A retrieval miss the judge did not score has no delta; it is still an + # empty retrieval, not a missing or errored answer. + unjudged = not context_scored + if unjudged and context_answer is not None and not context_answer.error and context_answer.doc_source in NO_RETRIEVAL_SOURCES: + return _result(EMPTY_RETRIEVAL, {"doc_source": context_answer.doc_source, "retrieved_count": 0}) return _result(INSUFFICIENT_DATA, {}) if delta >= 0: diff --git a/dokimi/dokimi/eval/judge.py b/dokimi/dokimi/eval/judge.py index 2f035df5..03de110c 100644 --- a/dokimi/dokimi/eval/judge.py +++ b/dokimi/dokimi/eval/judge.py @@ -22,8 +22,8 @@ from dokimi.llm import ChatOpenAI, ChatAnthropic, LANGCHAIN_AVAILABLE from dokimi.eval.diagnoser import diagnose from dokimi.eval.question_text import require_question_texts -from dokimi.records import BASELINE, CONTEXT, Record, RecordLike, records -from dokimi.defaults import DEFAULT_JUDGE_MODEL, DEFAULT_JUDGE_PROVIDER +from dokimi.records import BASELINE, CONTEXT, NO_RETRIEVAL_SOURCES, Record, RecordLike, records +from dokimi.defaults import ANSWERING_METHOD_VERSION, DEFAULT_JUDGE_MODEL, DEFAULT_JUDGE_PROVIDER # One calibration scale for both the single judge and the panel, so their # numbers mean the same thing and can be compared. from dokimi.eval.rubric import ( @@ -197,7 +197,12 @@ class Judge: - Code Quality - Actionability """ - + + #: Class default, so a Judge built without __init__ (tests build one with __new__) keeps the default. + judge_placeholder: bool = False + #: Same for the stamp: the eval artifact carries the method version of the answers it judged. + answers_method_version: int = ANSWERING_METHOD_VERSION + def __init__( self, model: str = DEFAULT_JUDGE_MODEL, @@ -207,6 +212,8 @@ def __init__( temperature: Optional[float] = None, seed: Optional[int] = None, prompt_template: Optional[str] = None, + judge_placeholder: bool = False, + answers_method_version: int = ANSWERING_METHOD_VERSION, ): """ Args: @@ -220,6 +227,12 @@ def __init__( prompt_template: Replaces :data:`JUDGE_PROMPT` on the ungrounded path. Exists so the judge can be measured under a rewording of its own prompt; production scoring leaves it unset. + judge_placeholder: Judge a context answer that is the no-retrieval + placeholder ("[FALLBACK: ...]") too. Off by default: a question + with no document reaching the prompt is not scored as an answer. + answers_method_version: The method version of the answers being judged, + stamped into the eval artifact. Answers read from an artifact with no + stamp are version 1; answers generated in this process are the current version. """ if not LANGCHAIN_AVAILABLE: raise ImportError( @@ -235,6 +248,8 @@ def __init__( self.prompt_sha256 = JUDGE_PROMPT_SHA256 self.run_metadata = dict(run_metadata) if run_metadata else {} self.prompt_template = prompt_template + self.judge_placeholder = judge_placeholder + self.answers_method_version = answers_method_version # Accumulate judge-side token/cost telemetry across all judge calls # (Phase-C.0 covered answering; the judge path was untracked). self._usage_lock = threading.Lock() @@ -325,7 +340,10 @@ def _process(idx: int, answer: Record) -> None: "error": str(e), "with_docs": None, "without_docs": None, - "delta": None + "delta": None, + # What retrieval returned does not depend on the judge: the trust + # gate's coverage floor must not count this row as unrecorded. + "retrieval_hit": answer.retrieval_hit, } elapsed = time.time() - t0 with lock: @@ -376,9 +394,11 @@ def _evaluate_answer_pair( context_answer = answer.answer(CONTEXT) baseline_answer = answer.answer(BASELINE) - # Evaluate context arm + # Evaluate context arm. The no-retrieval placeholder is not an answer: it is + # left unjudged unless the caller opted in with judge_placeholder. with_docs_eval = None - if context_answer is not None and context_answer.text: + placeholder = context_answer is not None and context_answer.doc_source in NO_RETRIEVAL_SOURCES + if context_answer is not None and context_answer.text and (self.judge_placeholder or not placeholder): with_docs_eval = self._judge_answer( library_name=library_name, question=question_text, @@ -415,7 +435,7 @@ def _evaluate_answer_pair( } # Classify retrieval outcome (why docs helped or didn't) - result["diagnosis"] = diagnose(context_answer, delta) + result["diagnosis"] = diagnose(context_answer, delta, context_scored=with_docs_eval is not None) # Carried onto the eval record because the trust gate reads eval.v1, not # the answers it was judged from. result["retrieval_hit"] = answer.retrieval_hit @@ -644,6 +664,8 @@ def _build_evaluation_output(self, evaluations: List[Dict[str, Any]]) -> Dict[st """Build the serialisable output structure for a set of evaluations.""" output: Dict[str, Any] = { "evaluated_at": self._get_timestamp(), + "method_version": self.answers_method_version, + "judge_placeholder": self.judge_placeholder, "judge_model": self.model, "judge_provider": self.provider, "judge_temperature": self.temperature, diff --git a/dokimi/dokimi/gate/trust_gate.py b/dokimi/dokimi/gate/trust_gate.py index fda12ff5..72b8ad91 100644 --- a/dokimi/dokimi/gate/trust_gate.py +++ b/dokimi/dokimi/gate/trust_gate.py @@ -15,7 +15,7 @@ from dataclasses import dataclass, field from typing import Any, Dict, Iterable, List, Optional -from dokimi.records import BASELINE, CONTEXT, RecordLike, records +from dokimi.records import BASELINE, CONTEXT, Record, RecordLike, records # ── Thresholds (can be overridden via spec/config) ──────────────────────────── @@ -94,6 +94,61 @@ def status(self) -> str: return "✅ PASS" +def _retrieval_check(every: List[Record], cfg: Dict[str, Any]) -> TrustCheck: + """The retrieval hit rate over every evaluation, ``measured=False`` when it cannot be read. + + A question "hit" retrieval when at least one document reached the + context-arm prompt. The judge stamps that per evaluation as + ``retrieval_hit`` (from the answer's retrieved_docs / doc_source); eval + artifacts judged before that carry no retrieval outcome at all, and the + check reports "not measured" for them rather than a pass. So does a mix of + the two below ``min_retrieval_coverage``: the rate would describe a subset. + Over every evaluation, not only the both-arms-scored ones: a question with no + document is not judged by default, and it still records that it missed. + """ + measured = [h for r in every if (h := r.retrieval_hit) is not None] + # A legacy judge-error row has no retrieval_hit: left out of the count, as before. + n_all = sum(1 for r in every if not (r.error is not None and r.retrieval_hit is None)) + if measured and len(measured) / n_all < cfg["min_retrieval_coverage"]: + return TrustCheck( + name="retrieval_hit_rate", + passed=True, + value=None, + threshold=cfg["min_retrieval_hit_rate"], + message=( + f"Not measured: only {len(measured)} of {n_all} evaluations record a retrieval " + f"outcome (retrieval_hit), under the {cfg['min_retrieval_coverage']:.0%} coverage " + f"floor; {sum(measured)} of those {len(measured)} hit" + ), + severity="warn", + measured=False, + ) + if measured: + retrieval_hits = sum(measured) + hit_rate = retrieval_hits / len(measured) + coverage = "" if len(measured) == n_all else f"; {n_all - len(measured)} unrecorded" + return TrustCheck( + name="retrieval_hit_rate", + passed=hit_rate >= cfg["min_retrieval_hit_rate"], + value=round(hit_rate, 3), + threshold=cfg["min_retrieval_hit_rate"], + message=( + f"Retrieval hit rate = {hit_rate:.1%} ({retrieval_hits}/{len(measured)}{coverage}) " + f"(min {cfg['min_retrieval_hit_rate']:.0%})" + ), + severity="warn", + ) + return TrustCheck( + name="retrieval_hit_rate", + passed=True, + value=None, + threshold=cfg["min_retrieval_hit_rate"], + message="Not measured: the evaluations record no retrieval outcome (retrieval_hit)", + severity="warn", + measured=False, + ) + + def evaluate_trust( evaluations: Iterable[RecordLike], thresholds: Optional[Dict[str, Any]] = None, @@ -114,9 +169,11 @@ def evaluate_trust( cfg = {**DEFAULTS, **(thresholds or {})} checks: List[TrustCheck] = [] - # Every check is over the questions scored in both arms. + # Parsed once: ``evaluations`` may be a one-shot iterable. + every = records(evaluations, kind="evaluations") + # The score checks below are over the questions scored in both arms. valid = [ - r for r in records(evaluations, kind="evaluations") + r for r in every if r.score(CONTEXT) is not None and r.score(BASELINE) is not None ] with_scores: List[float] = [r.score(CONTEXT) or 0 for r in valid] @@ -133,17 +190,31 @@ def evaluate_trust( severity="fail", )) if n == 0: + # Nothing is scored in both arms (every question an unjudged miss, say): the + # retrieval rate is still known, and it is the cause, so it goes beside the count. + if any(r.retrieval_hit is not None for r in every): + checks.append(_retrieval_check(every, cfg)) return TrustVerdict(checks=checks) # can't compute anything else # ── 2. Zero-score fraction ───────────────────────────────────────────── - zero_with = sum(1 for v in with_scores if v == 0) - zero_frac = zero_with / n + # Misses count as unusable, judge-error misses too (by their error field): else many misses would pass. + unjudged_misses = [ + r for r in every + if r.score(CONTEXT) is None and r.retrieval_hit is False + and (r.score(BASELINE) is not None or r.error is not None) + ] + zero_with = sum(1 for v in with_scores if v == 0) + len(unjudged_misses) + unusable_n = n + len(unjudged_misses) + zero_frac = zero_with / unusable_n checks.append(TrustCheck( name="zero_score_fraction", passed=zero_frac <= cfg["max_zero_fraction"], value=round(zero_frac, 3), threshold=cfg["max_zero_fraction"], - message=f"{zero_with}/{n} questions scored 0 in the context arm ({zero_frac:.1%})", + message=( + f"{zero_with}/{unusable_n} questions scored 0 or had no document in the context " + f"arm ({zero_frac:.1%})" + ), severity="fail", )) @@ -195,52 +266,7 @@ def evaluate_trust( )) # ── 7. Retrieval hit rate ───────────────────────────────────────────── - # A question "hit" retrieval when at least one document reached the - # context-arm prompt. The judge stamps that per evaluation as - # ``retrieval_hit`` (from the answer's retrieved_docs / doc_source); eval - # artifacts judged before that carry no retrieval outcome at all, and the - # check reports "not measured" for them rather than a pass. So does a mix of - # the two below ``min_retrieval_coverage``: the rate would describe a subset. - measured = [h for r in valid if (h := r.retrieval_hit) is not None] - if measured and len(measured) / n < cfg["min_retrieval_coverage"]: - checks.append(TrustCheck( - name="retrieval_hit_rate", - passed=True, - value=None, - threshold=cfg["min_retrieval_hit_rate"], - message=( - f"Not measured: only {len(measured)} of {n} evaluations record a retrieval " - f"outcome (retrieval_hit), under the {cfg['min_retrieval_coverage']:.0%} coverage " - f"floor; {sum(measured)} of those {len(measured)} hit" - ), - severity="warn", - measured=False, - )) - elif measured: - retrieval_hits = sum(measured) - hit_rate = retrieval_hits / len(measured) - coverage = "" if len(measured) == n else f"; {n - len(measured)} unrecorded" - checks.append(TrustCheck( - name="retrieval_hit_rate", - passed=hit_rate >= cfg["min_retrieval_hit_rate"], - value=round(hit_rate, 3), - threshold=cfg["min_retrieval_hit_rate"], - message=( - f"Retrieval hit rate = {hit_rate:.1%} ({retrieval_hits}/{len(measured)}{coverage}) " - f"(min {cfg['min_retrieval_hit_rate']:.0%})" - ), - severity="warn", - )) - else: - checks.append(TrustCheck( - name="retrieval_hit_rate", - passed=True, - value=None, - threshold=cfg["min_retrieval_hit_rate"], - message="Not measured: the evaluations record no retrieval outcome (retrieval_hit)", - severity="warn", - measured=False, - )) + checks.append(_retrieval_check(every, cfg)) # ── 8. Multi-run variance (optional) ───────────────────────────────── multirun_std = None diff --git a/dokimi/dokimi/orchestrator/pipeline.py b/dokimi/dokimi/orchestrator/pipeline.py index 690afc76..60bd22c1 100644 --- a/dokimi/dokimi/orchestrator/pipeline.py +++ b/dokimi/dokimi/orchestrator/pipeline.py @@ -9,9 +9,10 @@ from typing import List, Dict, Any, Optional from pathlib import Path -from dokimi.defaults import DEFAULT_ANSWER_MODEL, DEFAULT_ANSWER_PROVIDER, DEFAULT_JUDGE_MODEL, DEFAULT_JUDGE_PROVIDER, DEFAULT_RETRIEVAL_TOP_K +from dokimi.defaults import (ANSWERING_METHOD_VERSION, DEFAULT_ANSWER_MODEL, DEFAULT_ANSWER_PROVIDER, DEFAULT_JUDGE_MODEL, DEFAULT_JUDGE_PROVIDER, DEFAULT_RETRIEVAL_TOP_K, + answers_method_version as answers_method_version_of) from dokimi.eval.stats import paired_summary -from dokimi.mcp.factory import create_doc_source_client +from dokimi.mcp.factory import create_doc_source_client, resolve_library_id_for_product from dokimi.utils import same_evaluator logger = logging.getLogger(__name__) @@ -96,6 +97,8 @@ def __init__( questions_from: Optional[Path] = None, product_key: Optional[str] = None, questions_root: Optional[Path] = None, + registry: Optional[Any] = None, + judge_placeholder: bool = False, ): """ Initialize pipeline. @@ -136,6 +139,10 @@ def __init__( skipped; when it is absent a warning is logged and questions are generated. Without this argument (and without ``custom_questions_path``) the run has no golden set. + registry: ProductRegistry used to find the declared context7_id + (default: the bundled products.yaml) + judge_placeholder: Also judge a with-docs answer that is the + no-retrieval placeholder (default: recorded, not scored) """ if not repo and not description: raise ValueError( @@ -192,6 +199,8 @@ def __init__( self.judge_model = judge_model self.judge_provider = judge_provider self.judge_temperature = judge_temperature + # Judge the no-retrieval placeholder answers too (off: they are not scored). + self.judge_placeholder = judge_placeholder self.personas_count = personas_count self.questions_per_topic = questions_per_topic @@ -200,6 +209,8 @@ def __init__( self.debug_retrieval = debug_retrieval self.doc_source = doc_source self.context7_id = context7_id + # Product registry for the library id (defaults to the bundled products.yaml). + self.registry = registry self.max_tokens_per_question = max_tokens_per_question self.force_regen = force_regen @@ -383,6 +394,10 @@ def run(self, concurrency: int = 5) -> Dict[str, Any]: # Summary stats over the complete pairs, the numbers the reports print for the same file. summary = paired_summary(evaluations) + # Questions scored in one arm only (a retrieval miss is left unjudged, a judge may fail) + # are in none of the averages: say how many, so the line is not read as covering all. + results["steps"]["evaluation"]["paired"] = summary.paired + results["steps"]["evaluation"]["unpaired"] = summary.unpaired if summary.paired: results["steps"]["evaluation"]["summary"] = { @@ -393,7 +408,9 @@ def run(self, concurrency: int = 5) -> Dict[str, Any]: print(f"✓ Evaluated {len(evaluations)} answers: " f"context={results['steps']['evaluation']['summary']['with_avg']}, " f"baseline={results['steps']['evaluation']['summary']['without_avg']}, " - f"delta={results['steps']['evaluation']['summary']['delta_avg']:+.1f}") + f"delta={results['steps']['evaluation']['summary']['delta_avg']:+.1f}" + + (f" (over the {summary.paired} scored in both arms; {summary.unpaired} left out)" + if summary.unpaired else "")) else: print(f"✓ Evaluated {len(evaluations)} answers (summary unavailable)") @@ -442,15 +459,30 @@ def _discover_personas(self) -> Dict[str, Any]: return personas + def _library_id(self, mcp_client) -> str: + """The doc-source library id: explicit id, then the ``;id=`` of an ``mcp:`` source, then the + registry's ``context7_id``, then the client. + + The caller's own choices (the explicit id, the source's descriptor) go before the registry, as + they did before the pipeline used the registry lookup; ``eval/arms_run.py`` resolves registry-first. + """ + descriptor_id = getattr(mcp_client, "default_library_id", None) + return resolve_library_id_for_product( + self.product_key, mcp_client, explicit_id=self.context7_id or descriptor_id, registry=self.registry, + ) + def _generate_questions(self, personas: Dict[str, Any]) -> List[Dict[str, Any]]: """Generate questions from personas.""" from dokimi.questions import RagasSeedExtractor, QuestionGenerator # Extract topics mcp_client = create_doc_source_client(self.doc_source) - library_id = self.context7_id or mcp_client.resolve_library_id(self.product) - - extractor = RagasSeedExtractor(mcp_client=mcp_client, cache_dir=Path(".cache/topics")) + library_id = self._library_id(mcp_client) + + # The id is the registry's for every source, so the topics cache is also keyed on the source. + source_key = hashlib.sha256(self.doc_source.encode("utf-8")).hexdigest()[:16] + extractor = RagasSeedExtractor(mcp_client=mcp_client, + cache_dir=Path(".cache/topics") / source_key) topics = extractor.extract_topics( library_id=library_id, library_name=self.product, @@ -617,8 +649,8 @@ def _generate_answers(self, questions: List[Dict[str, Any]], concurrency: int = # Setup documentation source client mcp_client = create_doc_source_client(self.doc_source) - library_id = self.context7_id or mcp_client.resolve_library_id(self.product) - + library_id = self._library_id(mcp_client) + # Generate answers answerer = Answerer( mcp_client=mcp_client, @@ -653,15 +685,23 @@ def _evaluate_answers(self, answers: List[Dict[str, Any]] | Dict[str, Any], # Use in-memory answers from the previous step (avoid stale disk read); # fall back to the answers file when none were passed. if answers: - answers_list = list(read(answers, source="answers", section="answers").records) + run = read(answers, source="answers", section="answers") + else: + run = load(self.answers_path, section="answers") + answers_list = list(run.records) + # Fresh in-memory answers are this process's method; an artifact says its own, unstamped is legacy. + if answers and isinstance(answers, list): + answers_method_version = ANSWERING_METHOD_VERSION else: - answers_list = list(load(self.answers_path, section="answers").records) + answers_method_version = answers_method_version_of(run.meta) # Evaluate judge = Judge( model=self.judge_model, provider=self.judge_provider, temperature=self.judge_temperature, + judge_placeholder=self.judge_placeholder, + answers_method_version=answers_method_version, run_metadata={ "question_model": self.model, "question_provider": self.provider, diff --git a/dokimi/dokimi/schemas/answers.v1.json b/dokimi/dokimi/schemas/answers.v1.json index 48a6f34d..fd9abc96 100644 --- a/dokimi/dokimi/schemas/answers.v1.json +++ b/dokimi/dokimi/schemas/answers.v1.json @@ -25,6 +25,10 @@ "null" ] }, + "method_version": { + "type": "integer", + "description": "Answering method version (defaults.ANSWERING_METHOD_VERSION); absent on artifacts from before it was stamped." + }, "total_questions": { "type": "integer" }, diff --git a/dokimi/dokimi/schemas/eval.v1.json b/dokimi/dokimi/schemas/eval.v1.json index 61ae91b8..b3b609d7 100644 --- a/dokimi/dokimi/schemas/eval.v1.json +++ b/dokimi/dokimi/schemas/eval.v1.json @@ -85,6 +85,14 @@ } } }, + "method_version": { + "type": "integer", + "description": "Answering method version (defaults.ANSWERING_METHOD_VERSION); absent on artifacts from before it was stamped." + }, + "judge_placeholder": { + "type": "boolean", + "description": "True when the judge also scored a context answer that is the no-retrieval placeholder (--judge-placeholder), which the default leaves unjudged; absent on artifacts from before it was stamped. Two eval files with the same method_version but a different value here are not comparable." + }, "judge_temperature": { "description": "Sampling temperature the judge ran at; null when left to the provider default." }, diff --git a/dokimi/tests/answering_stack_parity.py b/dokimi/tests/answering_stack_parity.py index 14859652..da7af879 100644 --- a/dokimi/tests/answering_stack_parity.py +++ b/dokimi/tests/answering_stack_parity.py @@ -259,6 +259,8 @@ def run_stack_p( max_tokens: int = 4000, context7_id: Optional[str] = LIBRARY_ID, questions: Optional[List[Dict[str, Any]]] = None, + registry: Any = None, + judge_placeholder: bool = False, ) -> Observed: """Stack P end to end on the fixture: answer, then judge, through the real pipeline code.""" from dokimi.orchestrator import pipeline as pipeline_mod @@ -281,6 +283,8 @@ def run_stack_p( max_tokens_per_question=max_tokens, context7_id=context7_id, product_key=PRODUCT_KEY, + registry=registry, + judge_placeholder=judge_placeholder, ) qs = QUESTIONS if questions is None else questions mark = len(provider_calls.calls) diff --git a/dokimi/tests/data/cli_flags_allowlist.txt b/dokimi/tests/data/cli_flags_allowlist.txt index f8e037fd..4d744663 100644 --- a/dokimi/tests/data/cli_flags_allowlist.txt +++ b/dokimi/tests/data/cli_flags_allowlist.txt @@ -87,6 +87,7 @@ --judge-configs --judge-model --judge-panel +--judge-placeholder # placeholder judge (#274) adds it: no registry concept yet; a F1b migration names it --judge-provider --judge-temperature --judges diff --git a/dokimi/tests/test_answering_stack_p_step2.py b/dokimi/tests/test_answering_stack_p_step2.py new file mode 100644 index 00000000..85b60a8d --- /dev/null +++ b/dokimi/tests/test_answering_stack_p_step2.py @@ -0,0 +1,105 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Wiring checks for the Stack P fixes of #228 step 2. + +The parity table in ``test_answering_stack_parity.py`` pins what each fix changes +in numbers. These tests cover the parts that table cannot see: the value a flag +parses to, and where it lands. +""" + +import pytest + +from tests.test_answering_stack_parity import _subparser + + +class _FakePipeline: + """Stands in for EvaluationPipeline: keeps the kwargs ``benchmark run`` builds it with.""" + + built: list = [] + + def __init__(self, **kwargs): + _FakePipeline.built.append(kwargs) + + def run(self, concurrency): + return {"steps": {}} + + +@pytest.mark.parametrize("extra, runs", [([], 1), (["--multi-run", "2"], 2)], ids=["single-run", "multi-run"]) +def test_benchmark_run_hands_the_judge_flags_to_the_pipeline(monkeypatch, tmp_path, extra, runs): + """The real handler runs; only the pipeline is faked. Both flags reach it in every mode.""" + from dokimi.commands import benchmark + from dokimi import orchestrator + + _FakePipeline.built = [] + monkeypatch.setattr(orchestrator, "EvaluationPipeline", _FakePipeline) + args = _subparser("benchmark", "run").parse_args( + ["--library", "onetbb", "--output-dir", str(tmp_path), "--judge-temperature", "0.3", + "--judge-placeholder", *extra] + ) + + benchmark.cmd_benchmark_run(args) + + assert len(_FakePipeline.built) == runs + for kwargs in _FakePipeline.built: + assert kwargs["judge_temperature"] == 0.3 + assert kwargs["judge_placeholder"] is True + + +def test_benchmark_run_leaves_the_judge_flags_off_when_not_given(monkeypatch, tmp_path): + from dokimi.commands import benchmark + from dokimi import orchestrator + + _FakePipeline.built = [] + monkeypatch.setattr(orchestrator, "EvaluationPipeline", _FakePipeline) + args = _subparser("benchmark", "run").parse_args(["--library", "onetbb", "--output-dir", str(tmp_path)]) + + benchmark.cmd_benchmark_run(args) + + (kwargs,) = _FakePipeline.built + assert kwargs["judge_temperature"] is None + assert kwargs["judge_placeholder"] is False + + +def test_benchmark_run_judge_temperature_defaults_to_provider_default(): + args = _subparser("benchmark", "run").parse_args(["--library", "onetbb"]) + assert args.judge_temperature is None + + +def test_judge_placeholder_flag_is_off_by_default_and_parses_when_given(): + run = _subparser("benchmark", "run") + assert run.parse_args(["--library", "onetbb"]).judge_placeholder is False + assert run.parse_args(["--library", "onetbb", "--judge-placeholder"]).judge_placeholder is True + + +def test_judge_placeholder_flag_judges_the_placeholder_answers(monkeypatch, tmp_path, provider_calls): + from tests.answering_stack_parity import run_stack_p + + # q-empty and q-rerank carry the placeholder as their with-docs answer: with the flag they + # are judged again, so P's judge calls go back to the 12 that Stack A makes. + p = run_stack_p(monkeypatch, tmp_path, provider_calls, judge_placeholder=True) + assert len(p.judge_inputs) == 12 + assert len([e for e in p.evaluations if e["delta"] is not None]) == 6 + + +def test_eval_output_is_stamped_with_the_method_version(monkeypatch): + from dokimi.defaults import ANSWERING_METHOD_VERSION + from dokimi.eval.judge import Judge + + monkeypatch.setenv("OPENAI_API_KEY", "test-key-not-used") + out = Judge(model="gpt-x", provider="openai")._build_evaluation_output([]) + assert out["method_version"] == ANSWERING_METHOD_VERSION == 2 + + +def test_eval_output_says_whether_the_placeholder_was_judged(monkeypatch): + """The two modes give different numbers on the same answers under one method_version.""" + from tests.artifact_fixtures import artifact_fixture + from dokimi.eval.judge import Judge + + monkeypatch.setenv("OPENAI_API_KEY", "test-key-not-used") + default = Judge(model="gpt-x", provider="openai")._build_evaluation_output([]) + opted_in = Judge(model="gpt-x", provider="openai", judge_placeholder=True)._build_evaluation_output([]) + + assert default["judge_placeholder"] is False + assert opted_in["judge_placeholder"] is True + artifact_fixture("eval", opted_in) # eval.v1 declares the key diff --git a/dokimi/tests/test_answering_stack_parity.py b/dokimi/tests/test_answering_stack_parity.py index 7ad72a58..df6934de 100644 --- a/dokimi/tests/test_answering_stack_parity.py +++ b/dokimi/tests/test_answering_stack_parity.py @@ -19,7 +19,7 @@ import argparse from pathlib import Path -from typing import Any, Dict, Tuple +from typing import Any, Dict, List, Optional, Tuple import pytest @@ -55,15 +55,14 @@ PINNED: Tuple[Tuple[str, str, Any, Any], ...] = ( # shared: the two stacks agree today ("shared", "retrieval.count", 6, 6), - ("shared", "judge.calls", 12, 12), ("shared", "reranker.calls", 5, 5), ("shared", "answer.roles", ("user",), ("user",)), ("shared", "answer.system", None, None), ("shared", "answer.cache_blocks", False, False), # D1 retrieval request - ("D1", "retrieval.passed", ("max_tokens",), ("max_results",)), - # what each side asked for, not what came back: both get 4 docs; P keeps 3, A keeps 3 - ("D1", "retrieval.max_results", 5, 3), + ("D1", "retrieval.passed", ("max_results", "max_tokens"), ("max_results",)), + # what each side asked for, not what came back: both get 4 docs; both ask for top_k=3 + ("D1", "retrieval.max_results", 3, 3), ("D1", "retrieval.max_tokens", 4000, 8000), # D2 no docs found ("D2", "empty.answer_kinds", ("plain",), ("plain", "plain")), @@ -102,31 +101,35 @@ "(No documentation provided)", "(No documentation retrieved)", ), - # D6 judge temperature (CLI: P = benchmark run, A = arms run; benchmark run has no such flag) - ("D6", "cli.judge_temperature_flag", False, True), + # D6 judge temperature (CLI: P = benchmark run, A = arms run) + ("D6", "cli.judge_temperature_flag", True, True), # D7 --max-tokens (CLI defaults; where the configured value lands) ("D7", "cli.max_tokens_default", 4000, None), ("D7", "max_tokens.lands_in", "retrieval", "answer"), # D8 library id: EvaluationPipeline with no context7_id (the `orchestrate` path without # --context7-id). `benchmark run` passes the registry's context7_id, so it is not this row. - ("D8", "library_id", "client/resolved", "registry/lib"), - # D9 statistics. The delta also moves with D3: P's q-rerank is judged with no docs - # (delta 0), A's falls back to raw docs (delta 20), so a D3 fix flips this row too. - ("D9", "stats.delta_mean", 13.33, 16.67), - ("D9", "significance.n", 6, 6), + ("D8", "library_id", "registry/lib", "registry/lib"), + # D2 judge calls: P no longer judges the placeholder with-docs answers of q-empty and + # q-rerank (D2, D3), so 2 of the 12 calls are gone. A still judges all 12. + ("D2", "judge.calls", 10, 12), + # D9 statistics. P's paired deltas are q-docs, q-docs-2, q-docs-3, q-gt: 4 pairs, below + # MIN_PAIRS_FOR_TEST (6), so P has no significance test. A has 6 pairs. + ("D9", "stats.delta_mean", 20.0, 16.67), + ("D9", "stats.pairs", 4, 6), + ("D9", "significance.n", None, 6), ("D9", "significance.correction", None, "holm"), - ("D9", "significance.significant", False, False), + ("D9", "significance.significant", None, False), # what the judge sees per case: no docs (D2), reranker raises (D3), ground-truth chunk (D5) ( "D2", "judge.contexts.q-empty", - ("(No documentation provided)", "(No documentation retrieved)"), + ("(No documentation provided)",), ("(No documentation retrieved)",), ), ( "D3", "judge.contexts.q-rerank", - ("(No documentation provided)", "(No documentation retrieved)"), + ("(No documentation provided)",), ("(No documentation retrieved)", ("docs", "Passage", 3, True)), ), ( @@ -138,8 +141,8 @@ # what the judge is handed as ground truth: the q-gt chunk on both stacks, None on the rest ("shared", "judge.ground_truth.q-gt", (GROUND_TRUTH_CHUNK,), (GROUND_TRUTH_CHUNK,)), ("shared", "judge.ground_truth.others", (None,), (None,)), - # the per-question deltas that feed D9's statistics (P: stored "delta"; A: "docs" vs "baseline"). - # q-rerank is the one that moves with D3: P judges it with no docs (0), A with the raw docs (20). + # Per-question deltas feeding D9's statistics. q-empty and q-rerank have no P delta (no with-docs judgement, D2/D3); + # A's q-empty is 0 and its q-rerank falls back to the raw docs (20). ( "D9", "delta.per_question", @@ -147,9 +150,9 @@ ("q-docs", 20.0), ("q-docs-2", 20.0), ("q-docs-3", 20.0), - ("q-empty", 0.0), + ("q-empty", None), ("q-gt", 20.0), - ("q-rerank", 0.0), + ("q-rerank", None), ), ( ("q-docs", 20.0), @@ -164,6 +167,8 @@ ("only", "record.shape", "with_docs/without_docs", "arms"), ("only", "answer.token_usage.context_chars", True, False), ("only", "artifact.cost_summary", False, True), + # #228 step 2 stamp: the answers artifact carries method_version; the arms artifact does not + ("only", "artifact.method_version", 2, None), ) @@ -253,15 +258,6 @@ def _where_4000_lands(obs: Observed) -> str: def _observe(tmp_path: Path, provider_calls) -> Dict[str, Tuple[Any, Any]]: - with pytest.MonkeyPatch.context() as mp: - p = run_stack_p(mp, tmp_path / "p", provider_calls) - with pytest.MonkeyPatch.context() as mp: - a = run_stack_a(mp, tmp_path / "a", provider_calls) - with pytest.MonkeyPatch.context() as mp: - p_noid = run_stack_p( - mp, tmp_path / "p-noid", provider_calls, context7_id=None, questions=QUESTIONS[:1] - ) - from dokimi.eval.arms_run import resolve_doc_library_id from dokimi.registry import ProductRegistry from dokimi.treatments.arms import DocTreatment @@ -272,6 +268,22 @@ def _observe(tmp_path: Path, provider_calls) -> Dict[str, Tuple[Any, Any]]: f" description: fixture\n context7_id: {REGISTRY_ID}\n", encoding="utf-8", ) + registry = ProductRegistry(registry_path) + + with pytest.MonkeyPatch.context() as mp: + p = run_stack_p(mp, tmp_path / "p", provider_calls) + with pytest.MonkeyPatch.context() as mp: + a = run_stack_a(mp, tmp_path / "a", provider_calls) + with pytest.MonkeyPatch.context() as mp: + p_noid = run_stack_p( + mp, + tmp_path / "p-noid", + provider_calls, + context7_id=None, + questions=QUESTIONS[:1], + registry=registry, + ) + a_lib = resolve_doc_library_id( [DocTreatment(a.client, name="docs", top_k=3, reranker=a.reranker)], PRODUCT, @@ -303,6 +315,7 @@ def sampling(pairs, field): sig_p = _significance_p(p) sig_a = a.artifact["summary"]["per_arm"]["docs"]["significance"] + paired_p = _paired_deltas(p) return { "retrieval.count": (len(p.client.calls), len(a.client.calls)), @@ -311,7 +324,8 @@ def sampling(pairs, field): "answer.roles": (sampling(p_all, "roles"), sampling(a_all, "roles")), "answer.system": (sampling(p_all, "system"), sampling(a_all, "system")), "answer.cache_blocks": (sampling(p_all, "cache_blocks"), sampling(a_all, "cache_blocks")), - "stats.delta_mean": (sig_p["delta_mean"], sig_a["delta_mean"]), + "stats.delta_mean": (round(sum(paired_p) / len(paired_p), 2), sig_a["delta_mean"]), + "stats.pairs": (len(paired_p), sig_a["n"]), "retrieval.passed": ( distinct([c["passed"] for c in p.client.calls]), distinct([c["passed"] for c in a.client.calls]), @@ -365,15 +379,23 @@ def sampling(pairs, field): ), "max_tokens.lands_in": (_where_4000_lands(p), _where_4000_lands(a)), "library_id": (p_noid.client.calls[0]["library_id"], a_lib), - "significance.n": (sig_p["n"], sig_a["n"]), - "significance.correction": (sig_p.get("correction"), sig_a.get("correction")), - "significance.significant": (sig_p["significant"], sig_a["significant"]), + # None: the paired test does not run below MIN_PAIRS_FOR_TEST pairs (P has 4 here) + "significance.n": (sig_p["n"] if sig_p else None, sig_a["n"]), + "significance.correction": ( + sig_p.get("correction") if sig_p else None, + sig_a.get("correction"), + ), + "significance.significant": ( + sig_p["significant"] if sig_p else None, + sig_a["significant"], + ), "record.shape": (_record_shape(p_rec_empty), _record_shape(a_rec_empty)), "answer.token_usage.context_chars": ( "context_chars" in docs_block(p, p_rec_docs)["token_usage"], "context_chars" in docs_block(a, a_rec_docs)["token_usage"], ), "artifact.cost_summary": ("cost_summary" in p.artifact, "cost_summary" in a.artifact), + "artifact.method_version": (p.artifact.get("method_version"), a.artifact.get("method_version")), "judge.contexts.q-empty": (_context_kinds(p, EMPTY_QUERY), _context_kinds(a, EMPTY_QUERY)), "judge.contexts.q-rerank": ( _context_kinds(p, RERANK_QUERY), @@ -403,12 +425,16 @@ def sampling(pairs, field): } -def _significance_p(obs: Observed) -> Dict[str, Any]: +def _significance_p(obs: Observed) -> Optional[Dict[str, Any]]: + """Stack P's paired test; None when it does not run (fewer pairs than MIN_PAIRS_FOR_TEST).""" from dokimi.report.eval_report import significance_test - result = significance_test(obs.evaluations) - assert result is not None, "P paired test did not run on the fixture" - return result + return significance_test(obs.evaluations) + + +def _paired_deltas(obs: Observed) -> List[float]: + """The deltas of the questions both arms were judged on.""" + return [e["delta"] for e in obs.evaluations if e.get("delta") is not None] def _record_shape(rec: Dict[str, Any]) -> str: diff --git a/dokimi/tests/test_answers_command.py b/dokimi/tests/test_answers_command.py index 1eb72036..e806e470 100644 --- a/dokimi/tests/test_answers_command.py +++ b/dokimi/tests/test_answers_command.py @@ -308,3 +308,28 @@ def test_generate_unreadable_questions_exits_2_naming_the_path(tmp_path, monkeyp answers.cmd_answers_generate(_args(questions=str(path))) assert exc.value.code == 2 assert f"Error: cannot read {path}" in capsys.readouterr().err + + +@pytest.mark.parametrize("value", ["0", "-1", '"3"', "true"]) +def test_a_config_top_k_that_is_not_a_positive_int_exits_2(tmp_path, monkeypatch, capsys, value): + # top_k reaches the doc source as max_results: 0 would make every answer a placeholder. + path = tmp_path / "products.yaml" + path.write_text(f"retrieval:\n top_k: {value}\n", encoding="utf-8") + _point_config(monkeypatch, path) + monkeypatch.setattr(dokimi.questions.artifact, "load_questions", + lambda path: pytest.fail("questions loaded after a bad top_k")) + with pytest.raises(SystemExit) as exc: + answers.cmd_answers_generate(_args()) + assert exc.value.code == 2 + assert "retrieval.top_k in the config must be a positive integer" in capsys.readouterr().err + + +@pytest.mark.parametrize("value", ["0", "-1"]) +def test_top_k_flag_below_one_is_refused(capsys, value): + from dokimi.cli import build_parser + + with pytest.raises(SystemExit) as exc: + build_parser().parse_args(["answers", "generate", "--product", "L", + "--questions", "q.json", "--top-k", value]) + assert exc.value.code == 2 + assert "--top-k" in capsys.readouterr().err diff --git a/dokimi/tests/test_benchmark_registry_wiring.py b/dokimi/tests/test_benchmark_registry_wiring.py new file mode 100644 index 00000000..e7fef9f5 --- /dev/null +++ b/dokimi/tests/test_benchmark_registry_wiring.py @@ -0,0 +1,84 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""``benchmark run --registry`` resolves the library id through that registry.""" + +import pytest + +from dokimi import orchestrator +from dokimi.commands import benchmark +from dokimi.orchestrator.pipeline import EvaluationPipeline +from tests.test_answering_stack_parity import _subparser + +CUSTOM = """\ +products: + my_lib: + name: My Lib + description: A library only this registry knows + doc_sources: [context7] + onetbb: + name: Custom TBB + description: A key that the bundled registry also has, with no id here + doc_sources: [context7] +""" + + +class _DeclaredIdClient: + """A doc source like Context7: a name that is not a declared id is not resolved.""" + + requires_declared_library_id = True + + def resolve_library_id(self, name): + return name + + +class _Spy: + """Stands in for EvaluationPipeline: keeps what ``benchmark run`` hands it.""" + + built: list = [] + + def __init__(self, **kwargs): + self.kwargs = kwargs + _Spy.built.append(kwargs) + + def run(self, concurrency): + return {"steps": {}} + + +@pytest.fixture +def spy(monkeypatch): + _Spy.built = [] + monkeypatch.setattr(orchestrator, "EvaluationPipeline", _Spy) + return _Spy + + +def _run(tmp_path, library, *extra): + registry = tmp_path / "my.yaml" + registry.write_text(CUSTOM, encoding="utf-8") + args = _subparser("benchmark", "run").parse_args( + ["--library", library, "--registry", str(registry), "--output-dir", str(tmp_path / "out"), *extra] + ) + benchmark.cmd_benchmark_run(args) + + +@pytest.mark.parametrize("library", ["my_lib", "onetbb"]) +def test_a_custom_entry_without_an_id_is_not_looked_up_in_the_bundled_registry(spy, tmp_path, library): + _run(tmp_path, library) + + (kwargs,) = spy.built + real = EvaluationPipeline(**kwargs) + with pytest.raises(ValueError, match="declares no 'context7_id'"): + real._library_id(_DeclaredIdClient()) + + +def test_multi_run_and_batch_hand_the_registry_over_too(spy, tmp_path): + _run(tmp_path, "my_lib", "--multi-run", "2") + registry = tmp_path / "my.yaml" + args = _subparser("benchmark", "batch").parse_args( + ["--libraries", "my_lib", "--registry", str(registry), "--output-dir", str(tmp_path / "batch")] + ) + benchmark.cmd_benchmark_batch(args) + + assert len(spy.built) == 3 + for kwargs in spy.built: + assert "my_lib" in kwargs["registry"] diff --git a/dokimi/tests/test_diagnoser.py b/dokimi/tests/test_diagnoser.py index 5fabb7db..2931d49e 100644 --- a/dokimi/tests/test_diagnoser.py +++ b/dokimi/tests/test_diagnoser.py @@ -18,13 +18,15 @@ ) -def _make_answer(doc_source="context7", retrieved_docs=None): +def _make_answer(doc_source="context7", retrieved_docs=None, error=None): """The context-arm answer of an answers.v1 record, as the judge hands it to diagnose().""" + with_docs = {"answer": "an answer", "doc_source": doc_source, "retrieved_docs": retrieved_docs or []} + if error: + with_docs["error"] = error data = artifact_fixture("answers", {"answers": [{ "question_id": "q1", "question_text": "How?", - "with_docs": {"answer": "an answer", "doc_source": doc_source, - "retrieved_docs": retrieved_docs or []}, + "with_docs": with_docs, "without_docs": {"answer": "some answer"}, }]}) return records.read(data).records[0].answer(CONTEXT) @@ -53,6 +55,35 @@ def test_empty_retrieval_no_docs(): assert result["label"] == EMPTY_RETRIEVAL +def test_unjudged_retrieval_miss_is_an_empty_retrieval_not_insufficient_data(): + """The judge leaves the no-document placeholder unjudged, so its delta is None.""" + answer = _make_answer(doc_source="fallback_none", retrieved_docs=[]) + result = diagnose(answer, None) + assert result["label"] == EMPTY_RETRIEVAL + assert result["evidence"]["doc_source"] == "fallback_none" + + +def test_unjudged_none_source_is_an_empty_retrieval(): + answer = _make_answer(doc_source="none", retrieved_docs=[]) + result = diagnose(answer, None) + assert result["label"] == EMPTY_RETRIEVAL + assert result["evidence"]["doc_source"] == "none" + + +def test_judged_placeholder_with_no_baseline_is_insufficient_data(): + """With judge_placeholder the context is scored; a missing baseline is the reason there is no delta.""" + answer = _make_answer(doc_source="fallback_none", retrieved_docs=[]) + result = diagnose(answer, None, context_scored=True) + assert result["label"] == INSUFFICIENT_DATA + + +def test_errored_placeholder_answer_stays_insufficient_data(): + """An errored answer is missing, not a retrieval miss, whatever its doc source.""" + answer = _make_answer(doc_source="fallback_none", retrieved_docs=[], error="timeout") + result = diagnose(answer, None) + assert result["label"] == INSUFFICIENT_DATA + + def test_empty_retrieval_no_mcp_client(): answer = _make_answer(doc_source="none", retrieved_docs=[]) result = diagnose(answer, -2.0) diff --git a/dokimi/tests/test_eval_answerer.py b/dokimi/tests/test_eval_answerer.py index 7a9b6256..5c9e2824 100644 --- a/dokimi/tests/test_eval_answerer.py +++ b/dokimi/tests/test_eval_answerer.py @@ -18,6 +18,7 @@ sys.modules[mod_name] = mock_mod from dokimi.eval.answerer import Answerer, ANSWER_PROMPT_WITH_DOCS, ANSWER_PROMPT_WITHOUT_DOCS +from dokimi.defaults import DEFAULT_RETRIEVAL_TOP_K # Canonical return value for llm_call_with_usage mock _LLM_MOCK_RETURN = ("This is a test answer from LLM.", { @@ -74,29 +75,17 @@ def mock_llm_call(): class TestAnswererInit: - def test_init_openai(self): - mock_cls = Mock() - with patch('dokimi.eval.answerer.ChatOpenAI', mock_cls, create=True), \ - patch('dokimi.eval.answerer.LANGCHAIN_AVAILABLE', True): - answerer = Answerer(model="gpt-4o", provider="openai") - assert answerer.model == "gpt-4o" - assert answerer.provider == "openai" - mock_cls.assert_called_once() - - def test_init_anthropic(self): - mock_cls = Mock() - with patch('dokimi.eval.answerer.ChatAnthropic', mock_cls, create=True), \ - patch('dokimi.eval.answerer.LANGCHAIN_AVAILABLE', True): - answerer = Answerer(model="claude-opus", provider="anthropic", api_key="test") - # The anthropic client is the one built, with the caller's model and key. - mock_cls.assert_called_once_with(model="claude-opus", api_key="test") - assert answerer.llm is mock_cls.return_value - assert answerer.provider == "anthropic" + @pytest.mark.parametrize( + "provider", + ["openai", "anthropic", "google", "gemini", "google-vertex", "openrouter", + "amazon-bedrock", "vertex_ai"], + ) + def test_init_accepts_supported_provider(self, provider): # test-quality: ok must-not-raise smoke test of the provider allow-list + with patch('dokimi.eval.answerer.LANGCHAIN_AVAILABLE', True): + Answerer(model="m", provider=provider) def test_init_unsupported_provider(self): - mock_cls = Mock() - with patch('dokimi.eval.answerer.ChatOpenAI', mock_cls, create=True), \ - patch('dokimi.eval.answerer.LANGCHAIN_AVAILABLE', True): + with patch('dokimi.eval.answerer.LANGCHAIN_AVAILABLE', True): with pytest.raises(ValueError) as exc: Answerer(provider="unsupported") assert "Unsupported provider" in str(exc.value) @@ -191,18 +180,40 @@ def test_token_usage_summary_in_output(self, sample_questions, mock_mcp_client, class TestRetrieveDocs: + def test_asks_the_doc_source_for_top_k_docs(self, mock_mcp_client): + with patch('dokimi.eval.answerer.LANGCHAIN_AVAILABLE', True): + answerer = Answerer(mcp_client=mock_mcp_client, top_k=7) + answerer._retrieve_docs("uxlfoundation/oneTBB", "parallel_for", 4000) + + mock_mcp_client.get_library_docs.assert_called_once_with( + library_id="uxlfoundation/oneTBB", query="parallel_for", + max_results=7, max_tokens=4000, + ) + def test_calls_mcp_client(self, mock_mcp_client): with patch('dokimi.eval.answerer.LANGCHAIN_AVAILABLE', True): answerer = Answerer(mcp_client=mock_mcp_client) docs = answerer._retrieve_docs("uxlfoundation/oneTBB", "parallel_for", 4000) mock_mcp_client.get_library_docs.assert_called_once_with( - library_id="uxlfoundation/oneTBB", query="parallel_for", max_tokens=4000 + library_id="uxlfoundation/oneTBB", query="parallel_for", + max_results=DEFAULT_RETRIEVAL_TOP_K, max_tokens=4000, ) assert [d["content"] for d in docs] == [ mock_mcp_client.get_library_docs.return_value[0]["content"] ] + def test_no_top_k_asks_the_doc_source_for_the_default(self, mock_mcp_client): + """`answers generate` without --top-k reaches here as top_k=None: the doc source gets the default, not None.""" + with patch('dokimi.eval.answerer.LANGCHAIN_AVAILABLE', True): + answerer = Answerer(mcp_client=mock_mcp_client, top_k=None) + answerer._retrieve_docs("uxlfoundation/oneTBB", "parallel_for", 4000) + + mock_mcp_client.get_library_docs.assert_called_once_with( + library_id="uxlfoundation/oneTBB", query="parallel_for", + max_results=3, max_tokens=4000, + ) + def test_error_handling(self, mock_mcp_client): mock_mcp_client.get_library_docs.side_effect = Exception("Network error") @@ -228,15 +239,6 @@ def test_with_docs(self, mock_llm_call): assert result["doc_source"] == "context7" assert result["answer"] == "This is a test answer from LLM." - def test_empty_docs(self): - with patch('dokimi.eval.answerer.LANGCHAIN_AVAILABLE', True): - answerer = Answerer() - result = answerer._generate_with_docs("Test question?", []) - - assert result["answer"] == "[No documentation retrieved]" - assert result["retrieved_docs"] == [] - assert "token_usage" in result - class TestGenerateWithoutDocs: def test_generates_answer(self, mock_llm_call): diff --git a/dokimi/tests/test_eval_score_method_version.py b/dokimi/tests/test_eval_score_method_version.py new file mode 100644 index 00000000..9c8c30a7 --- /dev/null +++ b/dokimi/tests/test_eval_score_method_version.py @@ -0,0 +1,123 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""``eval score`` stamps the eval artifact with the method version of the answers it judged.""" + +import json +from types import SimpleNamespace + +import pytest + +from dokimi.cli import build_parser + +GOOD = ('{"reasoning":"r","correctness":80,"completeness":80,"specificity":80,' + '"code_quality":80,"actionability":80,"aggregate":80}') + +ANSWERS = [{"question_id": "Q1", "question_text": "q", "question": "q", + "with_docs": {"answer": "an answer", "doc_source": "x", "retrieved_docs": ["d"]}, + "without_docs": {"answer": "without docs"}}] + +JUDGE = ["--judge-provider", "amazon-bedrock", "--judge-model", "us.anthropic.x"] + + +@pytest.fixture +def judge_calls(monkeypatch): + import litellm + + sent = [] + + def completion(**kw): + sent.append(kw) + return SimpleNamespace(choices=[SimpleNamespace(message=SimpleNamespace(content=GOOD))], + usage=None) + + monkeypatch.setattr(litellm, "completion", completion) + return sent + + +def _eval_score(tmp_path, answers_doc): + answers = tmp_path / "answers.json" + answers.write_text(json.dumps(answers_doc), encoding="utf-8") + out = tmp_path / "eval.json" + args = build_parser().parse_args(["eval", "score", "--product", "L", "--answers", str(answers), + "--output", str(out), "--concurrency", "1", *JUDGE]) + args.func(args) + return json.loads(out.read_text(encoding="utf-8")) + + +def test_unstamped_answers_are_legacy_so_the_eval_says_version_1(tmp_path, judge_calls): + """Answers written before the stamp are the legacy method; re-judging them today does not make them version 2.""" + out = _eval_score(tmp_path, ANSWERS) + + assert len(judge_calls) == 2 # both arms judged: the stamp does not change what is judged + assert out["method_version"] == 1 + + +@pytest.mark.parametrize("version", [1, 2]) +def test_stamped_answers_keep_their_method_version(tmp_path, judge_calls, version): + out = _eval_score(tmp_path, {"method_version": version, "answers": ANSWERS}) + + assert out["method_version"] == version + + +def _pipeline_judge_version(monkeypatch, tmp_path, answers, answers_file=None): + """The method version the pipeline's judge is built with, for these answers.""" + from dokimi.eval.judge import Judge + from dokimi.orchestrator.pipeline import EvaluationPipeline + + monkeypatch.setenv("OPENAI_API_KEY", "test-key-not-used") + seen = [] + + def evaluate(self, product, records, concurrency=5): + seen.append(self.answers_method_version) + return [] + + monkeypatch.setattr(Judge, "evaluate_answers", evaluate) + monkeypatch.setattr(Judge, "save_evaluations", lambda self, evaluations, path: None) + p = EvaluationPipeline.__new__(EvaluationPipeline) # the attributes _evaluate_answers reads + p.product, p.model, p.provider = "L", "gpt-x", "openai" + p.judge_model, p.judge_provider, p.judge_temperature, p.judge_placeholder = "gpt-x", "openai", None, False + p.answers_path, p.eval_path = answers_file, tmp_path / "eval.json" + p._evaluate_answers(answers, concurrency=1) + return seen[0] + + +def test_pipeline_judges_a_wrapped_version_1_artifact_as_version_1(monkeypatch, tmp_path): + assert _pipeline_judge_version(monkeypatch, tmp_path, {"method_version": 1, "answers": ANSWERS}) == 1 + + +def test_pipeline_judges_an_unstamped_answers_file_as_legacy_version_1(monkeypatch, tmp_path): + answers_file = tmp_path / "answers.json" + answers_file.write_text(json.dumps({"answers": ANSWERS}), encoding="utf-8") + + assert _pipeline_judge_version(monkeypatch, tmp_path, [], answers_file) == 1 + + +def test_pipeline_judges_fresh_in_memory_answers_as_the_current_version(monkeypatch, tmp_path): + from dokimi.defaults import ANSWERING_METHOD_VERSION + + assert _pipeline_judge_version(monkeypatch, tmp_path, ANSWERS) == ANSWERING_METHOD_VERSION + + +@pytest.mark.parametrize("bad", ["2", True, 3.5, [1], 0, -7]) +def test_a_malformed_stamp_exits_2_before_any_judge_call(tmp_path, judge_calls, capsys, bad): + """main ignored the field; writing it unchecked failed eval validation after paid calls.""" + with pytest.raises(SystemExit) as exc: + _eval_score(tmp_path, {"method_version": bad, "answers": ANSWERS}) + + assert exc.value.code == 2 + assert judge_calls == [] + assert not (tmp_path / "eval.json").exists() + assert "method_version must be a positive integer" in capsys.readouterr().err + + +def test_a_null_stamp_is_legacy(tmp_path, judge_calls): + assert _eval_score(tmp_path, {"method_version": None, "answers": ANSWERS})["method_version"] == 1 + + +def test_pipeline_refuses_an_answers_file_with_a_malformed_stamp(monkeypatch, tmp_path): + answers_file = tmp_path / "answers.json" + answers_file.write_text(json.dumps({"method_version": "2", "answers": ANSWERS}), encoding="utf-8") + + with pytest.raises(ValueError, match="method_version must be a positive integer"): + _pipeline_judge_version(monkeypatch, tmp_path, [], answers_file) diff --git a/dokimi/tests/test_judge_placeholder_entry_points.py b/dokimi/tests/test_judge_placeholder_entry_points.py new file mode 100644 index 00000000..31f69d93 --- /dev/null +++ b/dokimi/tests/test_judge_placeholder_entry_points.py @@ -0,0 +1,96 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""--judge-placeholder reaches the judge of every entry point that takes it.""" + +import json +from types import SimpleNamespace + +import pytest + +from dokimi.cli import build_parser + +GOOD = ('{"reasoning":"r","correctness":80,"completeness":80,"specificity":80,' + '"code_quality":80,"actionability":80,"aggregate":80}') + +#: A question whose retrieval missed: the with-docs answer is the no-retrieval placeholder. +ANSWERS = [{"question_id": "Q1", "question_text": "q", "question": "q", + "with_docs": {"answer": "[FALLBACK: No relevant documentation found]", + "doc_source": "fallback_none", "retrieved_docs": []}, + "without_docs": {"answer": "without docs"}}] + +JUDGE = ["--judge-provider", "amazon-bedrock", "--judge-model", "us.anthropic.x"] + + +@pytest.fixture +def requests(monkeypatch): + import litellm + + sent = [] + + def completion(**kw): + sent.append(kw) + return SimpleNamespace(choices=[SimpleNamespace(message=SimpleNamespace(content=GOOD))], + usage=None) + + monkeypatch.setattr(litellm, "completion", completion) + return sent + + +def _run(argv): + args = build_parser().parse_args(argv) + args.func(args) + + +def _eval_score(tmp_path, *flag): + answers = tmp_path / "answers.json" + answers.write_text(json.dumps(ANSWERS), encoding="utf-8") + out = tmp_path / "eval.json" + _run(["eval", "score", "--product", "L", "--answers", str(answers), "--output", str(out), + "--concurrency", "1", *JUDGE, *flag]) + return json.loads(out.read_text(encoding="utf-8")) + + +def _evaluate(tmp_path, monkeypatch, *flag): + from dokimi.orchestrator import EvaluationPipeline + + def run(self, **kw): + self._evaluate_answers(ANSWERS) + raise RuntimeError("stop after the judge step") + + monkeypatch.setattr(EvaluationPipeline, "run", run) + with pytest.raises(SystemExit): + _run(["evaluate", "--product", "L", "--description", "d", "--output-dir", str(tmp_path), *JUDGE, *flag]) + return json.loads((tmp_path / "eval" / "L.json").read_text(encoding="utf-8")) + + +def test_eval_score_leaves_the_placeholder_unjudged_by_default(tmp_path, requests): + out = _eval_score(tmp_path) + + assert len(requests) == 1 # the baseline answer only + assert out["evaluations"][0]["with_docs"] is None + assert out["judge_placeholder"] is False + + +def test_eval_score_judge_placeholder_judges_it_and_says_so(tmp_path, requests): + out = _eval_score(tmp_path, "--judge-placeholder") + + assert len(requests) == 2 + assert out["evaluations"][0]["with_docs"]["aggregate"] == 80 + assert out["judge_placeholder"] is True + + +def test_evaluate_leaves_the_placeholder_unjudged_by_default(tmp_path, requests, monkeypatch): + out = _evaluate(tmp_path, monkeypatch) + + assert len(requests) == 1 + assert out["evaluations"][0]["with_docs"] is None + assert out["judge_placeholder"] is False + + +def test_evaluate_judge_placeholder_judges_it_and_says_so(tmp_path, requests, monkeypatch): + out = _evaluate(tmp_path, monkeypatch, "--judge-placeholder") + + assert len(requests) == 2 + assert out["evaluations"][0]["with_docs"]["aggregate"] == 80 + assert out["judge_placeholder"] is True diff --git a/dokimi/tests/test_llm_cli.py b/dokimi/tests/test_llm_cli.py index c70b40ad..8986ff57 100644 --- a/dokimi/tests/test_llm_cli.py +++ b/dokimi/tests/test_llm_cli.py @@ -831,6 +831,21 @@ def test_evaluate_with_a_cli_judge_given_a_temperature_exits_2_like_the_other_ju assert "Traceback" not in err +def test_benchmark_run_with_a_cli_judge_given_a_temperature_refuses_before_any_answer(monkeypatch, capsys): + """The refusal comes before the pipeline is built, so no answer is paid for.""" + from dokimi import cli, orchestrator + + built = [] + monkeypatch.setattr(orchestrator, "EvaluationPipeline", lambda **kwargs: built.append(kwargs)) + monkeypatch.setattr("sys.argv", ["dokimi", "benchmark", "run", "--library", "onetbb", + "--judge-provider", "claude-cli", "--judge-temperature", "0"]) + with pytest.raises(SystemExit) as exc: + cli.main() + err = capsys.readouterr().err + assert exc.value.code == 2 and err.startswith("Error: claude-cli takes no temperature") + assert built == [] + + def test_the_main_model_falls_back_to_most_output_when_the_requested_family_is_unknown(): usage = {"helper-a": {"outputTokens": 5}, "helper-b": {"outputTokens": 500}} assert llm_cli._main_model(usage, "claude-fable-5-1") == "helper-b" diff --git a/dokimi/tests/test_pipeline_library_id.py b/dokimi/tests/test_pipeline_library_id.py new file mode 100644 index 00000000..64d07be0 --- /dev/null +++ b/dokimi/tests/test_pipeline_library_id.py @@ -0,0 +1,151 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""``EvaluationPipeline._library_id``: which doc-source library a run queries.""" + +import pytest + +from dokimi.orchestrator.pipeline import EvaluationPipeline +from dokimi.registry import ProductRegistry + +REGISTRY = """\ +products: + my_lib: + name: My Lib + description: The registry key differs from the display name + context7_id: org/declared + doc_sources: [context7] +""" + + +class _RecordingClient: + """Records what the pipeline asks it to resolve; a bare name resolves to a marker.""" + + requires_declared_library_id = True + + def __init__(self): + self.asked = [] + + def resolve_library_id(self, name): + self.asked.append(name) + return f"client:{name}" + + +@pytest.fixture +def registry(tmp_path): + path = tmp_path / "products.yaml" + path.write_text(REGISTRY, encoding="utf-8") + return ProductRegistry(path=path) + + +def _pipeline(tmp_path, registry, **kwargs): + return EvaluationPipeline( + product="My Lib", description="d", product_key="my_lib", output_dir=tmp_path / "run", registry=registry, **kwargs + ) + + +def test_an_explicit_id_wins_over_the_registry_entry(tmp_path, registry): + client = _RecordingClient() + + library_id = _pipeline(tmp_path, registry, context7_id="org/explicit")._library_id(client) + + assert library_id == "org/explicit" + assert client.asked == [] + + +def test_an_mcp_descriptor_id_wins_over_the_registry_entry(tmp_path, registry): + """``mcp:...;id=X`` names the library for that source; the registry's id does not replace it.""" + from dokimi.mcp.factory import create_doc_source_client + + client = create_doc_source_client("mcp:http=https://example.com/mcp;id=org/custom") + + library_id = _pipeline(tmp_path, registry)._library_id(client) + + assert library_id == "org/custom" + + +def test_an_explicit_id_wins_over_the_mcp_descriptor_id(tmp_path, registry): + from dokimi.mcp.factory import create_doc_source_client + + client = create_doc_source_client("mcp:http=https://example.com/mcp;id=org/custom") + + library_id = _pipeline(tmp_path, registry, context7_id="org/explicit")._library_id(client) + + assert library_id == "org/explicit" + + +def test_the_registry_key_not_the_display_name_finds_the_declared_id(tmp_path, registry): + client = _RecordingClient() + + library_id = _pipeline(tmp_path, registry)._library_id(client) + + assert library_id == "org/declared" + assert client.asked == [] + + +def test_an_unregistered_product_asks_the_client_for_its_key(tmp_path, registry): + """A key that is not in the registry goes to the client, by its key and not its display name.""" + client = _RecordingClient() + pipeline = EvaluationPipeline( + product="Other Lib", description="d", product_key="other_lib", output_dir=tmp_path / "run", registry=registry + ) + + library_id = pipeline._library_id(client) + + assert library_id == "client:other_lib" + assert client.asked == ["other_lib"] + + +def test_question_generation_extracts_topics_for_the_declared_library(tmp_path, registry, monkeypatch): + """The question step asks the extractor for the registry's id, not for whatever the + client resolves the display name to: the arms stack and this step must agree.""" + import dokimi.questions as questions + + client = _RecordingClient() + seen = {} + + class _StopHere(Exception): + pass + + class _Extractor: + def __init__(self, **kwargs): + pass + + def extract_topics(self, library_id, **kwargs): + seen["library_id"] = library_id + raise _StopHere + + monkeypatch.setattr("dokimi.orchestrator.pipeline.create_doc_source_client", lambda source: client) + monkeypatch.setattr(questions, "RagasSeedExtractor", _Extractor) + + with pytest.raises(_StopHere): + _pipeline(tmp_path, registry)._generate_questions({}) + + assert seen["library_id"] == "org/declared" + + +def test_two_local_doc_sets_of_one_product_get_their_own_topics(tmp_path, registry, monkeypatch): + """Both runs resolve the registry's id, so a topics cache keyed on the id alone gave the + second doc set the first one's topics.""" + import dokimi.questions as questions + + seen = [] + + class _Generator: + def __init__(self, **kwargs): + pass + + def generate_questions(self, topics, **kwargs): + seen.append(sorted(topics)) + return [] + + monkeypatch.setattr(questions, "QuestionGenerator", _Generator) + monkeypatch.chdir(tmp_path) + docs = {"docsA": "`parallel_for` `task_arena`", "docsB": "`flow_graph` `concurrent_queue`"} + for name, words in docs.items(): + (tmp_path / name).mkdir() + (tmp_path / name / "api.md").write_text(f"My Lib API overview {words}\n", encoding="utf-8") + pipeline = _pipeline(tmp_path, registry, doc_source=f"local:{tmp_path / name}") + pipeline._generate_questions({"personas": []}) + + assert seen == [["parallel_for", "task_arena"], ["concurrent_queue", "flow_graph"]] diff --git a/dokimi/tests/test_pipeline_summary.py b/dokimi/tests/test_pipeline_summary.py index 35e80203..a2c38ef0 100644 --- a/dokimi/tests/test_pipeline_summary.py +++ b/dokimi/tests/test_pipeline_summary.py @@ -70,3 +70,28 @@ def test_summary_is_left_out_when_no_question_was_scored(tmp_path, monkeypatch): result = _run(tmp_path, monkeypatch, [_evaluation("q1", None, None)]) assert "summary" not in result["steps"]["evaluation"] + + +def test_summary_says_how_many_questions_it_left_out(tmp_path, monkeypatch, capsys): + """3 of 10 questions are unjudged retrieval misses: the means are over 7, and the + run result and the printed line say so instead of reading as a result over all 10.""" + evaluations = [_evaluation(f"q{i}", 80, 60) for i in range(7)] + evaluations += [_evaluation(f"m{i}", None, 60) for i in range(3)] + + result = _run(tmp_path, monkeypatch, evaluations) + + step = result["steps"]["evaluation"] + assert (step["paired"], step["unpaired"]) == (7, 3) + printed = capsys.readouterr().out + assert "Evaluated 10 answers: context=80.0, baseline=60.0, delta=+20.0" in printed + assert "over the 7 scored in both arms; 3 left out" in printed + + +def test_summary_does_not_say_left_out_when_every_question_is_paired(tmp_path, monkeypatch, capsys): + result = _run(tmp_path, monkeypatch, [_evaluation("q1", 80, 60), _evaluation("q2", 70, 50)]) + + step = result["steps"]["evaluation"] + assert (step["paired"], step["unpaired"]) == (2, 0) + printed = capsys.readouterr().out + assert "Evaluated 2 answers: context=75.0, baseline=55.0, delta=+20.0" in printed + assert "left out" not in printed diff --git a/dokimi/tests/test_provider_call_budgets.py b/dokimi/tests/test_provider_call_budgets.py index cbcd7627..47229e1e 100644 --- a/dokimi/tests/test_provider_call_budgets.py +++ b/dokimi/tests/test_provider_call_budgets.py @@ -328,7 +328,7 @@ def test_eval_panel_score_of_1_record_with_3_judges_makes_6_provider_calls( class _DocSource: """A doc-source client with one relevant chunk per query.""" - def get_library_docs(self, library_id, query, max_tokens): + def get_library_docs(self, library_id, query, max_tokens, max_results=5): return [{"content": f"{query} Use parallel_for with a blocked_range.", "source": "s"}] diff --git a/dokimi/tests/test_trust_gate.py b/dokimi/tests/test_trust_gate.py index ecb3932c..81382eae 100644 --- a/dokimi/tests/test_trust_gate.py +++ b/dokimi/tests/test_trust_gate.py @@ -182,11 +182,24 @@ def _judged(n_hit, n_miss): ] def test_fails_when_retrieval_never_returned_a_doc(self): - evals = self._judged(n_hit=0, n_miss=12) - assert all(e["with_docs"]["aggregate"] > 0 for e in evals) + evals = self._judged(n_hit=0, n_miss=14) + # Every question is an unjudged miss: nothing is scored in both arms, the run is not + # trusted, and the message names the cause (0 of 14 hit) beside the question count. + verdict = evaluate_trust(evals) + assert not verdict.is_trusted + assert [c.name for c in verdict.failures] == ["min_questions"] + check = next(c for c in verdict.checks if c.name == "retrieval_hit_rate") + assert not check.passed and check.measured and check.value == 0.0 + assert "(0/14)" in check.message + + def test_fails_when_retrieval_rarely_returned_a_doc(self): + evals = self._judged(n_hit=2, n_miss=12) + # The placeholder is not judged by default: only the two hits carry a with-docs score, + # and the twelve misses still count against the rate. + assert sum(e["with_docs"] is None for e in evals) == 12 check = next(c for c in evaluate_trust(evals).checks if c.name == "retrieval_hit_rate") assert not check.passed, check.message - assert check.value == 0.0 + assert check.value == round(2 / 14, 3) assert check.measured def test_rate_counts_recorded_hits(self): @@ -353,6 +366,15 @@ def test_coverage_below_the_default_floor_is_not_measured(self): assert not _check(v, "retrieval_hit_rate").measured assert v.not_measured == ["retrieval_hit_rate"] + def test_legacy_judge_error_rows_leave_a_stored_rate_measured(self): + # A judge-error row as eval score wrote it before error rows carried retrieval_hit. + legacy_error = {"question_id": "q", "error": "judge failed", "with_docs": None, + "without_docs": None, "delta": None} + evals = ([_row(70, 50, hit=True)] * 5 + [_row(70, 50, hit=False)] * 10 + + [legacy_error] * 5) + check = _check(evaluate_trust(evals), "retrieval_hit_rate") + assert (check.measured, check.value, check.passed) == (True, 0.333, False) + def test_a_zero_baseline_counts_as_zero(self): check = _check(evaluate_trust(_evals(10, with_score=40, without_score=0)), "min_delta") assert check.value == 40.0 @@ -432,7 +454,7 @@ def test_failures_and_warnings_render_apart(self): "| Check | Status | Value | Threshold | Notes |", "|-------|--------|-------|-----------|-------|", "| min_questions | ❌ | 3 | 10 | 3 questions evaluated (min 10) |", - "| zero_score_fraction | ❌ | 0.333 | 0.2 | 1/3 questions scored 0 in the context arm (33.3%) |", + "| zero_score_fraction | ❌ | 0.333 | 0.2 | 1/3 questions scored 0 or had no document in the context arm (33.3%) |", "| min_with_docs_avg | ✅ | 33.3 | 30.0 | Context-arm avg = 33.3 (min 30.0) |", "| min_delta | ✅ | 13.3 | -5.0 | Avg delta (context arm minus baseline) = +13.3 (min -5.0) |", "| score_cv | ⚠️ | 0.917 | 0.6 | Score CV = 0.92 std=30.6 (max CV 0.6) |", @@ -442,7 +464,7 @@ def test_failures_and_warnings_render_apart(self): "", "**Blocking issues:**", "- ❌ **min_questions**: 3 questions evaluated (min 10)", - "- ❌ **zero_score_fraction**: 1/3 questions scored 0 in the context arm (33.3%)", + "- ❌ **zero_score_fraction**: 1/3 questions scored 0 or had no document in the context arm (33.3%)", "", "**Warnings:**", "- ⚠️ **score_cv**: Score CV = 0.92 std=30.6 (max CV 0.6)", diff --git a/dokimi/tests/test_trust_gate_unjudged_misses.py b/dokimi/tests/test_trust_gate_unjudged_misses.py new file mode 100644 index 00000000..0b205328 --- /dev/null +++ b/dokimi/tests/test_trust_gate_unjudged_misses.py @@ -0,0 +1,196 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The zero-score check counts unjudged retrieval misses as unusable. + +A question whose retrieval missed gets no with-docs judgement, so it has no +context score. The zero-score check must still count it. Otherwise a run where +most questions miss passes a fail-severity check it fails on intel/main. +""" + +from unittest.mock import MagicMock + +import pytest + +from dokimi.eval.diagnoser import EMPTY_RETRIEVAL, INSUFFICIENT_DATA, summarise_diagnoses +from dokimi.eval.judge import Judge +from dokimi.gate.trust_gate import evaluate_trust + +GOOD = MagicMock(content=( + '{"correctness": 60, "completeness": 60, "specificity": 60, ' + '"code_quality": 60, "actionability": 60, "aggregate": 0}' +)) +HIT = { + "answer": "use parallel_for", + "doc_source": "context7", + "retrieved_docs": [{"source": "context7", "snippet": "parallel_for ...", + "relevance_score": 0.8}], +} +MISS = { + "answer": "[FALLBACK: No relevant documentation found]", + "doc_source": "fallback_none", + "retrieved_docs": [], +} + + +def _rows(n_hit, n_miss): + """Judge n_hit questions with a document and n_miss without, as the judge does.""" + judge = Judge.__new__(Judge) + judge._accumulate_usage = lambda usage: None + judge.temperature = judge.seed = None + judge.llm = MagicMock() + judge.llm.invoke.return_value = GOOD + rows = [] + for i in range(n_hit + n_miss): + rows.append(judge._evaluate_answer_pair("L", { + "question_id": f"q{i}", + "question_text": "q?", + "with_docs": dict(HIT if i < n_hit else MISS), + "without_docs": {"answer": "baseline"}, + })) + return rows + + +def _zero_check(verdict): + (check,) = [c for c in verdict.checks if c.name == "zero_score_fraction"] + return check + + +def test_four_misses_of_fourteen_fail_the_zero_check(): + check = _zero_check(evaluate_trust(_rows(10, 4))) + assert check.severity == "fail" + assert check.value == 0.286 + assert check.passed is False + + +def test_all_hits_pass_the_zero_check(): + check = _zero_check(evaluate_trust(_rows(14, 0))) + assert check.value == 0.0 + assert check.passed is True + + +def _retrieval_check(verdict): + (check,) = [c for c in verdict.checks if c.name == "retrieval_hit_rate"] + return check + + +def test_judge_error_rows_keep_the_retrieval_outcome_of_their_answer(monkeypatch): + """5 of 20 answers fail in the judge: the retrieval rate is still measured over all 20.""" + monkeypatch.setenv("OPENAI_API_KEY", "test-key-not-used") + judge = Judge(model="gpt-x", provider="openai") + judge.llm = MagicMock() + judge.llm.invoke.side_effect = lambda msgs, *a, **k: GOOD + real = judge._evaluate_answer_pair + + def flaky(library_name, item): + if int(item.question_id[1:]) >= 15: + raise RuntimeError("judge down") + return real(library_name, item) + + judge._evaluate_answer_pair = flaky + answers = [ + {"question_id": f"q{i}", "question_text": "q?", + "with_docs": dict(HIT), "without_docs": {"answer": "baseline"}} + for i in range(20) + ] + rows = judge.evaluate_answers("L", answers) + assert sum("error" in r for r in rows) == 5 + check = _retrieval_check(evaluate_trust(rows)) + assert check.value == 1.0 + assert check.measured is not False + assert "(20/20)" in check.message + + +def test_unjudged_misses_stay_empty_retrievals_in_the_failure_analysis(): + summary = summarise_diagnoses(_rows(10, 4)) + assert summary["counts"][EMPTY_RETRIEVAL] == 4 + assert summary["counts"][INSUFFICIENT_DATA] == 0 + + +def _row(question_id, context, baseline, retrieval_hit): + """An eval record as the judge writes it: a null condition is one that was not scored.""" + return { + "question_id": question_id, + "question_text": "q?", + "with_docs": None if context is None else {"aggregate": context}, + "without_docs": None if baseline is None else {"aggregate": baseline}, + "delta": None if context is None or baseline is None else context - baseline, + "retrieval_hit": retrieval_hit, + } + + +@pytest.mark.parametrize("extra", [ + # no context score, and no retrieval outcome recorded: an older artifact, or a judge error + # on an answer whose outcome was never stamped + _row("legacy", None, 60, None), + # a document reached the prompt, but the context answer was not scored (judge error) + _row("judge-error", None, 60, True), + # a miss with no baseline score: it is not a question both arms were meant to score + _row("no-baseline", None, None, False), +], ids=["legacy-row", "hit-not-scored", "miss-without-baseline"]) +def test_only_a_recorded_miss_with_a_baseline_counts_as_unusable(extra): + rows = [_row(f"q{i}", 70, 60, True) for i in range(10)] + + check = _zero_check(evaluate_trust(rows + [extra])) + + assert (check.value, check.message.split(" ")[0]) == (0.0, "0/10") + + +def test_a_scored_miss_counts_once_in_the_zero_check(): + # A miss the judge scored (--judge-placeholder) with a 0 context score is a zero already; it + # must not also count as an unjudged miss, or the numerator and the denominator both grow. + rows = [_row(f"q{i}", 70, 60, True) for i in range(10)] + [_row("scored-miss", 0, 60, False)] + + check = _zero_check(evaluate_trust(rows)) + + assert (check.value, check.message.split(" ")[0]) == (round(1 / 11, 3), "1/11") + + +def _judged_with_failures(n_hit, n_miss, failing): + """Judge the answers through ``evaluate_answers``; each question id in ``failing`` raises in the judge.""" + judge = Judge(model="gpt-x", provider="openai") + judge.llm = MagicMock() + judge.llm.invoke.side_effect = lambda msgs, *a, **k: GOOD + real = judge._evaluate_answer_pair + + def flaky(library_name, item): + if item.question_id in failing: + raise RuntimeError("judge down") + return real(library_name, item) + + judge._evaluate_answer_pair = flaky + answers = [ + {"question_id": f"h{i}", "question_text": "q?", + "with_docs": dict(HIT), "without_docs": {"answer": "baseline"}} + for i in range(n_hit) + ] + [ + {"question_id": f"m{i}", "question_text": "q?", + "with_docs": dict(MISS), "without_docs": {"answer": "baseline"}} + for i in range(n_miss) + ] + return judge.evaluate_answers("L", answers) + + +def test_a_judge_error_on_a_retrieval_miss_counts_as_unusable(monkeypatch): + """The judge raises on a miss: its error row has no scores, but it is still an unusable question.""" + monkeypatch.setenv("OPENAI_API_KEY", "test-key-not-used") + rows = _judged_with_failures(3, 1, failing={"m0"}) + error_row = next(r for r in rows if r["question_id"] == "m0") + assert "error" in error_row and error_row["retrieval_hit"] is False + + check = _zero_check(evaluate_trust(rows)) + + assert (check.value, check.passed) == (0.25, False) + assert check.message.split(" ")[0] == "1/4" + + +def test_a_judge_error_on_a_retrieval_hit_stays_out_of_the_zero_check(monkeypatch): + monkeypatch.setenv("OPENAI_API_KEY", "test-key-not-used") + rows = _judged_with_failures(3, 0, failing={"h0"}) + error_row = next(r for r in rows if r["question_id"] == "h0") + assert "error" in error_row and error_row["retrieval_hit"] is True + + check = _zero_check(evaluate_trust(rows)) + + assert (check.value, check.passed) == (0.0, True) + assert check.message.split(" ")[0] == "0/2"