From e159659cbea5bf5dda8cfee29619527dc0381d89 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 28 Oct 2026 16:29:17 -0700 Subject: [PATCH 01/18] fix(model): one same-model rule for baseline, self-judge and repo compare utils.normalize_model_ref (baseline check, self-judge warning) and repos/report.model_matches (repo compare) disagreed on dated ids, regional profiles, provider routes, -vN:M revisions and case. Replace both with utils.same_model: route, region profile, revision and case are not identity; a dated id matches its undated alias; two dated ids match only on the same date. Refs intel/dokimi#229 (P2-3). Signed-off-by: Nikolay Petrov --- ...28_130000_napetrov_one-model-match-rule.md | 11 ++++ dokimi/docs/benchmarking-and-comparison.md | 7 ++- dokimi/dokimi/commands/baseline.py | 40 ++++++++++-- dokimi/dokimi/commands/evaluate.py | 6 +- dokimi/dokimi/repos/compare.py | 16 ++--- dokimi/dokimi/repos/report.py | 41 +------------ dokimi/dokimi/utils.py | 47 +++++++++++--- dokimi/tests/test_baseline_compare.py | 23 +++++++ dokimi/tests/test_model_defaults.py | 6 +- dokimi/tests/test_repo_compare.py | 6 +- dokimi/tests/test_repo_run.py | 5 +- dokimi/tests/test_same_model.py | 61 +++++++++++++++++++ 12 files changed, 198 insertions(+), 71 deletions(-) create mode 100644 dokimi/changelog.d/20261028_130000_napetrov_one-model-match-rule.md create mode 100644 dokimi/tests/test_same_model.py diff --git a/dokimi/changelog.d/20261028_130000_napetrov_one-model-match-rule.md b/dokimi/changelog.d/20261028_130000_napetrov_one-model-match-rule.md new file mode 100644 index 000000000..2ce3c4134 --- /dev/null +++ b/dokimi/changelog.d/20261028_130000_napetrov_one-model-match-rule.md @@ -0,0 +1,11 @@ +### Fixed + +- **"Same model" means one thing in `baseline compare`, the self-judge warning and `repo compare`** — they + used two rules that disagreed. A dated id (`claude-sonnet-4-5-20250929`) and its undated alias, a Bedrock + regional profile (`eu.anthropic.…`) and its bare id, and the same model behind a route + (`anthropic/`, `openrouter/anthropic/`, `bedrock/`, `vertex_ai/`) or a `-v1:0` revision are now one model + everywhere; two different dated snapshots are still two models, and case no longer matters. Effects: the + self-judge warning fires when the answer and the judge are the same model on another provider too; `baseline + compare` no longer reports or refuses (`--strict`) a judge or answer that only changed its route, region or + snapshot alias; `repo compare` pairs arms by the same rule. Scores are not recomputed, but a comparison that + used to refuse or warn about such a pair may now pass. diff --git a/dokimi/docs/benchmarking-and-comparison.md b/dokimi/docs/benchmarking-and-comparison.md index 31fb03f75..cfa562783 100644 --- a/dokimi/docs/benchmarking-and-comparison.md +++ b/dokimi/docs/benchmarking-and-comparison.md @@ -780,11 +780,14 @@ python cli.py baseline compare \ ``` When the answer or judge model recorded in the baseline differs from the -current eval's (a Bedrock alias and its inference profile count as one model), +current eval's (one model under `utils.same_model`: a Bedrock alias and its inference profile, a +dated snapshot and its undated alias, a regional profile, a `-vN:M` revision or another route count +as the same model; two different dated snapshots do not), `compare` prints a warning naming both before the table: the score changes then mix the model change with the content change. With `--strict` it exits 1 instead. A baseline saved before a default-model change is such a baseline. A panel -artifact (`eval panel-score`) is compared by its set of judge models. A role either +artifact (`eval panel-score`) is compared by its judge models: the panels must have the same +number of judges, and each judge must pair with a different judge of the other panel. A role either side does not record (an older artifact, or answers with no model) is named on stderr as not compared, and `--strict` exits 1 on it as well, since it cannot vouch for that role. A malformed record in either file (no `question_id`) exits 2, diff --git a/dokimi/dokimi/commands/baseline.py b/dokimi/dokimi/commands/baseline.py index 20769c53d..0d0ba8881 100644 --- a/dokimi/dokimi/commands/baseline.py +++ b/dokimi/dokimi/commands/baseline.py @@ -11,7 +11,7 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Tuple -from dokimi.utils import normalize_model_ref +from dokimi.utils import same_model BASELINES_DIR = Path("baselines/eval") @@ -88,7 +88,8 @@ def _recorded_models(eval_data: Dict[str, Any]) -> Dict[str, Optional[Tuple[str, The judge is top-level (``judge_provider``/``judge_model``); the answerer is in ``run_metadata``. A panel (``judge_model: "panel(N)"``) records its judges only in - ``panel_config.models``; it maps to ``("panel", )``. A role + ``panel_config.models``; it maps to ``("panel", )``; +``_same_recorded`` splits and compares them. A role the artifact does not record maps to ``None``. """ meta = eval_data.get("run_metadata") @@ -106,8 +107,7 @@ def pair(provider: Any, model: Any) -> Optional[Tuple[str, str]]: models = panel.get("models") if isinstance(panel, dict) else None if not (isinstance(models, list) and all(isinstance(m, str) for m in models)): models = [] - judge = ("panel", ",".join(sorted( - normalize_model_ref(*m.split("/", 1)) if "/" in m else m for m in models))) if models else None + judge = ("panel", ",".join(sorted(models))) if models else None else: judge = pair(eval_data.get("judge_provider") or meta.get("judge_provider"), judge_model) return { @@ -121,7 +121,8 @@ def _model_mismatches(baseline_data: Dict[str, Any], """Per role (answer, judge): lines for a recorded model that differs between the two runs, and the roles that could not be compared because a side does not record them. - A Bedrock alias and its inference profile count as one model. + Two model names that ``same_model`` matches (a Bedrock alias and its profile, a snapshot and its + alias, one model behind another route) count as one model. """ base, cur = _recorded_models(baseline_data), _recorded_models(eval_data) lines, unverified = [], [] @@ -129,11 +130,38 @@ def _model_mismatches(baseline_data: Dict[str, Any], b, c = base[role], cur[role] if not (b and c): unverified.append(role) - elif normalize_model_ref(*b) != normalize_model_ref(*c): + elif not _same_recorded(b, c): lines.append(f"{role} model: baseline {b[0]}/{b[1]}, current {c[0]}/{c[1]}") return lines, unverified +def _same_recorded(b: Tuple[str, str], c: Tuple[str, str]) -> bool: + """Whether two recorded (provider, model) pairs name one model. A panel is the same when both + have the same number of judges and each judge of one pairs with a different judge of the other.""" + if (b[0] == "panel") != (c[0] == "panel"): + return False + if b[0] != "panel": + return same_model(b[1], c[1]) + xs, ys = b[1].split(","), c[1].split(",") + if len(xs) != len(ys): + return False + # same_model is not transitive (a snapshot matches its alias, two snapshots do not match each other), + # so pair the judges by augmenting paths rather than greedily + partner: dict[int, int] = {} # index in ys -> index in xs + + def place(i: int, tried: set[int]) -> bool: + for j, y in enumerate(ys): + if j in tried or not same_model(xs[i], y): + continue + tried.add(j) + if j not in partner or place(partner[j], tried): + partner[j] = i + return True + return False + + return all(place(i, set()) for i in range(len(xs))) + + def cmd_baseline_save(args: argparse.Namespace) -> None: """Save an eval result as a named baseline.""" from datetime import datetime, timezone diff --git a/dokimi/dokimi/commands/evaluate.py b/dokimi/dokimi/commands/evaluate.py index 9d3602f31..c0cef62fd 100644 --- a/dokimi/dokimi/commands/evaluate.py +++ b/dokimi/dokimi/commands/evaluate.py @@ -20,7 +20,7 @@ ) from dokimi.eval.judge import JUDGE_PROVIDERS from dokimi.eval.judge_validity import DEFAULT_MIN_ACCURACY, NoUsablePairs -from dokimi.utils import normalize_model_ref, write_text_or_exit +from dokimi.utils import same_model, write_text_or_exit def _warn_judge_independence( @@ -30,8 +30,8 @@ def _warn_judge_independence( judge_model: str, context: str = "run", ) -> bool: - """Emit a non-fatal warning when answer and judge use the same model/provider.""" - same = normalize_model_ref(answer_provider, answer_model) == normalize_model_ref(judge_provider, judge_model) + """Emit a non-fatal warning when answer and judge are the same model (on any provider route).""" + same = same_model(answer_model, judge_model) if same: print( "⚠ Evaluator independence warning " diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index 81d386a1f..2717fde36 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -75,9 +75,10 @@ from typing import Any from dokimi.eval.stats import adjust_pvalues, bootstrap_ci, mean_ci +from dokimi.utils import same_model from .process_counts import process_order, process_values -from .report import (load_cells, mismatch_warnings, model_matches, model_mismatch, resolved_models, unmeasured_note, +from .report import (load_cells, mismatch_warnings, model_mismatch, resolved_models, unmeasured_note, uptake_counts) #: Largest number of non-zero deltas whose sign-flip distribution is enumerated (2**16 patterns). @@ -198,23 +199,24 @@ def _provenance(batch_dirs: list[Path], cells: list[dict[str, Any]], baseline: s def _models_equivalent(base_models: set[str], treat_models: set[str]) -> bool: - """Whether base and treatment model sets are equivalent under model_matches() logic. + """Whether base and treatment model sets are equivalent under same_model() logic. - Two resolved model sets are equivalent if they differ only by provider prefixes or - version suffixes that model_matches() considers equivalent. For each base model, + Two resolved model sets are equivalent if each model in one matches a model in the other + under same_model(): provider prefixes, regional profiles, revision suffixes, case and + undated snapshot aliases do not count as different models. For each base model, there must be a matching treatment model and the reverse; the set sizes may differ, since one arm can hold several spellings of the same model across repetitions. """ if base_models == treat_models: return True # Check if each base model has an equivalent in treatment models - # (model_matches is already bidirectional, so one call per pair suffices) + # (same_model is symmetric, so one call per pair suffices) for base_model in base_models: - if not any(model_matches(base_model, treat_model) for treat_model in treat_models): + if not any(same_model(base_model, treat_model) for treat_model in treat_models): return False # Check the reverse: every treatment model matches some base model for treat_model in treat_models: - if not any(model_matches(treat_model, base_model) for base_model in base_models): + if not any(same_model(treat_model, base_model) for base_model in base_models): return False return True diff --git a/dokimi/dokimi/repos/report.py b/dokimi/dokimi/repos/report.py index 30a0d6e69..ca51002f2 100644 --- a/dokimi/dokimi/repos/report.py +++ b/dokimi/dokimi/repos/report.py @@ -36,13 +36,14 @@ from __future__ import annotations import json -import re from collections import defaultdict from pathlib import Path from typing import Any import math +from dokimi.utils import same_model + from .process_counts import process_line, process_means from .runner import UNSUPPORTED_FILE, slug @@ -206,47 +207,11 @@ def resolved_models(rec: dict[str, Any]) -> list[str]: return [found] if isinstance(found, str) else [str(m) for m in found] if isinstance(found, list) else [] -def _normalize_model_id(model_id: str) -> str: - """Strip Bedrock's revision suffix: any ``-vN:M`` (``-v1:0``, ``-v2:0``) counts as the same model. - A bare ``-v2`` is part of the id: deepseek-v2 is not v3.""" - return re.sub(r"-v\d+:\d+$", "", model_id) - - -# Bedrock cross-region inference profile prefixes: the same model served from another region. -_REGION_PREFIXES = ("us-gov.", "global.", "apac.", "us.", "eu.", "jp.", "au.", "ca.") - - -def _strip_region(model_id: str) -> str: - for prefix in _REGION_PREFIXES: - if model_id.startswith(prefix): - return model_id[len(prefix):] - return model_id - - -def _model_prefix_matches(base: str, candidate: str) -> bool: - """Check if candidate is base optionally preceded by provider prefix (., /).""" - # base could be the suffix of candidate, with optional prefix - if base == candidate: - return True - # Check if candidate ends with . + base or / + base - return candidate.endswith(("." + base, "/" + base)) - - -def model_matches(requested: str, resolved: str) -> bool: - """Whether ``resolved`` is the ``requested`` model: the same id, that id pinned to a snapshot - date, or either behind a provider prefix (Bedrock's ``us.anthropic.claude-x`` answers as - ``claude-x``, with any ``-vN:M`` revision too).""" - # Vertex's @20250929 is the same snapshot as -20250929; an undated side matches any snapshot of it - req, res = (re.sub(r"@(?=\d{8}$)", "-", _strip_region(_normalize_model_id(m))) for m in (requested, resolved)) - pairs = [(req, res), (re.sub(r"-\d{8}$", "", req), res), (req, re.sub(r"-\d{8}$", "", res))] - return any(_model_prefix_matches(a, b) or _model_prefix_matches(b, a) for a, b in pairs) - - def model_mismatch(cells: list[dict[str, Any]]) -> dict[str, list[str]]: """Requested model -> the other models that answered its cells.""" out: dict[str, set[str]] = defaultdict(set) for rec in cells: - out[rec["model"]] |= {m for m in resolved_models(rec) if not model_matches(rec["model"], m)} + out[rec["model"]] |= {m for m in resolved_models(rec) if not same_model(rec["model"], m)} return {k: sorted(v) for k, v in sorted(out.items()) if v} diff --git a/dokimi/dokimi/utils.py b/dokimi/dokimi/utils.py index cda2ff68c..6547c09d1 100644 --- a/dokimi/dokimi/utils.py +++ b/dokimi/dokimi/utils.py @@ -3,10 +3,11 @@ import json import os +import re from pathlib import Path from typing import Tuple, Dict, Any -from dokimi.defaults import BEDROCK_MODEL_IDS, bedrock_profile +from dokimi.defaults import bedrock_profile def parse_frontmatter(text: str) -> "Tuple[Dict[str, Any], str]": @@ -113,15 +114,45 @@ def retry_writable(func: Any, path: str, _exc: Any) -> None: pass -def normalize_model_ref(provider: str, model: str) -> str: - """Normalize provider and model names for comparison. +# Bedrock cross-region inference profiles: the same model served from another region. They are dropped +# from any provider's name, not only Bedrock's: no non-Bedrock id is known to start with one of them. +_REGION_PROFILES = ("us-gov.", "global.", "apac.", "us.", "eu.", "jp.", "au.", "ca.") - On Bedrock a bare known name and its inference profile are the same model. + +def _model_core(model: str) -> tuple[str, str | None]: + """A model name reduced to ``(id, snapshot date or None)``. + + The route goes (``openrouter/anthropic/``, ``bedrock/``, ``vertex_ai/``, an ARN's + ``inference-profile/``), and so do the case, the region profile (``eu.``) and a Bedrock + revision (``-v1:0``). Vertex's ``@20250929`` is the snapshot ``-20250929``. + """ + name = (model or "").strip().lower().rsplit("/", 1)[-1] + name = re.sub(r"-v\d+:\d+$", "", name) + for prefix in _REGION_PROFILES: + if name.startswith(prefix): + name = name[len(prefix):] + break + name = re.sub(r"@(?=\d{8}$)", "-", name) + snapshot = re.search(r"-(\d{8})$", name) + return (name[:snapshot.start()], snapshot.group(1)) if snapshot else (name, None) + + +def same_model(a: str, b: str) -> bool: + """Whether two model names are one model. + + The same id matches, and so does a dated snapshot and its undated alias (the alias runs + some snapshot of it), or the same id behind another route or region profile. Two dated + ids match only on the same date, and a bare family name (``haiku``) is not a full id. + + The provider is not part of the identity: the same weights on Bedrock and on the direct + API are still one model for the self-judge warning and for a baseline's model check. """ - prov, name = (provider or "").strip().lower(), (model or "").strip() - if prov == "amazon-bedrock": - name = BEDROCK_MODEL_IDS.get(name, name) - return f"{prov}/{name.lower()}" + (base_a, date_a), (base_b, date_b) = _model_core(a), _model_core(b) + if not (base_a and base_b) or (date_a and date_b and date_a != date_b): + return False + # a lowercase word before the first dot is taken as a vendor prefix (``anthropic.``, ``amazon.``) and + # dropped; a dot after a digit or a dash (``gpt-4.1``, ``qwen2.5``) is part of the name + return re.sub(r"^[a-z]+\.", "", base_a) == re.sub(r"^[a-z]+\.", "", base_b) def _read_key(value: str) -> str: diff --git a/dokimi/tests/test_baseline_compare.py b/dokimi/tests/test_baseline_compare.py index 72ffc3733..f9ecb8915 100644 --- a/dokimi/tests/test_baseline_compare.py +++ b/dokimi/tests/test_baseline_compare.py @@ -128,6 +128,29 @@ def test_compare_treats_a_bedrock_alias_and_its_profile_as_one_judge(compare, ca assert capsys.readouterr().err == "" +def test_a_snapshot_its_alias_and_another_route_are_one_judge_model(): + base = _eval(judge=("anthropic", "claude-sonnet-4-5")) + assert baseline_mod._model_mismatches(base, _eval(judge=("anthropic", "claude-sonnet-4-5-20250929"))) == ([], []) + assert baseline_mod._model_mismatches(base, _eval(judge=("amazon-bedrock", "us.anthropic.claude-sonnet-4-5"))) == ( + [], []) + + +def test_a_panel_is_the_same_only_when_its_judges_pair_one_to_one(): + same = baseline_mod._same_recorded + assert same(("panel", "anthropic/m,anthropic/m"), ("panel", "m,anthropic/m")) + assert not same(("panel", "anthropic/m,anthropic/m,anthropic/m"), ("panel", "anthropic/m,anthropic/m")) + assert not same(("panel", "m-20240101,m-20240101"), ("panel", "m,m-20250101")) + assert not same(("panel", "m"), ("anthropic", "m")) + + +def test_two_snapshots_of_one_judge_model_are_a_mismatch(): + lines, _ = baseline_mod._model_mismatches( + _eval(judge=("anthropic", "claude-sonnet-4-5-20240620")), + _eval(judge=("anthropic", "claude-sonnet-4-5-20250929"))) + assert lines == ["judge model: baseline anthropic/claude-sonnet-4-5-20240620, " + "current anthropic/claude-sonnet-4-5-20250929"] + + def test_compare_names_a_role_the_baseline_does_not_record(compare, capsys): old = _eval() del old["run_metadata"] diff --git a/dokimi/tests/test_model_defaults.py b/dokimi/tests/test_model_defaults.py index 300644e76..ed45d416d 100644 --- a/dokimi/tests/test_model_defaults.py +++ b/dokimi/tests/test_model_defaults.py @@ -152,8 +152,10 @@ def test_independence_warning_sees_a_bedrock_alias_and_its_profile_as_one_model( assert _warn_judge_independence("amazon-bedrock", profile, "amazon-bedrock", bare) assert "independence warning" in capsys.readouterr().err assert not _warn_judge_independence("amazon-bedrock", "claude-haiku-5-5", "amazon-bedrock", bare) - # the alias only exists on Bedrock: the same strings on another provider stay distinct - assert not _warn_judge_independence("anthropic", profile, "anthropic", bare) + # the same model answering and judging is the same bias on any route, so it warns on the direct API too + assert _warn_judge_independence("anthropic", profile, "anthropic", bare) + assert _warn_judge_independence("anthropic", "claude-sonnet-5-5", "amazon-bedrock", "claude-sonnet-5-5") + assert not _warn_judge_independence("anthropic", "claude-sonnet-5-5", "anthropic", "claude-haiku-5-5") @pytest.mark.parametrize("model, provider", [ diff --git a/dokimi/tests/test_repo_compare.py b/dokimi/tests/test_repo_compare.py index 9a46eba5d..108cc9e21 100644 --- a/dokimi/tests/test_repo_compare.py +++ b/dokimi/tests/test_repo_compare.py @@ -874,7 +874,7 @@ def test_model_warnings_follow_the_selected_model(tmp_path): @pytest.mark.outcome("cjm-model-answered") -def test_model_matches_equivalence_does_not_warn(tmp_path): +def test_same_model_equivalence_does_not_warn(tmp_path): """Equivalent model IDs (Bedrock provider-prefix variants) do not trigger a warning.""" b = tmp_path / "b1" # Request: Bedrock-prefixed, baseline resolve: shorter form, treatment resolve: base form without version @@ -883,7 +883,7 @@ def test_model_matches_equivalence_does_not_warn(tmp_path): _cell(b, "fix_a", "guidance:raw", 1, model="us.anthropic.claude-haiku-4-5-20251001-v1:0", model_resolved="claude-haiku-4-5-20251001") s = compare([b], "guidance:none") - # No warning about different resolved models—they are equivalent under model_matches() + # No warning about different resolved models—they are equivalent under same_model() assert [w for w in s["warnings"] if "was answered by" in w and "baseline" in w and "treatment" in w] == [] # But cells mismatch still warns (the actual mismatch, not the compare mismatch) @@ -912,7 +912,7 @@ def test_compare_models_equivalent_pins_symmetric_coverage(tmp_path): m = "us.anthropic.claude-haiku-4-5-20251001-v1:0" # Baseline: full Bedrock-prefixed model _cell(b, "fix_a", "guidance:none", 1, model=m, model_resolved="us.anthropic.claude-haiku-4-5-20251001-v1:0") - # Treatment: shorter form (treated as equivalent by model_matches) + # Treatment: shorter form (treated as equivalent by same_model) _cell(b, "fix_a", "guidance:raw", 1, model=m, model_resolved="claude-haiku-4-5-20251001") # No warning because both forms are equivalent assert [w for w in compare([b], "guidance:none")["warnings"] if "was answered by" in w] == [] diff --git a/dokimi/tests/test_repo_run.py b/dokimi/tests/test_repo_run.py index 64753444e..6e232e2b7 100644 --- a/dokimi/tests/test_repo_run.py +++ b/dokimi/tests/test_repo_run.py @@ -15,7 +15,8 @@ from dokimi.cli import build_parser from dokimi.repos.container import CONTENT_DIR, HARNESS_DIR, RUN_DIR, SOURCE_DIR, WORKSPACE -from dokimi.repos.report import format_report, model_matches, summarize, wilson +from dokimi.repos.report import format_report, summarize, wilson +from dokimi.utils import same_model from repo_eval_fixture import FakeRunner, fake_harness, git, make_repo @@ -824,7 +825,7 @@ def test_report_warns_when_another_model_answered(setup, capsys): ("m-v1:0", "m-v3:0", True), ("gpt-4.1", "gpt-4.2", False), ("us.anthropic.claude-x", "eu.anthropic.claude-y", False), ("vertex_ai/claude-x", "claude-x", True), ("claude-sonnet-4-5", "claude-sonnet-4-5-1", False)]) def test_a_provider_prefix_is_not_another_model(requested, resolved, same): - assert model_matches(requested, resolved) is same + assert same_model(requested, resolved) is same @pytest.mark.parametrize("key, value", [("budget_usd", float("nan")), ("timeout_s", float("inf")), diff --git a/dokimi/tests/test_same_model.py b/dokimi/tests/test_same_model.py new file mode 100644 index 000000000..81522c6d1 --- /dev/null +++ b/dokimi/tests/test_same_model.py @@ -0,0 +1,61 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""One rule for "same model": the self-judge warning, `baseline compare` and `repo compare` all use it.""" + +from __future__ import annotations + +import pytest + +from dokimi.defaults import BEDROCK_MODEL_IDS +from dokimi.utils import same_model + +# (a, b, same) with a route-qualified name ("provider/model") where the route is part of the input. +CASES = [ + ("claude-sonnet-4-5", "claude-sonnet-4-5", True), + ("Claude-Sonnet-4-5", "claude-sonnet-4-5", True), + (" claude-sonnet-4-5 ", "claude-sonnet-4-5", True), + ("claude-sonnet-4-5-20250929", "claude-sonnet-4-5", True), # a snapshot and its undated alias + ("claude-3-5-sonnet-20240620", "claude-3-5-sonnet-20241022", False), # two snapshots + ("claude-sonnet-4-5@20250929", "claude-sonnet-4-5-20250929", True), # Vertex date form + ("claude-sonnet-4-5", "claude-opus-4-5-20251101", False), + ("claude-sonnet-4-5", "claude-sonnet-4-5-1", False), + ("haiku", "claude-haiku-3", False), # a family name is not a full id + ("x.claude-haiku", "haiku", False), + ("gpt-4.1", "1", False), # a dot after a digit or dash is not a vendor prefix + ("", "", False), # an unset model is not evidence of sameness + ("claude-sonnet-4-5", "", False), + ("deepseek-v2", "deepseek-v3", False), # a bare -v2 is part of the id + ("us.anthropic.claude-sonnet-4-5", "eu.anthropic.claude-sonnet-4-5", True), # regional profiles + ("global.anthropic.claude-sonnet-4-5", "claude-sonnet-4-5", True), + ("us.anthropic.claude-x", "eu.anthropic.claude-y", False), + ("us.anthropic.claude-haiku-4-5-20251001-v1:0", "claude-haiku-4-5-20251001", True), + ("eu.anthropic.claude-sonnet-4-5-20250929-v1:0", "us.anthropic.claude-sonnet-4-5", True), + ("m-v1:0", "m-v3:0", True), # Bedrock revision suffix + ("claude-sonnet-4-5", "us.anthropic.claude-sonnet-4-5-20250929-v1:0", True), # alias vs full Bedrock id + ("claude-x-20250929", "claude-x-20241022", False), + ("anthropic/claude-sonnet-4-5", "amazon-bedrock/claude-sonnet-4-5", True), # same model, another route + ("anthropic/claude-x", "claude-x", True), + ("vertex_ai/claude-x", "claude-x", True), + ("vertex_ai/claude-x@20250929", "vertex_ai/claude-x", True), + ("bedrock/us.anthropic.claude-x", "us.anthropic.claude-x", True), + ("amazon-bedrock/us.anthropic.claude-sonnet-4-5", "amazon-bedrock/eu.anthropic.claude-sonnet-4-5", True), + ("openrouter/anthropic/claude-sonnet-4-5", "anthropic/claude-sonnet-4-5", True), + ("anthropic/claude-sonnet-5-5", "anthropic/us.anthropic.claude-sonnet-5-5", True), +] + + +@pytest.mark.parametrize("a, b, same", CASES) +def test_same_model_table(a, b, same): + assert same_model(a, b) is same + + +@pytest.mark.parametrize("a, b, same", CASES) +def test_same_model_is_symmetric(a, b, same): + assert same_model(b, a) is same + + +@pytest.mark.parametrize("bare, profile", sorted(BEDROCK_MODEL_IDS.items())) +def test_a_bedrock_alias_and_its_profile_are_one_model(bare, profile): + assert same_model(bare, profile) + assert same_model(f"amazon-bedrock/{bare}", f"amazon-bedrock/{profile}") From 58eafbab4f230cc25a6a08317b13ad4b13c4e491 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:39:52 -0700 Subject: [PATCH 02/18] Address review: dokimi run stores the self-judge flag by same_model The pipeline compared provider and model as exact strings, so the printed warning and run_metadata.evaluator_independence_warning disagreed for one model on two routes. Use same_model, as eval does. Thread: PRRT_kwDOVBg7g86xgsTm Thread: PRR_kwDOVBg7g88AAAABU4ae8g Signed-off-by: Nikolay Petrov --- ...28_130000_napetrov_one-model-match-rule.md | 2 + dokimi/dokimi/orchestrator/pipeline.py | 6 +-- .../tests/test_pipeline_independence_flag.py | 43 +++++++++++++++++++ 3 files changed, 47 insertions(+), 4 deletions(-) create mode 100644 dokimi/tests/test_pipeline_independence_flag.py diff --git a/dokimi/changelog.d/20261028_130000_napetrov_one-model-match-rule.md b/dokimi/changelog.d/20261028_130000_napetrov_one-model-match-rule.md index 2ce3c4134..026ce2e40 100644 --- a/dokimi/changelog.d/20261028_130000_napetrov_one-model-match-rule.md +++ b/dokimi/changelog.d/20261028_130000_napetrov_one-model-match-rule.md @@ -9,3 +9,5 @@ compare` no longer reports or refuses (`--strict`) a judge or answer that only changed its route, region or snapshot alias; `repo compare` pairs arms by the same rule. Scores are not recomputed, but a comparison that used to refuse or warn about such a pair may now pass. + `dokimi run` stores `run_metadata.evaluator_independence_warning` by this rule too, as `dokimi eval` does (it + was an exact provider-and-model match, so the printed warning and the stored flag disagreed). diff --git a/dokimi/dokimi/orchestrator/pipeline.py b/dokimi/dokimi/orchestrator/pipeline.py index 99cd48029..a8f2dc4fe 100644 --- a/dokimi/dokimi/orchestrator/pipeline.py +++ b/dokimi/dokimi/orchestrator/pipeline.py @@ -12,6 +12,7 @@ from dokimi.defaults import DEFAULT_ANSWER_MODEL, DEFAULT_ANSWER_PROVIDER, DEFAULT_JUDGE_MODEL, DEFAULT_JUDGE_PROVIDER, DEFAULT_RETRIEVAL_TOP_K from dokimi.eval.stats import paired_summary from dokimi.mcp.factory import create_doc_source_client +from dokimi.utils import same_model logger = logging.getLogger(__name__) @@ -669,10 +670,7 @@ def _evaluate_answers(self, answers: List[Dict[str, Any]] | Dict[str, Any], "judge_model": self.judge_model, "judge_provider": self.judge_provider, "question_set_hash": question_set_hash, - "evaluator_independence_warning": ( - str(self.model).strip().lower() == str(self.judge_model).strip().lower() - and str(self.provider).strip().lower() == str(self.judge_provider).strip().lower() - ), + "evaluator_independence_warning": same_model(str(self.model), str(self.judge_model)), }, ) diff --git a/dokimi/tests/test_pipeline_independence_flag.py b/dokimi/tests/test_pipeline_independence_flag.py new file mode 100644 index 000000000..ac17fd288 --- /dev/null +++ b/dokimi/tests/test_pipeline_independence_flag.py @@ -0,0 +1,43 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""``dokimi run`` stores the same ``evaluator_independence_warning`` that ``dokimi eval`` does.""" + +from __future__ import annotations + +import types + +import pytest + +from dokimi.orchestrator.pipeline import EvaluationPipeline + + +@pytest.mark.parametrize("provider, model, judge_provider, judge_model, expected", [ + ("anthropic", "claude-sonnet-5-5", "amazon-bedrock", "claude-sonnet-5-5", True), # one model, two routes + ("anthropic", "claude-sonnet-5-5", "anthropic", "claude-sonnet-5-5", True), + ("amazon-bedrock", "us.anthropic.claude-sonnet-5-5-v1:0", "amazon-bedrock", "claude-sonnet-5-5", True), + ("anthropic", "claude-sonnet-5-5", "anthropic", "claude-haiku-5-5", False), +]) +def test_the_stored_flag_follows_the_same_model_rule(tmp_path, monkeypatch, provider, model, judge_provider, + judge_model, expected): + seen = {} + + class FakeJudge: + def __init__(self, **kwargs): + seen.update(kwargs["run_metadata"]) + + def evaluate_answers(self, *args, **kwargs): + return [] + + def save_evaluations(self, *args, **kwargs): + pass + + monkeypatch.setattr("dokimi.eval.Judge", FakeJudge) + monkeypatch.setattr("dokimi.records.read", lambda *a, **k: types.SimpleNamespace(records=[{"id": "q1"}])) + pipeline = EvaluationPipeline(product="oneTBB", output_dir=tmp_path, description="d", model=model, + provider=provider, + judge_model=judge_model, judge_provider=judge_provider) + + pipeline._evaluate_answers([{"id": "q1"}]) + + assert seen["evaluator_independence_warning"] is expected From 020b53eb173e04fa3a6a504ec04db518de4a559f Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:41:17 -0700 Subject: [PATCH 03/18] Address review: same_model keeps the organization of an org/name id Only a known route (provider or gateway prefix, an ARN's inference-profile/) is dropped; the rest of the path stays in the id, so meta-llama/X and myorg/X are two models. Thread: PRRT_kwDOVBg7g86xgsT1 Signed-off-by: Nikolay Petrov --- ...028_130000_napetrov_one-model-match-rule.md | 2 ++ dokimi/dokimi/utils.py | 18 ++++++++++++++++-- dokimi/tests/test_same_model.py | 9 +++++++++ 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/dokimi/changelog.d/20261028_130000_napetrov_one-model-match-rule.md b/dokimi/changelog.d/20261028_130000_napetrov_one-model-match-rule.md index 026ce2e40..bc76e1043 100644 --- a/dokimi/changelog.d/20261028_130000_napetrov_one-model-match-rule.md +++ b/dokimi/changelog.d/20261028_130000_napetrov_one-model-match-rule.md @@ -11,3 +11,5 @@ used to refuse or warn about such a pair may now pass. `dokimi run` stores `run_metadata.evaluator_independence_warning` by this rule too, as `dokimi eval` does (it was an exact provider-and-model match, so the printed warning and the stored flag disagreed). + A route is a provider or gateway prefix; the organization of an `org/name` id is part of the id, so + `meta-llama/X` and `myorg/X` are two models. diff --git a/dokimi/dokimi/utils.py b/dokimi/dokimi/utils.py index 6547c09d1..b7f8cff0b 100644 --- a/dokimi/dokimi/utils.py +++ b/dokimi/dokimi/utils.py @@ -119,14 +119,28 @@ def retry_writable(func: Any, path: str, _exc: Any) -> None: _REGION_PROFILES = ("us-gov.", "global.", "apac.", "us.", "eu.", "jp.", "au.", "ca.") +# Leading path segments that name how a model is reached, not which model it is: a provider or a gateway +# (``anthropic/``, ``openrouter/``, ``bedrock/converse/``, ``vertex_ai/``). Any other segment (a Hugging Face +# organization such as ``meta-llama/``) is part of the id. +_ROUTES = frozenset({"openai", "anthropic", "google", "google-vertex", "vertex_ai", "openrouter", "amazon-bedrock", + "bedrock", "converse"}) + + def _model_core(model: str) -> tuple[str, str | None]: """A model name reduced to ``(id, snapshot date or None)``. The route goes (``openrouter/anthropic/``, ``bedrock/``, ``vertex_ai/``, an ARN's ``inference-profile/``), and so do the case, the region profile (``eu.``) and a Bedrock - revision (``-v1:0``). Vertex's ``@20250929`` is the snapshot ``-20250929``. + revision (``-v1:0``). Vertex's ``@20250929`` is the snapshot ``-20250929``. The organization of an + ``org/name`` id stays, so a re-upload under another organization is another model. """ - name = (model or "").strip().lower().rsplit("/", 1)[-1] + name = (model or "").strip().lower() + if "inference-profile/" in name: + name = name.rsplit("inference-profile/", 1)[-1] + parts = name.split("/") + while len(parts) > 1 and parts[0] in _ROUTES: + parts.pop(0) + name = "/".join(parts) name = re.sub(r"-v\d+:\d+$", "", name) for prefix in _REGION_PROFILES: if name.startswith(prefix): diff --git a/dokimi/tests/test_same_model.py b/dokimi/tests/test_same_model.py index 81522c6d1..36031a61b 100644 --- a/dokimi/tests/test_same_model.py +++ b/dokimi/tests/test_same_model.py @@ -42,6 +42,15 @@ ("amazon-bedrock/us.anthropic.claude-sonnet-4-5", "amazon-bedrock/eu.anthropic.claude-sonnet-4-5", True), ("openrouter/anthropic/claude-sonnet-4-5", "anthropic/claude-sonnet-4-5", True), ("anthropic/claude-sonnet-5-5", "anthropic/us.anthropic.claude-sonnet-5-5", True), + # the organization is part of a Hugging Face style id; only a route is dropped + ("meta-llama/Llama-3.1-8B-Instruct", "myorg/Llama-3.1-8B-Instruct", False), + ("meta-llama/Llama-3.1-8B-Instruct", "meta-llama/Llama-3.1-8B-Instruct", True), + ("openrouter/meta-llama/llama-3.1-8b-instruct", "meta-llama/Llama-3.1-8B-Instruct", True), + ("openrouter/meta-llama/llama-3.1-8b-instruct", "openrouter/myorg/llama-3.1-8b-instruct", False), + ("meta-llama/Llama-3.1-8B-Instruct", "Llama-3.1-8B-Instruct", False), + ("arn:aws:bedrock:us-east-1:123456789012:inference-profile/us.anthropic.claude-sonnet-4-5-20250929-v1:0", + "claude-sonnet-4-5-20250929", True), + ("bedrock/converse/openai.gpt-oss-120b", "openai/gpt-oss-120b", True), ] From 1752b920abfb26e14149d3be16ef0971ed91ca59 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:41:30 -0700 Subject: [PATCH 04/18] Address review: same_model strips a plain -N:M Bedrock revision openai.gpt-oss-120b-1:0 did not match gpt-oss-120b, so the self-judge warning missed it and repo compare reported a change that did not happen. Thread: PRRT_kwDOVBg7g86xgsT- Signed-off-by: Nikolay Petrov --- dokimi/dokimi/utils.py | 4 ++-- dokimi/tests/test_same_model.py | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/utils.py b/dokimi/dokimi/utils.py index b7f8cff0b..224861e32 100644 --- a/dokimi/dokimi/utils.py +++ b/dokimi/dokimi/utils.py @@ -131,7 +131,7 @@ def _model_core(model: str) -> tuple[str, str | None]: The route goes (``openrouter/anthropic/``, ``bedrock/``, ``vertex_ai/``, an ARN's ``inference-profile/``), and so do the case, the region profile (``eu.``) and a Bedrock - revision (``-v1:0``). Vertex's ``@20250929`` is the snapshot ``-20250929``. The organization of an + revision (``-v1:0`` or ``-1:0``). Vertex's ``@20250929`` is the snapshot ``-20250929``. The organization of an ``org/name`` id stays, so a re-upload under another organization is another model. """ name = (model or "").strip().lower() @@ -141,7 +141,7 @@ def _model_core(model: str) -> tuple[str, str | None]: while len(parts) > 1 and parts[0] in _ROUTES: parts.pop(0) name = "/".join(parts) - name = re.sub(r"-v\d+:\d+$", "", name) + name = re.sub(r"-v?\d+:\d+$", "", name) for prefix in _REGION_PROFILES: if name.startswith(prefix): name = name[len(prefix):] diff --git a/dokimi/tests/test_same_model.py b/dokimi/tests/test_same_model.py index 36031a61b..cecb37387 100644 --- a/dokimi/tests/test_same_model.py +++ b/dokimi/tests/test_same_model.py @@ -32,6 +32,9 @@ ("us.anthropic.claude-haiku-4-5-20251001-v1:0", "claude-haiku-4-5-20251001", True), ("eu.anthropic.claude-sonnet-4-5-20250929-v1:0", "us.anthropic.claude-sonnet-4-5", True), ("m-v1:0", "m-v3:0", True), # Bedrock revision suffix + ("openai.gpt-oss-120b-1:0", "gpt-oss-120b", True), # a plain -N:M revision + ("openai.gpt-oss-120b-1:0", "openai.gpt-oss-120b-2:0", True), + ("openai.gpt-oss-120b-1:0", "gpt-oss-20b", False), ("claude-sonnet-4-5", "us.anthropic.claude-sonnet-4-5-20250929-v1:0", True), # alias vs full Bedrock id ("claude-x-20250929", "claude-x-20241022", False), ("anthropic/claude-sonnet-4-5", "amazon-bedrock/claude-sonnet-4-5", True), # same model, another route From 1e27fcdb7b34b32176c86400dd6b981cb19d9c84 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:41:41 -0700 Subject: [PATCH 05/18] Address review: same_model reads OpenAI's dashed snapshot ids gpt-4o-2024-08-06 did not match gpt-4o, so the self-judge warning was skipped and a baseline check reported a false model mismatch. A dashed and a compact date of one day compare equal. Thread: PRRT_kwDOVBg7g86xlsyt Signed-off-by: Nikolay Petrov --- dokimi/dokimi/utils.py | 6 +++--- dokimi/tests/test_same_model.py | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/dokimi/dokimi/utils.py b/dokimi/dokimi/utils.py index 224861e32..a47071e13 100644 --- a/dokimi/dokimi/utils.py +++ b/dokimi/dokimi/utils.py @@ -127,7 +127,7 @@ def retry_writable(func: Any, path: str, _exc: Any) -> None: def _model_core(model: str) -> tuple[str, str | None]: - """A model name reduced to ``(id, snapshot date or None)``. + """A model name reduced to ``(id, snapshot date or None)``. A snapshot is ``-YYYYMMDD`` or OpenAI's ``-YYYY-MM-DD``. The route goes (``openrouter/anthropic/``, ``bedrock/``, ``vertex_ai/``, an ARN's ``inference-profile/``), and so do the case, the region profile (``eu.``) and a Bedrock @@ -147,8 +147,8 @@ def _model_core(model: str) -> tuple[str, str | None]: name = name[len(prefix):] break name = re.sub(r"@(?=\d{8}$)", "-", name) - snapshot = re.search(r"-(\d{8})$", name) - return (name[:snapshot.start()], snapshot.group(1)) if snapshot else (name, None) + snapshot = re.search(r"-(\d{8}|\d{4}-\d{2}-\d{2})$", name) + return (name[:snapshot.start()], snapshot.group(1).replace("-", "")) if snapshot else (name, None) def same_model(a: str, b: str) -> bool: diff --git a/dokimi/tests/test_same_model.py b/dokimi/tests/test_same_model.py index cecb37387..9b4f147d9 100644 --- a/dokimi/tests/test_same_model.py +++ b/dokimi/tests/test_same_model.py @@ -37,6 +37,9 @@ ("openai.gpt-oss-120b-1:0", "gpt-oss-20b", False), ("claude-sonnet-4-5", "us.anthropic.claude-sonnet-4-5-20250929-v1:0", True), # alias vs full Bedrock id ("claude-x-20250929", "claude-x-20241022", False), + ("gpt-4o-2024-08-06", "gpt-4o", True), # OpenAI's dashed snapshot and its alias + ("gpt-4o-2024-08-06", "gpt-4o-2024-05-13", False), + ("gpt-4o-2024-08-06", "gpt-4o-20240806", True), ("anthropic/claude-sonnet-4-5", "amazon-bedrock/claude-sonnet-4-5", True), # same model, another route ("anthropic/claude-x", "claude-x", True), ("vertex_ai/claude-x", "claude-x", True), From 19c701f0c78761d54d9fc58de07cf4698af16396 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:41:47 -0700 Subject: [PATCH 06/18] Address review: rewrap the _recorded_models docstring The edited line was 123 characters and the next one started at column 0. Thread: PRRT_kwDOVBg7g86xgsUD Thread: PRRT_kwDOVBg7g86xlsyi Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/baseline.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dokimi/dokimi/commands/baseline.py b/dokimi/dokimi/commands/baseline.py index 0d0ba8881..8d4cfc538 100644 --- a/dokimi/dokimi/commands/baseline.py +++ b/dokimi/dokimi/commands/baseline.py @@ -88,9 +88,9 @@ def _recorded_models(eval_data: Dict[str, Any]) -> Dict[str, Optional[Tuple[str, The judge is top-level (``judge_provider``/``judge_model``); the answerer is in ``run_metadata``. A panel (``judge_model: "panel(N)"``) records its judges only in - ``panel_config.models``; it maps to ``("panel", )``; -``_same_recorded`` splits and compares them. A role - the artifact does not record maps to ``None``. + ``panel_config.models``; it maps to ``("panel", )``, which ``_same_recorded`` splits and compares. A role the + artifact does not record maps to ``None``. """ meta = eval_data.get("run_metadata") if not isinstance(meta, dict): From d48334f24830322a29f8d48a5c6fc6ed61c9e1ed Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:42:14 -0700 Subject: [PATCH 07/18] Address review: pin the panel-size check of _same_recorded A baseline panel of two judges against a current panel of three, where every baseline judge pairs, was accepted when the size check was deleted. Thread: PRRT_kwDOVBg7g86xiXNo Signed-off-by: Nikolay Petrov --- dokimi/tests/test_baseline_compare.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dokimi/tests/test_baseline_compare.py b/dokimi/tests/test_baseline_compare.py index f9ecb8915..6265fdef5 100644 --- a/dokimi/tests/test_baseline_compare.py +++ b/dokimi/tests/test_baseline_compare.py @@ -143,6 +143,14 @@ def test_a_panel_is_the_same_only_when_its_judges_pair_one_to_one(): assert not same(("panel", "m"), ("anthropic", "m")) +def test_a_panel_that_gained_a_judge_is_a_mismatch(): + """Every judge of the baseline panel can pair with one of the current, and a third one is added: + pairing alone would pass it, so the sizes are compared first.""" + same = baseline_mod._same_recorded + assert not same(("panel", "m,m-20240101"), ("panel", "m,m-20240101,m")) + assert not same(("panel", "m,m-20240101,m"), ("panel", "m,m-20240101")) + + def test_two_snapshots_of_one_judge_model_are_a_mismatch(): lines, _ = baseline_mod._model_mismatches( _eval(judge=("anthropic", "claude-sonnet-4-5-20240620")), From 8a2f9527ee9791de8989dd8f0452f8c071ad70b2 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:42:30 -0700 Subject: [PATCH 08/18] Address review: pin the augmenting-path pairing of _same_recorded Baseline m,m-20240101 against current m-20240101,m-20250101 is the same panel, and only an augmenting path finds the pairing: a first-match pairing strands the second judge. Replacing the augment step with a first match left the suite green; the new case, in both orders, fails it. Thread: PRRT_kwDOVBg7g86xiXN0 Thread: PRRT_kwDOVBg7g86xlsy9 Thread: PRR_kwDOVBg7g88AAAABU5SrZA Signed-off-by: Nikolay Petrov --- dokimi/tests/test_baseline_compare.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/dokimi/tests/test_baseline_compare.py b/dokimi/tests/test_baseline_compare.py index 6265fdef5..cd06041a7 100644 --- a/dokimi/tests/test_baseline_compare.py +++ b/dokimi/tests/test_baseline_compare.py @@ -143,6 +143,16 @@ def test_a_panel_is_the_same_only_when_its_judges_pair_one_to_one(): assert not same(("panel", "m"), ("anthropic", "m")) +def test_a_panel_pairs_its_judges_by_augmenting_paths_not_first_match(): + """``m`` matches both snapshots and ``m-20240101`` only its own: a first-match pairing gives ``m`` the + judge ``m-20240101`` and strands the second baseline judge, though a valid pairing exists.""" + same = baseline_mod._same_recorded + assert same(("panel", "m,m-20240101"), ("panel", "m-20240101,m-20250101")) + assert same(("panel", "m-20240101,m-20250101"), ("panel", "m,m-20240101")) # the reverse side and order + assert same(("panel", "m-20240101,m"), ("panel", "m-20240101,m-20250101")) + assert not same(("panel", "m-20240101,m-20240101"), ("panel", "m,m-20250101")) + + def test_a_panel_that_gained_a_judge_is_a_mismatch(): """Every judge of the baseline panel can pair with one of the current, and a third one is added: pairing alone would pass it, so the sizes are compared first.""" From 795dcb29291cf3030be4935a2f3a1aa5d3a24509 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:42:53 -0700 Subject: [PATCH 09/18] Address review: the independence tests check what is printed to stderr The assertions checked only the return value. A warning printed for every pair passed the old test; the new one fails it. It also covers an unset answer or judge model, which must not warn. Thread: PRRT_kwDOVBg7g86xlszG Signed-off-by: Nikolay Petrov --- dokimi/tests/test_model_defaults.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dokimi/tests/test_model_defaults.py b/dokimi/tests/test_model_defaults.py index ed45d416d..07c7d1e01 100644 --- a/dokimi/tests/test_model_defaults.py +++ b/dokimi/tests/test_model_defaults.py @@ -153,9 +153,17 @@ def test_independence_warning_sees_a_bedrock_alias_and_its_profile_as_one_model( assert "independence warning" in capsys.readouterr().err assert not _warn_judge_independence("amazon-bedrock", "claude-haiku-5-5", "amazon-bedrock", bare) # the same model answering and judging is the same bias on any route, so it warns on the direct API too + capsys.readouterr() assert _warn_judge_independence("anthropic", profile, "anthropic", bare) + assert "independence warning" in capsys.readouterr().err assert _warn_judge_independence("anthropic", "claude-sonnet-5-5", "amazon-bedrock", "claude-sonnet-5-5") + assert "independence warning" in capsys.readouterr().err assert not _warn_judge_independence("anthropic", "claude-sonnet-5-5", "anthropic", "claude-haiku-5-5") + assert capsys.readouterr().err == "" + # an unset model is not evidence of sameness + assert not _warn_judge_independence("anthropic", "", "anthropic", "") + assert not _warn_judge_independence("anthropic", "claude-sonnet-5-5", "anthropic", "") + assert capsys.readouterr().err == "" @pytest.mark.parametrize("model, provider", [ From 82c42ad102333bbc43fef908510876e03a291455 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:43:02 -0700 Subject: [PATCH 10/18] Address review: say in the docs that the organization is part of a model id Thread: PRRT_kwDOVBg7g86xgsT1 Signed-off-by: Nikolay Petrov --- dokimi/docs/benchmarking-and-comparison.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/docs/benchmarking-and-comparison.md b/dokimi/docs/benchmarking-and-comparison.md index f46b5ad57..64e8b6f35 100644 --- a/dokimi/docs/benchmarking-and-comparison.md +++ b/dokimi/docs/benchmarking-and-comparison.md @@ -788,7 +788,7 @@ python cli.py baseline compare \ When the answer or judge model recorded in the baseline differs from the current eval's (one model under `utils.same_model`: a Bedrock alias and its inference profile, a dated snapshot and its undated alias, a regional profile, a `-vN:M` revision or another route count -as the same model; two different dated snapshots do not), +as the same model; two different dated snapshots, or the same name under another organization, do not), `compare` prints a warning naming both before the table: the score changes then mix the model change with the content change. With `--strict` it exits 1 instead. A baseline saved before a default-model change is such a baseline. A panel From 7140e6dccaf1e15cd3b905bf90f1eca5c63e07b4 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:49:19 -0700 Subject: [PATCH 11/18] Address review: same_model drops every route dokimi accepts, not only nine Found by the self-review of this round: with the organization kept, a route missing from the list (azure/, gemini/, groq/, models/, a foundation-model ARN) read as an organization, so azure/gpt-4o did not match gpt-4o. Add the litellm routes, strip a foundation-model ARN like an inference profile, and pin that every provider in llm.PROVIDERS and its aliases is a route. Thread: PRRT_kwDOVBg7g86xgsT1 Signed-off-by: Nikolay Petrov --- dokimi/dokimi/utils.py | 17 +++++++++++------ dokimi/tests/test_same_model.py | 13 +++++++++++++ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/dokimi/dokimi/utils.py b/dokimi/dokimi/utils.py index a47071e13..485d1503a 100644 --- a/dokimi/dokimi/utils.py +++ b/dokimi/dokimi/utils.py @@ -120,10 +120,14 @@ def retry_writable(func: Any, path: str, _exc: Any) -> None: # Leading path segments that name how a model is reached, not which model it is: a provider or a gateway -# (``anthropic/``, ``openrouter/``, ``bedrock/converse/``, ``vertex_ai/``). Any other segment (a Hugging Face -# organization such as ``meta-llama/``) is part of the id. -_ROUTES = frozenset({"openai", "anthropic", "google", "google-vertex", "vertex_ai", "openrouter", "amazon-bedrock", - "bedrock", "converse"}) +# (``anthropic/``, ``openrouter/``, ``bedrock/converse/``, ``azure/``). Any other segment (a Hugging Face +# organization such as ``meta-llama/``) is part of the id. Every provider of ``llm.PROVIDERS`` and its aliases is +# one (pinned by a test); the rest are litellm route prefixes. +_ROUTES = frozenset({ + "openai", "openai-codex", "anthropic", "google", "gemini", "google-vertex", "openrouter", "amazon-bedrock", + "bedrock", "converse", "vertex_ai", "azure", "azure_ai", "models", "groq", "ollama", "mistral", "deepseek", + "together_ai", "fireworks_ai", "xai", "cohere", "huggingface", +}) def _model_core(model: str) -> tuple[str, str | None]: @@ -135,8 +139,9 @@ def _model_core(model: str) -> tuple[str, str | None]: ``org/name`` id stays, so a re-upload under another organization is another model. """ name = (model or "").strip().lower() - if "inference-profile/" in name: - name = name.rsplit("inference-profile/", 1)[-1] + for marker in ("inference-profile/", "foundation-model/"): # the tail of a Bedrock ARN + if marker in name: + name = name.rsplit(marker, 1)[-1] parts = name.split("/") while len(parts) > 1 and parts[0] in _ROUTES: parts.pop(0) diff --git a/dokimi/tests/test_same_model.py b/dokimi/tests/test_same_model.py index 9b4f147d9..1e4997205 100644 --- a/dokimi/tests/test_same_model.py +++ b/dokimi/tests/test_same_model.py @@ -57,6 +57,12 @@ ("arn:aws:bedrock:us-east-1:123456789012:inference-profile/us.anthropic.claude-sonnet-4-5-20250929-v1:0", "claude-sonnet-4-5-20250929", True), ("bedrock/converse/openai.gpt-oss-120b", "openai/gpt-oss-120b", True), + ("azure/gpt-4o", "gpt-4o", True), # a litellm route that is not one of dokimi's providers + ("groq/llama-3.1-8b", "llama-3.1-8b", True), + ("together_ai/meta-llama/llama-3-8b", "meta-llama/llama-3-8b", True), + ("models/gemini-2.0-flash", "gemini-2.0-flash", True), + ("arn:aws:bedrock:us-east-1::foundation-model/anthropic.claude-sonnet-4-5-20250929-v1:0", + "claude-sonnet-4-5-20250929", True), ] @@ -74,3 +80,10 @@ def test_same_model_is_symmetric(a, b, same): def test_a_bedrock_alias_and_its_profile_are_one_model(bare, profile): assert same_model(bare, profile) assert same_model(f"amazon-bedrock/{bare}", f"amazon-bedrock/{profile}") + + +def test_every_provider_dokimi_accepts_is_a_route_not_an_organization(): + from dokimi.llm import PROVIDER_ALIASES, PROVIDERS + + for provider in [*PROVIDERS, *PROVIDER_ALIASES]: + assert same_model(f"{provider}/gpt-4o", "gpt-4o"), provider From 66f715b191593048f62234c08e198492a0010d17 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 18:08:40 -0700 Subject: [PATCH 12/18] fix(baseline): a dated snapshot against its undated alias is not verified `baseline compare` matched a dated judge against its undated alias through same_model, so --strict passed a judge change the alias cannot show: the alias does not record which snapshot answered. Compare the exact identity (id, snapshot date, route-free) first. A pair that only same_model links, including a panel that pairs only through such an alias, is reported as not verified: named on stderr, refused by --strict. Anything else is still a mismatch. The self-judge warning and repo compare keep the alias rule. Signed-off-by: Nikolay Petrov --- ...28_130000_napetrov_one-model-match-rule.md | 8 ++- dokimi/docs/benchmarking-and-comparison.md | 17 +++-- dokimi/dokimi/commands/baseline.py | 71 +++++++++++++------ dokimi/dokimi/utils.py | 19 +++-- dokimi/tests/test_baseline_compare.py | 70 +++++++++++++----- 5 files changed, 132 insertions(+), 53 deletions(-) diff --git a/dokimi/changelog.d/20261028_130000_napetrov_one-model-match-rule.md b/dokimi/changelog.d/20261028_130000_napetrov_one-model-match-rule.md index bc76e1043..2e7f30777 100644 --- a/dokimi/changelog.d/20261028_130000_napetrov_one-model-match-rule.md +++ b/dokimi/changelog.d/20261028_130000_napetrov_one-model-match-rule.md @@ -6,9 +6,11 @@ (`anthropic/`, `openrouter/anthropic/`, `bedrock/`, `vertex_ai/`) or a `-v1:0` revision are now one model everywhere; two different dated snapshots are still two models, and case no longer matters. Effects: the self-judge warning fires when the answer and the judge are the same model on another provider too; `baseline - compare` no longer reports or refuses (`--strict`) a judge or answer that only changed its route, region or - snapshot alias; `repo compare` pairs arms by the same rule. Scores are not recomputed, but a comparison that - used to refuse or warn about such a pair may now pass. + compare` no longer reports or refuses (`--strict`) a judge or answer that only changed its route or region; + `repo compare` pairs arms by the same rule. A dated snapshot against its undated alias is the exception in + `baseline compare`: the alias does not record which snapshot answered, so it is named on stderr as not + verified and `--strict` refuses it, as before. Scores are not recomputed, but a comparison + that used to refuse or warn about such a pair may now pass. `dokimi run` stores `run_metadata.evaluator_independence_warning` by this rule too, as `dokimi eval` does (it was an exact provider-and-model match, so the printed warning and the stored flag disagreed). A route is a provider or gateway prefix; the organization of an `org/name` id is part of the id, so diff --git a/dokimi/docs/benchmarking-and-comparison.md b/dokimi/docs/benchmarking-and-comparison.md index 64e8b6f35..e007a7ec0 100644 --- a/dokimi/docs/benchmarking-and-comparison.md +++ b/dokimi/docs/benchmarking-and-comparison.md @@ -786,14 +786,17 @@ python cli.py baseline compare \ ``` When the answer or judge model recorded in the baseline differs from the -current eval's (one model under `utils.same_model`: a Bedrock alias and its inference profile, a -dated snapshot and its undated alias, a regional profile, a `-vN:M` revision or another route count -as the same model; two different dated snapshots, or the same name under another organization, do not), -`compare` prints a warning naming both before the table: the score changes then +current eval's, `compare` prints a warning naming both before the table: the score changes then mix the model change with the content change. With `--strict` it exits 1 -instead. A baseline saved before a default-model change is such a baseline. A panel -artifact (`eval panel-score`) is compared by its judge models: the panels must have the same -number of judges, and each judge must pair with a different judge of the other panel. A role either +instead. Two records name one model when they have the same id and snapshot: a Bedrock alias and its +inference profile, a regional profile, a `-vN:M` revision or the same model behind another route count +as the same model; two different dated snapshots, or the same name under another organization, are two models. +A dated snapshot and its undated alias are neither: the alias does not record which snapshot answered, so the +role is named on stderr as not verified, and `--strict` exits 1 on it. A baseline saved before a default-model +change is such a baseline. A panel +artifact (`eval panel-score`) is compared by its judge models: the panels must have the same number of judges +and the same models, and a judge that matches the other panel's only through such an alias makes the role +not verified. A role either side does not record (an older artifact, or answers with no model) is named on stderr as not compared, and `--strict` exits 1 on it as well, since it cannot vouch for that role. A malformed record in either file (no `question_id`) exits 2, diff --git a/dokimi/dokimi/commands/baseline.py b/dokimi/dokimi/commands/baseline.py index 8d4cfc538..2069c4a69 100644 --- a/dokimi/dokimi/commands/baseline.py +++ b/dokimi/dokimi/commands/baseline.py @@ -8,10 +8,11 @@ import argparse import json import sys +from collections import Counter from pathlib import Path from typing import Any, Dict, List, Optional, Tuple -from dokimi.utils import same_model +from dokimi.utils import model_identity, same_model BASELINES_DIR = Path("baselines/eval") @@ -89,7 +90,7 @@ def _recorded_models(eval_data: Dict[str, Any]) -> Dict[str, Optional[Tuple[str, The judge is top-level (``judge_provider``/``judge_model``); the answerer is in ``run_metadata``. A panel (``judge_model: "panel(N)"``) records its judges only in ``panel_config.models``; it maps to ``("panel", )``, which ``_same_recorded`` splits and compares. A role the + sorted and comma-joined>)``, which ``_recorded_verdict`` splits and compares. A role the artifact does not record maps to ``None``. """ meta = eval_data.get("run_metadata") @@ -119,10 +120,11 @@ def pair(provider: Any, model: Any) -> Optional[Tuple[str, str]]: def _model_mismatches(baseline_data: Dict[str, Any], eval_data: Dict[str, Any]) -> Tuple[List[str], List[str]]: """Per role (answer, judge): lines for a recorded model that differs between the two - runs, and the roles that could not be compared because a side does not record them. + runs, and the roles that cannot be verified: a side does not record them, or they are a dated + snapshot against an undated alias (see ``_recorded_verdict``). - Two model names that ``same_model`` matches (a Bedrock alias and its profile, a snapshot and its - alias, one model behind another route) count as one model. + Two model names with the same id, snapshot and route (a Bedrock alias and its profile, one model + behind another route) count as one model. """ base, cur = _recorded_models(baseline_data), _recorded_models(eval_data) lines, unverified = [], [] @@ -130,23 +132,25 @@ def _model_mismatches(baseline_data: Dict[str, Any], b, c = base[role], cur[role] if not (b and c): unverified.append(role) - elif not _same_recorded(b, c): + continue + verdict = _recorded_verdict(b, c) + if verdict == "mismatch": lines.append(f"{role} model: baseline {b[0]}/{b[1]}, current {c[0]}/{c[1]}") + elif verdict == "unverified": + unverified.append(role) return lines, unverified -def _same_recorded(b: Tuple[str, str], c: Tuple[str, str]) -> bool: - """Whether two recorded (provider, model) pairs name one model. A panel is the same when both - have the same number of judges and each judge of one pairs with a different judge of the other.""" - if (b[0] == "panel") != (c[0] == "panel"): - return False - if b[0] != "panel": - return same_model(b[1], c[1]) - xs, ys = b[1].split(","), c[1].split(",") - if len(xs) != len(ys): - return False - # same_model is not transitive (a snapshot matches its alias, two snapshots do not match each other), - # so pair the judges by augmenting paths rather than greedily +def _same_identity(x: str, y: str) -> bool: + """Whether two model names have the same id, snapshot date and route-free identity (``model_identity``).""" + ix = model_identity(x) + return bool(ix[0]) and ix == model_identity(y) + + +def _pairs_up(xs: List[str], ys: List[str]) -> bool: + """Whether every name in ``xs`` can be paired with a different name in ``ys`` that ``same_model`` matches. + ``same_model`` is not transitive (a snapshot matches its alias, two snapshots do not match each other), + so the pairing uses augmenting paths rather than greedy first-match.""" partner: dict[int, int] = {} # index in ys -> index in xs def place(i: int, tried: set[int]) -> bool: @@ -162,6 +166,28 @@ def place(i: int, tried: set[int]) -> bool: return all(place(i, set()) for i in range(len(xs))) +def _recorded_verdict(b: Tuple[str, str], c: Tuple[str, str]) -> str: + """How two recorded (provider, model) pairs compare: ``"match"``, ``"unverified"`` or ``"mismatch"``. + + ``match``: the same id and snapshot (a panel: the same number of judges, and the same multiset of + identities). ``unverified``: ``same_model`` links them, but they are not exactly equal: a dated snapshot + against its undated alias, which does not record which snapshot answered, so ``--strict`` cannot + verify it. Anything else is a ``mismatch``. + """ + if (b[0] == "panel") != (c[0] == "panel"): + return "mismatch" + if b[0] != "panel": + if _same_identity(b[1], c[1]): + return "match" + return "unverified" if same_model(b[1], c[1]) else "mismatch" + xs, ys = b[1].split(","), c[1].split(",") + if len(xs) != len(ys): + return "mismatch" + if Counter(model_identity(x) for x in xs) == Counter(model_identity(y) for y in ys): + return "match" + return "unverified" if _pairs_up(xs, ys) else "mismatch" + + def cmd_baseline_save(args: argparse.Namespace) -> None: """Save an eval result as a named baseline.""" from datetime import datetime, timezone @@ -246,9 +272,14 @@ def _pick_entry(args: argparse.Namespace, baselines: list, eval_data: dict) -> d def _report_model_checks(args: argparse.Namespace, entry: dict, baseline_data: dict, eval_data: dict) -> None: mismatches, unverified = _model_mismatches(baseline_data, eval_data) strict = getattr(args, "strict", False) + base_rec, cur_rec = _recorded_models(baseline_data), _recorded_models(eval_data) for role in unverified: - sides = [name for name, data in (("baseline", baseline_data), ("current eval", eval_data)) - if _recorded_models(data)[role] is None] + b, c = base_rec[role], cur_rec[role] + if b and c: + print(f"⚠ {role} model not verified: baseline {b[0]}/{b[1]}, current {c[0]}/{c[1]} " + f"(an undated alias does not say which snapshot answered)", file=sys.stderr) + continue + sides = [name for name, rec in (("baseline", base_rec), ("current eval", cur_rec)) if rec[role] is None] print(f"⚠ {role} model not recorded in {' and '.join(sides)}; not compared", file=sys.stderr) if unverified and strict: print(f"❌ --strict cannot verify the {' and '.join(unverified)} model against " diff --git a/dokimi/dokimi/utils.py b/dokimi/dokimi/utils.py index 485d1503a..abd853ff7 100644 --- a/dokimi/dokimi/utils.py +++ b/dokimi/dokimi/utils.py @@ -156,6 +156,17 @@ def _model_core(model: str) -> tuple[str, str | None]: return (name[:snapshot.start()], snapshot.group(1).replace("-", "")) if snapshot else (name, None) +def model_identity(model: str) -> tuple[str | None, str | None]: + """``(id, snapshot date or None)`` of a model name with its route, region profile and vendor prefix dropped; + the id is ``None`` for a name with no id. Two names with equal identities are one model and one snapshot.""" + base, date = _model_core(model) + if not base: + return None, date + # a lowercase word before the first dot is taken as a vendor prefix (``anthropic.``, ``amazon.``) and + # dropped; a dot after a digit or a dash (``gpt-4.1``, ``qwen2.5``) is part of the name + return re.sub(r"^[a-z]+\.", "", base), date + + def same_model(a: str, b: str) -> bool: """Whether two model names are one model. @@ -166,12 +177,10 @@ def same_model(a: str, b: str) -> bool: The provider is not part of the identity: the same weights on Bedrock and on the direct API are still one model for the self-judge warning and for a baseline's model check. """ - (base_a, date_a), (base_b, date_b) = _model_core(a), _model_core(b) - if not (base_a and base_b) or (date_a and date_b and date_a != date_b): + (base_a, date_a), (base_b, date_b) = model_identity(a), model_identity(b) + if base_a is None or base_b is None or (date_a and date_b and date_a != date_b): return False - # a lowercase word before the first dot is taken as a vendor prefix (``anthropic.``, ``amazon.``) and - # dropped; a dot after a digit or a dash (``gpt-4.1``, ``qwen2.5``) is part of the name - return re.sub(r"^[a-z]+\.", "", base_a) == re.sub(r"^[a-z]+\.", "", base_b) + return base_a == base_b def _read_key(value: str) -> str: diff --git a/dokimi/tests/test_baseline_compare.py b/dokimi/tests/test_baseline_compare.py index cd06041a7..4eecba713 100644 --- a/dokimi/tests/test_baseline_compare.py +++ b/dokimi/tests/test_baseline_compare.py @@ -128,37 +128,49 @@ def test_compare_treats_a_bedrock_alias_and_its_profile_as_one_judge(compare, ca assert capsys.readouterr().err == "" -def test_a_snapshot_its_alias_and_another_route_are_one_judge_model(): - base = _eval(judge=("anthropic", "claude-sonnet-4-5")) - assert baseline_mod._model_mismatches(base, _eval(judge=("anthropic", "claude-sonnet-4-5-20250929"))) == ([], []) - assert baseline_mod._model_mismatches(base, _eval(judge=("amazon-bedrock", "us.anthropic.claude-sonnet-4-5"))) == ( +def test_one_judge_id_and_snapshot_behind_another_route_is_one_judge_model(): + dated = _eval(judge=("anthropic", "claude-sonnet-4-5-20250929")) + assert baseline_mod._model_mismatches(dated, _eval(judge=("amazon-bedrock", "us.anthropic.claude-sonnet-4-5-20250929"))) == ( [], []) + alias = _eval(judge=("anthropic", "claude-sonnet-4-5")) + assert baseline_mod._model_mismatches(alias, _eval(judge=("amazon-bedrock", "us.anthropic.claude-sonnet-4-5"))) == ( + [], []) + + +def test_a_dated_snapshot_and_its_undated_alias_are_not_verified_in_either_direction(): + """The alias does not record which snapshot answered: a role that was not measured, so it is named, not matched.""" + dated = _eval(judge=("anthropic", "claude-sonnet-4-5-20250929")) + alias = _eval(judge=("anthropic", "claude-sonnet-4-5")) + assert baseline_mod._model_mismatches(dated, alias) == ([], ["judge"]) + assert baseline_mod._model_mismatches(alias, dated) == ([], ["judge"]) def test_a_panel_is_the_same_only_when_its_judges_pair_one_to_one(): - same = baseline_mod._same_recorded - assert same(("panel", "anthropic/m,anthropic/m"), ("panel", "m,anthropic/m")) - assert not same(("panel", "anthropic/m,anthropic/m,anthropic/m"), ("panel", "anthropic/m,anthropic/m")) - assert not same(("panel", "m-20240101,m-20240101"), ("panel", "m,m-20250101")) - assert not same(("panel", "m"), ("anthropic", "m")) + verdict = baseline_mod._recorded_verdict + assert verdict(("panel", "anthropic/m,anthropic/m"), ("panel", "m,anthropic/m")) == "match" + assert verdict(("panel", "anthropic/m,anthropic/m,anthropic/m"), ("panel", "anthropic/m,anthropic/m")) == "mismatch" + assert verdict(("panel", "m-20240101,m-20240101"), ("panel", "m,m-20250101")) == "mismatch" + assert verdict(("panel", "m"), ("anthropic", "m")) == "mismatch" def test_a_panel_pairs_its_judges_by_augmenting_paths_not_first_match(): """``m`` matches both snapshots and ``m-20240101`` only its own: a first-match pairing gives ``m`` the - judge ``m-20240101`` and strands the second baseline judge, though a valid pairing exists.""" - same = baseline_mod._same_recorded - assert same(("panel", "m,m-20240101"), ("panel", "m-20240101,m-20250101")) - assert same(("panel", "m-20240101,m-20250101"), ("panel", "m,m-20240101")) # the reverse side and order - assert same(("panel", "m-20240101,m"), ("panel", "m-20240101,m-20250101")) - assert not same(("panel", "m-20240101,m-20240101"), ("panel", "m,m-20250101")) + judge ``m-20240101`` and strands the second baseline judge, though a valid pairing exists. The pairing + alone is only "not verified" (a dated judge against an undated one), never a match.""" + verdict = baseline_mod._recorded_verdict + assert verdict(("panel", "m,m-20240101"), ("panel", "m-20240101,m-20250101")) == "unverified" + assert verdict(("panel", "m-20240101,m-20250101"), ("panel", "m,m-20240101")) == "unverified" # the reverse side + assert verdict(("panel", "m-20240101,m"), ("panel", "m-20240101,m-20250101")) == "unverified" + assert verdict(("panel", "m-20240101,m-20240101"), ("panel", "m,m-20250101")) == "mismatch" + assert verdict(("panel", "m,m-20240101"), ("panel", "m-20240101,m")) == "match" # the same judges, reordered def test_a_panel_that_gained_a_judge_is_a_mismatch(): """Every judge of the baseline panel can pair with one of the current, and a third one is added: pairing alone would pass it, so the sizes are compared first.""" - same = baseline_mod._same_recorded - assert not same(("panel", "m,m-20240101"), ("panel", "m,m-20240101,m")) - assert not same(("panel", "m,m-20240101,m"), ("panel", "m,m-20240101")) + verdict = baseline_mod._recorded_verdict + assert verdict(("panel", "m,m-20240101"), ("panel", "m,m-20240101,m")) == "mismatch" + assert verdict(("panel", "m,m-20240101,m"), ("panel", "m,m-20240101")) == "mismatch" def test_two_snapshots_of_one_judge_model_are_a_mismatch(): @@ -169,6 +181,28 @@ def test_two_snapshots_of_one_judge_model_are_a_mismatch(): "current anthropic/claude-sonnet-4-5-20250929"] +@pytest.mark.parametrize("base_judge, cur_judge", [ + ("claude-sonnet-4-5-20240620", "claude-sonnet-4-5"), # the baseline dated, the current an undated alias + ("claude-sonnet-4-5", "claude-sonnet-4-5-20240620"), # and the other way round +]) +def test_compare_strict_refuses_a_dated_judge_against_its_undated_alias(compare, capsys, base_judge, cur_judge): + """The alias does not say which snapshot answered, so --strict cannot verify that the judge did not change.""" + with pytest.raises(SystemExit) as exc: + compare(_eval(judge=("anthropic", base_judge)), _eval(judge=("anthropic", cur_judge)), "--strict") + assert exc.value.code == 1 + out, err = capsys.readouterr() + assert "--strict cannot verify the judge model" in err + assert "Avg delta" not in out + + +def test_compare_warns_that_a_dated_judge_against_its_undated_alias_is_not_verified(compare, capsys): + compare(_eval(judge=("anthropic", "claude-sonnet-4-5-20240620")), _eval(judge=("anthropic", "claude-sonnet-4-5"))) + out, err = capsys.readouterr() + assert "judge model" in err and "not verified" in err + assert "Baseline model warning" not in err and "judge model: baseline" not in err + assert "Avg delta" in out + + def test_compare_names_a_role_the_baseline_does_not_record(compare, capsys): old = _eval() del old["run_metadata"] From c95c3e551d43e5e8df353adeb49724f29c162083 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 19:37:20 -0700 Subject: [PATCH 13/18] fix(utils): a model name with no id after its route is no model model_identity returned an empty id for a name such as "anthropic." or "us.anthropic.", so same_model paired two such names as one model. Return None for an empty id, as its docstring says. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/utils.py | 2 +- dokimi/tests/test_same_model.py | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/utils.py b/dokimi/dokimi/utils.py index abd853ff7..b36d6cf84 100644 --- a/dokimi/dokimi/utils.py +++ b/dokimi/dokimi/utils.py @@ -164,7 +164,7 @@ def model_identity(model: str) -> tuple[str | None, str | None]: return None, date # a lowercase word before the first dot is taken as a vendor prefix (``anthropic.``, ``amazon.``) and # dropped; a dot after a digit or a dash (``gpt-4.1``, ``qwen2.5``) is part of the name - return re.sub(r"^[a-z]+\.", "", base), date + return (re.sub(r"^[a-z]+\.", "", base) or None), date def same_model(a: str, b: str) -> bool: diff --git a/dokimi/tests/test_same_model.py b/dokimi/tests/test_same_model.py index 1e4997205..cbeb75c43 100644 --- a/dokimi/tests/test_same_model.py +++ b/dokimi/tests/test_same_model.py @@ -8,7 +8,7 @@ import pytest from dokimi.defaults import BEDROCK_MODEL_IDS -from dokimi.utils import same_model +from dokimi.utils import model_identity, same_model # (a, b, same) with a route-qualified name ("provider/model") where the route is part of the input. CASES = [ @@ -82,6 +82,17 @@ def test_a_bedrock_alias_and_its_profile_are_one_model(bare, profile): assert same_model(f"amazon-bedrock/{bare}", f"amazon-bedrock/{profile}") +@pytest.mark.parametrize("a, b", [ + ("anthropic.", "openai."), # a vendor prefix with no id after it + ("anthropic.", "anthropic."), + ("us.anthropic.", "us.anthropic."), + ("openai/", "openai/"), # a route with no id after it +]) +def test_a_name_with_no_id_is_no_model(a, b): + assert same_model(a, b) is False + assert model_identity(a)[0] is None + + def test_every_provider_dokimi_accepts_is_a_route_not_an_organization(): from dokimi.llm import PROVIDER_ALIASES, PROVIDERS From e058b14b285f2f490e2446b875c8dfe0364a34ab Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 19:40:00 -0700 Subject: [PATCH 14/18] fix(utils): separators or digits alone name no model The no-id fix covered only names that empty out after the route. A name of separators ("/", ".", "eu./", "-") or a revision ("1:0") still gave an id, so same_model paired them. An id needs a letter; digit-led ids with a letter (01-ai/Yi-34B) are unchanged. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/utils.py | 3 ++- dokimi/tests/test_same_model.py | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/utils.py b/dokimi/dokimi/utils.py index b36d6cf84..df56cb484 100644 --- a/dokimi/dokimi/utils.py +++ b/dokimi/dokimi/utils.py @@ -164,7 +164,8 @@ def model_identity(model: str) -> tuple[str | None, str | None]: return None, date # a lowercase word before the first dot is taken as a vendor prefix (``anthropic.``, ``amazon.``) and # dropped; a dot after a digit or a dash (``gpt-4.1``, ``qwen2.5``) is part of the name - return (re.sub(r"^[a-z]+\.", "", base) or None), date + ident = re.sub(r"^[a-z]+\.", "", base) + return (ident if re.search(r"[a-z]", ident) else None), date # separators or digits alone name no model def same_model(a: str, b: str) -> bool: diff --git a/dokimi/tests/test_same_model.py b/dokimi/tests/test_same_model.py index cbeb75c43..377705748 100644 --- a/dokimi/tests/test_same_model.py +++ b/dokimi/tests/test_same_model.py @@ -87,6 +87,11 @@ def test_a_bedrock_alias_and_its_profile_are_one_model(bare, profile): ("anthropic.", "anthropic."), ("us.anthropic.", "us.anthropic."), ("openai/", "openai/"), # a route with no id after it + ("/", "/"), # separators only + (".", "."), + ("eu./", "/"), # a region profile with no id after it + ("-", "-"), + ("1:0", "1:0"), # a revision with no name before it ]) def test_a_name_with_no_id_is_no_model(a, b): assert same_model(a, b) is False From b8ca7ecbe403f077cfef05fcbe8aa2c3561fb3d9 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 19:43:03 -0700 Subject: [PATCH 15/18] fix(baseline): digit ids are ids; a panel judge with no id is a mismatch The previous commit made a digits-only name no model, so "8" against "8" turned into a mismatch and two panels of separator-only judges compared as the same panel. An id needs a letter or a digit; a panel whose judge has no id is a mismatch. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/baseline.py | 2 ++ dokimi/dokimi/utils.py | 2 +- dokimi/tests/test_baseline_compare.py | 10 ++++++++++ dokimi/tests/test_same_model.py | 1 - 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/commands/baseline.py b/dokimi/dokimi/commands/baseline.py index 2069c4a69..df67e1361 100644 --- a/dokimi/dokimi/commands/baseline.py +++ b/dokimi/dokimi/commands/baseline.py @@ -183,6 +183,8 @@ def _recorded_verdict(b: Tuple[str, str], c: Tuple[str, str]) -> str: xs, ys = b[1].split(","), c[1].split(",") if len(xs) != len(ys): return "mismatch" + if not all(model_identity(x)[0] for x in xs + ys): # a judge with no id is no model: no pair of them is the same + return "mismatch" if Counter(model_identity(x) for x in xs) == Counter(model_identity(y) for y in ys): return "match" return "unverified" if _pairs_up(xs, ys) else "mismatch" diff --git a/dokimi/dokimi/utils.py b/dokimi/dokimi/utils.py index df56cb484..fc8d69a8d 100644 --- a/dokimi/dokimi/utils.py +++ b/dokimi/dokimi/utils.py @@ -165,7 +165,7 @@ def model_identity(model: str) -> tuple[str | None, str | None]: # a lowercase word before the first dot is taken as a vendor prefix (``anthropic.``, ``amazon.``) and # dropped; a dot after a digit or a dash (``gpt-4.1``, ``qwen2.5``) is part of the name ident = re.sub(r"^[a-z]+\.", "", base) - return (ident if re.search(r"[a-z]", ident) else None), date # separators or digits alone name no model + return (ident if re.search(r"[a-z0-9]", ident) else None), date # separators alone name no model def same_model(a: str, b: str) -> bool: diff --git a/dokimi/tests/test_baseline_compare.py b/dokimi/tests/test_baseline_compare.py index 4eecba713..de7df153e 100644 --- a/dokimi/tests/test_baseline_compare.py +++ b/dokimi/tests/test_baseline_compare.py @@ -165,6 +165,16 @@ def test_a_panel_pairs_its_judges_by_augmenting_paths_not_first_match(): assert verdict(("panel", "m,m-20240101"), ("panel", "m-20240101,m")) == "match" # the same judges, reordered +def test_a_panel_of_names_with_no_id_is_a_mismatch_and_digit_ids_still_match(): + """A judge name with no id (separators only) is no model, so two such panels are not the same panel; + a digit-only id is an id, as it was before.""" + verdict = baseline_mod._recorded_verdict + assert verdict(("panel", "/,/"), ("panel", "-,-")) == "mismatch" + assert verdict(("panel", "/,/"), ("panel", "/,/")) == "mismatch" + assert verdict(("anthropic", "8"), ("anthropic", "8")) == "match" + assert verdict(("panel", "anthropic/8,anthropic/9"), ("panel", "openai/9,openai/8")) == "match" + + def test_a_panel_that_gained_a_judge_is_a_mismatch(): """Every judge of the baseline panel can pair with one of the current, and a third one is added: pairing alone would pass it, so the sizes are compared first.""" diff --git a/dokimi/tests/test_same_model.py b/dokimi/tests/test_same_model.py index 377705748..808f3668d 100644 --- a/dokimi/tests/test_same_model.py +++ b/dokimi/tests/test_same_model.py @@ -91,7 +91,6 @@ def test_a_bedrock_alias_and_its_profile_are_one_model(bare, profile): (".", "."), ("eu./", "/"), # a region profile with no id after it ("-", "-"), - ("1:0", "1:0"), # a revision with no name before it ]) def test_a_name_with_no_id_is_no_model(a, b): assert same_model(a, b) is False From dee8fcc12ae4277fc6e7f83454be4ee1bc3ccd19 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 19:48:05 -0700 Subject: [PATCH 16/18] test(baseline): say which digit-only ids were an id before this round Signed-off-by: Nikolay Petrov --- dokimi/tests/test_baseline_compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/tests/test_baseline_compare.py b/dokimi/tests/test_baseline_compare.py index de7df153e..16500ca0d 100644 --- a/dokimi/tests/test_baseline_compare.py +++ b/dokimi/tests/test_baseline_compare.py @@ -167,7 +167,7 @@ def test_a_panel_pairs_its_judges_by_augmenting_paths_not_first_match(): def test_a_panel_of_names_with_no_id_is_a_mismatch_and_digit_ids_still_match(): """A judge name with no id (separators only) is no model, so two such panels are not the same panel; - a digit-only id is an id, as it was before.""" + a digit-only id is an id, as it was on this branch before the round's first fix.""" verdict = baseline_mod._recorded_verdict assert verdict(("panel", "/,/"), ("panel", "-,-")) == "mismatch" assert verdict(("panel", "/,/"), ("panel", "/,/")) == "mismatch" From 6a14fc1f427ece3905f2f8b2b3bb4e692a833400 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 19:49:34 -0700 Subject: [PATCH 17/18] docs(changelog): the org-prefix warning in repo compare and report is a behaviour change Signed-off-by: Nikolay Petrov --- .../20261028_130000_napetrov_one-model-match-rule.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dokimi/changelog.d/20261028_130000_napetrov_one-model-match-rule.md b/dokimi/changelog.d/20261028_130000_napetrov_one-model-match-rule.md index 2e7f30777..b88eee1f1 100644 --- a/dokimi/changelog.d/20261028_130000_napetrov_one-model-match-rule.md +++ b/dokimi/changelog.d/20261028_130000_napetrov_one-model-match-rule.md @@ -14,4 +14,5 @@ `dokimi run` stores `run_metadata.evaluator_independence_warning` by this rule too, as `dokimi eval` does (it was an exact provider-and-model match, so the printed warning and the stored flag disagreed). A route is a provider or gateway prefix; the organization of an `org/name` id is part of the id, so - `meta-llama/X` and `myorg/X` are two models. + `meta-llama/X` and `myorg/X` are two models. So a bare requested name answered by an org-prefixed id (`X` answered by `myorg/X`) + now gets the `repo compare` and `repo report` model-mismatch warning, which it did not before. From ce6da31cbd80fd8036c4c45cf145e0764ce8dd8c Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 19:53:13 -0700 Subject: [PATCH 18/18] docs(utils): keep the route-prefix comment to two lines Signed-off-by: Nikolay Petrov --- dokimi/dokimi/utils.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/dokimi/dokimi/utils.py b/dokimi/dokimi/utils.py index fc8d69a8d..3d803b043 100644 --- a/dokimi/dokimi/utils.py +++ b/dokimi/dokimi/utils.py @@ -119,10 +119,8 @@ def retry_writable(func: Any, path: str, _exc: Any) -> None: _REGION_PROFILES = ("us-gov.", "global.", "apac.", "us.", "eu.", "jp.", "au.", "ca.") -# Leading path segments that name how a model is reached, not which model it is: a provider or a gateway -# (``anthropic/``, ``openrouter/``, ``bedrock/converse/``, ``azure/``). Any other segment (a Hugging Face -# organization such as ``meta-llama/``) is part of the id. Every provider of ``llm.PROVIDERS`` and its aliases is -# one (pinned by a test); the rest are litellm route prefixes. +# Leading segments that name a route (provider or gateway: ``anthropic/``, ``openrouter/``), not a model; any other +# segment (an HF org such as ``meta-llama/``) is part of the id. Holds ``llm.PROVIDERS`` (test-pinned) + litellm routes. _ROUTES = frozenset({ "openai", "openai-codex", "anthropic", "google", "gemini", "google-vertex", "openrouter", "amazon-bedrock", "bedrock", "converse", "vertex_ai", "azure", "azure_ai", "models", "groq", "ollama", "mistral", "deepseek",