fix(model): one same-model rule for baseline, self-judge and repo compare

dokimi/changelog.d/20261028_130000_napetrov_one-model-match-rule.md

@@ -0,0 +1,15 @@
+### Fixed
+
+- **"Same model" means one thing in `baseline compare`, the self-judge warning and `repo compare`** — they
+  used two rules that disagreed. A dated id (`claude-sonnet-4-5-20250929`) and its undated alias, a Bedrock
+  regional profile (`eu.anthropic.…`) and its bare id, and the same model behind a route
+  (`anthropic/`, `openrouter/anthropic/`, `bedrock/`, `vertex_ai/`) or a `-v1:0` revision are now one model
+  everywhere; two different dated snapshots are still two models, and case no longer matters. Effects: the
+  self-judge warning fires when the answer and the judge are the same model on another provider too; `baseline
+  compare` no longer reports or refuses (`--strict`) a judge or answer that only changed its route, region or
+  snapshot alias; `repo compare` pairs arms by the same rule. Scores are not recomputed, but a comparison that
+  used to refuse or warn about such a pair may now pass.
+  `dokimi run` stores `run_metadata.evaluator_independence_warning` by this rule too, as `dokimi eval` does (it
+  was an exact provider-and-model match, so the printed warning and the stored flag disagreed).
+  A route is a provider or gateway prefix; the organization of an `org/name` id is part of the id, so
+  `meta-llama/X` and `myorg/X` are two models.

dokimi/docs/benchmarking-and-comparison.md

@@ -786,11 +786,14 @@ python cli.py baseline compare \
 ```
 
 When the answer or judge model recorded in the baseline differs from the
-current eval's (a Bedrock alias and its inference profile count as one model),
+current eval's (one model under `utils.same_model`: a Bedrock alias and its inference profile, a
+dated snapshot and its undated alias, a regional profile, a `-vN:M` revision or another route count
+as the same model; two different dated snapshots, or the same name under another organization, do not),
 `compare` prints a warning naming both before the table: the score changes then
 mix the model change with the content change. With `--strict` it exits 1
 instead. A baseline saved before a default-model change is such a baseline. A panel
-artifact (`eval panel-score`) is compared by its set of judge models. A role either
+artifact (`eval panel-score`) is compared by its judge models: the panels must have the same
+number of judges, and each judge must pair with a different judge of the other panel. A role either
 side does not record (an older artifact, or answers with no model) is named on
 stderr as not compared, and `--strict` exits 1 on it as well, since it cannot
 vouch for that role. A malformed record in either file (no `question_id`) exits 2,

dokimi/dokimi/commands/baseline.py

@@ -11,7 +11,7 @@
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple
 
-from dokimi.utils import normalize_model_ref
+from dokimi.utils import same_model
 
 BASELINES_DIR = Path("baselines/eval")
 
@@ -88,8 +88,9 @@ def _recorded_models(eval_data: Dict[str, Any]) -> Dict[str, Optional[Tuple[str,
 
     The judge is top-level (``judge_provider``/``judge_model``); the answerer is in
     ``run_metadata``. A panel (``judge_model: "panel(N)"``) records its judges only in
-    ``panel_config.models``; it maps to ``("panel", <sorted, normalized models>)``. A role
-    the artifact does not record maps to ``None``.
+    ``panel_config.models``; it maps to ``("panel", <the judges' raw "provider/model" strings,
+    sorted and comma-joined>)``, which ``_same_recorded`` splits and compares. A role the
+    artifact does not record maps to ``None``.
     """
     meta = eval_data.get("run_metadata")
     if not isinstance(meta, dict):
@@ -106,8 +107,7 @@ def pair(provider: Any, model: Any) -> Optional[Tuple[str, str]]:
         models = panel.get("models") if isinstance(panel, dict) else None
         if not (isinstance(models, list) and all(isinstance(m, str) for m in models)):
             models = []
-        judge = ("panel", ",".join(sorted(
-            normalize_model_ref(*m.split("/", 1)) if "/" in m else m for m in models))) if models else None
+        judge = ("panel", ",".join(sorted(models))) if models else None
     else:
         judge = pair(eval_data.get("judge_provider") or meta.get("judge_provider"), judge_model)
     return {
@@ -121,19 +121,47 @@ def _model_mismatches(baseline_data: Dict[str, Any],
     """Per role (answer, judge): lines for a recorded model that differs between the two
     runs, and the roles that could not be compared because a side does not record them.
 
-    A Bedrock alias and its inference profile count as one model.
+    Two model names that ``same_model`` matches (a Bedrock alias and its profile, a snapshot and its
+    alias, one model behind another route) count as one model.
     """
     base, cur = _recorded_models(baseline_data), _recorded_models(eval_data)
     lines, unverified = [], []
     for role in ("answer", "judge"):
         b, c = base[role], cur[role]
         if not (b and c):
             unverified.append(role)
-        elif normalize_model_ref(*b) != normalize_model_ref(*c):
+        elif not _same_recorded(b, c):
             lines.append(f"{role} model: baseline {b[0]}/{b[1]}, current {c[0]}/{c[1]}")
     return lines, unverified
 
 
+def _same_recorded(b: Tuple[str, str], c: Tuple[str, str]) -> bool:
+    """Whether two recorded (provider, model) pairs name one model. A panel is the same when both
+    have the same number of judges and each judge of one pairs with a different judge of the other."""
+    if (b[0] == "panel") != (c[0] == "panel"):
+        return False
+    if b[0] != "panel":
+        return same_model(b[1], c[1])
+    xs, ys = b[1].split(","), c[1].split(",")
+    if len(xs) != len(ys):
+        return False
+    # same_model is not transitive (a snapshot matches its alias, two snapshots do not match each other),
+    # so pair the judges by augmenting paths rather than greedily
+    partner: dict[int, int] = {}  # index in ys -> index in xs
+
+    def place(i: int, tried: set[int]) -> bool:
+        for j, y in enumerate(ys):
+            if j in tried or not same_model(xs[i], y):
+                continue
+            tried.add(j)
+            if j not in partner or place(partner[j], tried):
+                partner[j] = i
+                return True
+        return False
+
+    return all(place(i, set()) for i in range(len(xs)))
+
+
 def cmd_baseline_save(args: argparse.Namespace) -> None:
     """Save an eval result as a named baseline."""
     from datetime import datetime, timezone

dokimi/dokimi/commands/evaluate.py

@@ -20,7 +20,7 @@
 )
 from dokimi.eval.judge import JUDGE_PROVIDERS
 from dokimi.eval.judge_validity import DEFAULT_MIN_ACCURACY, NoUsablePairs
-from dokimi.utils import normalize_model_ref, write_text_or_exit
+from dokimi.utils import same_model, write_text_or_exit
 
 
 def _warn_judge_independence(
@@ -30,8 +30,8 @@ def _warn_judge_independence(
     judge_model: str,
     context: str = "run",
 ) -> bool:
-    """Emit a non-fatal warning when answer and judge use the same model/provider."""
-    same = normalize_model_ref(answer_provider, answer_model) == normalize_model_ref(judge_provider, judge_model)
+    """Emit a non-fatal warning when answer and judge are the same model (on any provider route)."""
+    same = same_model(answer_model, judge_model)
     if same:
         print(
             "⚠ Evaluator independence warning "

dokimi/dokimi/orchestrator/pipeline.py

@@ -12,6 +12,7 @@
 from dokimi.defaults import DEFAULT_ANSWER_MODEL, DEFAULT_ANSWER_PROVIDER, DEFAULT_JUDGE_MODEL, DEFAULT_JUDGE_PROVIDER, DEFAULT_RETRIEVAL_TOP_K
 from dokimi.eval.stats import paired_summary
 from dokimi.mcp.factory import create_doc_source_client
+from dokimi.utils import same_model
 
 logger = logging.getLogger(__name__)
 
@@ -669,10 +670,7 @@ def _evaluate_answers(self, answers: List[Dict[str, Any]] | Dict[str, Any],
                 "judge_model": self.judge_model,
                 "judge_provider": self.judge_provider,
                 "question_set_hash": question_set_hash,
-                "evaluator_independence_warning": (
-                    str(self.model).strip().lower() == str(self.judge_model).strip().lower()
-                    and str(self.provider).strip().lower() == str(self.judge_provider).strip().lower()
-                ),
+                "evaluator_independence_warning": same_model(str(self.model), str(self.judge_model)),
             },
         )
 

dokimi/dokimi/repos/compare.py

@@ -84,9 +84,10 @@
 
 from dokimi.context.consistency import Run, order_distance, pass_hat, run_from_events
 from dokimi.eval.stats import adjust_pvalues, bootstrap_ci, mean_ci
+from dokimi.utils import same_model
 
 from .process_counts import as_measure, process_order, process_values
-from .report import (load_cells, mismatch_warnings, model_matches, model_mismatch, reading_cells, resolved_models,
+from .report import (load_cells, mismatch_warnings, model_mismatch, reading_cells, resolved_models,
                      unmeasured_note, uptake_counts)
 
 #: Largest number of non-zero deltas whose sign-flip distribution is enumerated (2**16 patterns).
@@ -216,23 +217,24 @@ def _provenance(batch_dirs: list[Path], cells: list[dict[str, Any]], baseline: s
 
 
 def _models_equivalent(base_models: set[str], treat_models: set[str]) -> bool:
-    """Whether base and treatment model sets are equivalent under model_matches() logic.
+    """Whether base and treatment model sets are equivalent under same_model() logic.
 
-    Two resolved model sets are equivalent if they differ only by provider prefixes or
-    version suffixes that model_matches() considers equivalent. For each base model,
+    Two resolved model sets are equivalent if each model in one matches a model in the other
+    under same_model(): provider prefixes, regional profiles, revision suffixes, case and
+    undated snapshot aliases do not count as different models. For each base model,
     there must be a matching treatment model and the reverse; the set sizes may differ, since one arm can
     hold several spellings of the same model across repetitions.
     """
     if base_models == treat_models:
         return True
     # Check if each base model has an equivalent in treatment models
-    # (model_matches is already bidirectional, so one call per pair suffices)
+    # (same_model is symmetric, so one call per pair suffices)
     for base_model in base_models:
-        if not any(model_matches(base_model, treat_model) for treat_model in treat_models):
+        if not any(same_model(base_model, treat_model) for treat_model in treat_models):
             return False
     # Check the reverse: every treatment model matches some base model
     for treat_model in treat_models:
-        if not any(model_matches(treat_model, base_model) for base_model in base_models):
+        if not any(same_model(treat_model, base_model) for base_model in base_models):
             return False
     return True
 

dokimi/dokimi/repos/report.py

@@ -36,7 +36,6 @@
 from __future__ import annotations
 
 import json
-import re
 from collections import defaultdict
 from collections.abc import Iterator
 from contextlib import contextmanager
@@ -45,6 +44,8 @@
 
 import math
 
+from dokimi.utils import same_model
+
 from .grading import SCORE_BOUND
 from .process_counts import process_line, process_means
 from .runner import UNSUPPORTED_FILE, slug
@@ -331,47 +332,11 @@ def resolved_models(rec: dict[str, Any]) -> list[str]:
     return [found] if isinstance(found, str) else [str(m) for m in found] if isinstance(found, list) else []
 
 
-def _normalize_model_id(model_id: str) -> str:
-    """Strip Bedrock's revision suffix: any ``-vN:M`` (``-v1:0``, ``-v2:0``) counts as the same model.
-    A bare ``-v2`` is part of the id: deepseek-v2 is not v3."""
-    return re.sub(r"-v\d+:\d+$", "", model_id)
-
-
-# Bedrock cross-region inference profile prefixes: the same model served from another region.
-_REGION_PREFIXES = ("us-gov.", "global.", "apac.", "us.", "eu.", "jp.", "au.", "ca.")
-
-
-def _strip_region(model_id: str) -> str:
-    for prefix in _REGION_PREFIXES:
-        if model_id.startswith(prefix):
-            return model_id[len(prefix):]
-    return model_id
-
-
-def _model_prefix_matches(base: str, candidate: str) -> bool:
-    """Check if candidate is base optionally preceded by provider prefix (., /)."""
-    # base could be the suffix of candidate, with optional prefix
-    if base == candidate:
-        return True
-    # Check if candidate ends with . + base or / + base
-    return candidate.endswith(("." + base, "/" + base))
-
-
-def model_matches(requested: str, resolved: str) -> bool:
-    """Whether ``resolved`` is the ``requested`` model: the same id, that id pinned to a snapshot
-    date, or either behind a provider prefix (Bedrock's ``us.anthropic.claude-x`` answers as
-    ``claude-x``, with any ``-vN:M`` revision too)."""
-    # Vertex's @20250929 is the same snapshot as -20250929; an undated side matches any snapshot of it
-    req, res = (re.sub(r"@(?=\d{8}$)", "-", _strip_region(_normalize_model_id(m))) for m in (requested, resolved))
-    pairs = [(req, res), (re.sub(r"-\d{8}$", "", req), res), (req, re.sub(r"-\d{8}$", "", res))]
-    return any(_model_prefix_matches(a, b) or _model_prefix_matches(b, a) for a, b in pairs)
-
-
 def model_mismatch(cells: list[dict[str, Any]]) -> dict[str, list[str]]:
     """Requested model -> the other models that answered its cells."""
     out: dict[str, set[str]] = defaultdict(set)
     for rec in cells:
-        out[rec["model"]] |= {m for m in resolved_models(rec) if not model_matches(rec["model"], m)}
+        out[rec["model"]] |= {m for m in resolved_models(rec) if not same_model(rec["model"], m)}
     return {k: sorted(v) for k, v in sorted(out.items()) if v}
 
 

dokimi/dokimi/utils.py

@@ -3,10 +3,11 @@
 
 import json
 import os
+import re
 from pathlib import Path
 from typing import Tuple, Dict, Any
 
-from dokimi.defaults import BEDROCK_MODEL_IDS, bedrock_profile
+from dokimi.defaults import bedrock_profile
 
 
 def parse_frontmatter(text: str) -> "Tuple[Dict[str, Any], str]":
@@ -113,15 +114,64 @@ def retry_writable(func: Any, path: str, _exc: Any) -> None:
         pass
 
 
-def normalize_model_ref(provider: str, model: str) -> str:
-    """Normalize provider and model names for comparison.
+# Bedrock cross-region inference profiles: the same model served from another region. They are dropped
+# from any provider's name, not only Bedrock's: no non-Bedrock id is known to start with one of them.
+_REGION_PROFILES = ("us-gov.", "global.", "apac.", "us.", "eu.", "jp.", "au.", "ca.")
 
-    On Bedrock a bare known name and its inference profile are the same model.
+
+# Leading path segments that name how a model is reached, not which model it is: a provider or a gateway
+# (``anthropic/``, ``openrouter/``, ``bedrock/converse/``, ``azure/``). Any other segment (a Hugging Face
+# organization such as ``meta-llama/``) is part of the id. Every provider of ``llm.PROVIDERS`` and its aliases is
+# one (pinned by a test); the rest are litellm route prefixes.
+_ROUTES = frozenset({
+    "openai", "openai-codex", "anthropic", "google", "gemini", "google-vertex", "openrouter", "amazon-bedrock",
+    "bedrock", "converse", "vertex_ai", "azure", "azure_ai", "models", "groq", "ollama", "mistral", "deepseek",
+    "together_ai", "fireworks_ai", "xai", "cohere", "huggingface",
+})
+
+
+def _model_core(model: str) -> tuple[str, str | None]:
+    """A model name reduced to ``(id, snapshot date or None)``. A snapshot is ``-YYYYMMDD`` or OpenAI's ``-YYYY-MM-DD``.
+
+    The route goes (``openrouter/anthropic/``, ``bedrock/``, ``vertex_ai/``, an ARN's
+    ``inference-profile/``), and so do the case, the region profile (``eu.``) and a Bedrock
+    revision (``-v1:0`` or ``-1:0``). Vertex's ``@20250929`` is the snapshot ``-20250929``. The organization of an
+    ``org/name`` id stays, so a re-upload under another organization is another model.
+    """
+    name = (model or "").strip().lower()
+    for marker in ("inference-profile/", "foundation-model/"):  # the tail of a Bedrock ARN
+        if marker in name:
+            name = name.rsplit(marker, 1)[-1]
+    parts = name.split("/")
+    while len(parts) > 1 and parts[0] in _ROUTES:
+        parts.pop(0)
+    name = "/".join(parts)
+    name = re.sub(r"-v?\d+:\d+$", "", name)
+    for prefix in _REGION_PROFILES:
+        if name.startswith(prefix):
+            name = name[len(prefix):]
+            break
+    name = re.sub(r"@(?=\d{8}$)", "-", name)
+    snapshot = re.search(r"-(\d{8}|\d{4}-\d{2}-\d{2})$", name)
+    return (name[:snapshot.start()], snapshot.group(1).replace("-", "")) if snapshot else (name, None)
+
+
+def same_model(a: str, b: str) -> bool:
+    """Whether two model names are one model.
+
+    The same id matches, and so does a dated snapshot and its undated alias (the alias runs
+    some snapshot of it), or the same id behind another route or region profile. Two dated
+    ids match only on the same date, and a bare family name (``haiku``) is not a full id.
+
+    The provider is not part of the identity: the same weights on Bedrock and on the direct
+    API are still one model for the self-judge warning and for a baseline's model check.
     """
-    prov, name = (provider or "").strip().lower(), (model or "").strip()
-    if prov == "amazon-bedrock":
-        name = BEDROCK_MODEL_IDS.get(name, name)
-    return f"{prov}/{name.lower()}"
+    (base_a, date_a), (base_b, date_b) = _model_core(a), _model_core(b)
+    if not (base_a and base_b) or (date_a and date_b and date_a != date_b):
+        return False
+    # a lowercase word before the first dot is taken as a vendor prefix (``anthropic.``, ``amazon.``) and
+    # dropped; a dot after a digit or a dash (``gpt-4.1``, ``qwen2.5``) is part of the name
+    return re.sub(r"^[a-z]+\.", "", base_a) == re.sub(r"^[a-z]+\.", "", base_b)
 
 
 def _read_key(value: str) -> str:

dokimi/tests/test_baseline_compare.py

@@ -128,6 +128,47 @@ def test_compare_treats_a_bedrock_alias_and_its_profile_as_one_judge(compare, ca
     assert capsys.readouterr().err == ""
 
 
+def test_a_snapshot_its_alias_and_another_route_are_one_judge_model():
+    base = _eval(judge=("anthropic", "claude-sonnet-4-5"))
+    assert baseline_mod._model_mismatches(base, _eval(judge=("anthropic", "claude-sonnet-4-5-20250929"))) == ([], [])
+    assert baseline_mod._model_mismatches(base, _eval(judge=("amazon-bedrock", "us.anthropic.claude-sonnet-4-5"))) == (
+        [], [])
+
+
+def test_a_panel_is_the_same_only_when_its_judges_pair_one_to_one():
+    same = baseline_mod._same_recorded
+    assert same(("panel", "anthropic/m,anthropic/m"), ("panel", "m,anthropic/m"))
+    assert not same(("panel", "anthropic/m,anthropic/m,anthropic/m"), ("panel", "anthropic/m,anthropic/m"))
+    assert not same(("panel", "m-20240101,m-20240101"), ("panel", "m,m-20250101"))
+    assert not same(("panel", "m"), ("anthropic", "m"))
+
+
+def test_a_panel_pairs_its_judges_by_augmenting_paths_not_first_match():
+    """``m`` matches both snapshots and ``m-20240101`` only its own: a first-match pairing gives ``m`` the
+    judge ``m-20240101`` and strands the second baseline judge, though a valid pairing exists."""
+    same = baseline_mod._same_recorded
+    assert same(("panel", "m,m-20240101"), ("panel", "m-20240101,m-20250101"))
+    assert same(("panel", "m-20240101,m-20250101"), ("panel", "m,m-20240101"))  # the reverse side and order
+    assert same(("panel", "m-20240101,m"), ("panel", "m-20240101,m-20250101"))
+    assert not same(("panel", "m-20240101,m-20240101"), ("panel", "m,m-20250101"))
+
+
+def test_a_panel_that_gained_a_judge_is_a_mismatch():
+    """Every judge of the baseline panel can pair with one of the current, and a third one is added:
+    pairing alone would pass it, so the sizes are compared first."""
+    same = baseline_mod._same_recorded
+    assert not same(("panel", "m,m-20240101"), ("panel", "m,m-20240101,m"))
+    assert not same(("panel", "m,m-20240101,m"), ("panel", "m,m-20240101"))
+
+
+def test_two_snapshots_of_one_judge_model_are_a_mismatch():
+    lines, _ = baseline_mod._model_mismatches(
+        _eval(judge=("anthropic", "claude-sonnet-4-5-20240620")),
+        _eval(judge=("anthropic", "claude-sonnet-4-5-20250929")))
+    assert lines == ["judge model: baseline anthropic/claude-sonnet-4-5-20240620, "
+                     "current anthropic/claude-sonnet-4-5-20250929"]
+
+
 def test_compare_names_a_role_the_baseline_does_not_record(compare, capsys):
     old = _eval()
     del old["run_metadata"]