diff --git a/dokimi/changelog.d/20261028_130000_napetrov_one-model-match-rule.md b/dokimi/changelog.d/20261028_130000_napetrov_one-model-match-rule.md new file mode 100644 index 000000000..b88eee1f1 --- /dev/null +++ b/dokimi/changelog.d/20261028_130000_napetrov_one-model-match-rule.md @@ -0,0 +1,18 @@ +### Fixed + +- **"Same model" means one thing in `baseline compare`, the self-judge warning and `repo compare`** — they + used two rules that disagreed. A dated id (`claude-sonnet-4-5-20250929`) and its undated alias, a Bedrock + regional profile (`eu.anthropic.…`) and its bare id, and the same model behind a route + (`anthropic/`, `openrouter/anthropic/`, `bedrock/`, `vertex_ai/`) or a `-v1:0` revision are now one model + everywhere; two different dated snapshots are still two models, and case no longer matters. Effects: the + self-judge warning fires when the answer and the judge are the same model on another provider too; `baseline + compare` no longer reports or refuses (`--strict`) a judge or answer that only changed its route or region; + `repo compare` pairs arms by the same rule. A dated snapshot against its undated alias is the exception in + `baseline compare`: the alias does not record which snapshot answered, so it is named on stderr as not + verified and `--strict` refuses it, as before. Scores are not recomputed, but a comparison + that used to refuse or warn about such a pair may now pass. + `dokimi run` stores `run_metadata.evaluator_independence_warning` by this rule too, as `dokimi eval` does (it + was an exact provider-and-model match, so the printed warning and the stored flag disagreed). + A route is a provider or gateway prefix; the organization of an `org/name` id is part of the id, so + `meta-llama/X` and `myorg/X` are two models. So a bare requested name answered by an org-prefixed id (`X` answered by `myorg/X`) + now gets the `repo compare` and `repo report` model-mismatch warning, which it did not before. diff --git a/dokimi/docs/benchmarking-and-comparison.md b/dokimi/docs/benchmarking-and-comparison.md index 4d771b83e..e007a7ec0 100644 --- a/dokimi/docs/benchmarking-and-comparison.md +++ b/dokimi/docs/benchmarking-and-comparison.md @@ -786,11 +786,17 @@ python cli.py baseline compare \ ``` When the answer or judge model recorded in the baseline differs from the -current eval's (a Bedrock alias and its inference profile count as one model), -`compare` prints a warning naming both before the table: the score changes then +current eval's, `compare` prints a warning naming both before the table: the score changes then mix the model change with the content change. With `--strict` it exits 1 -instead. A baseline saved before a default-model change is such a baseline. A panel -artifact (`eval panel-score`) is compared by its set of judge models. A role either +instead. Two records name one model when they have the same id and snapshot: a Bedrock alias and its +inference profile, a regional profile, a `-vN:M` revision or the same model behind another route count +as the same model; two different dated snapshots, or the same name under another organization, are two models. +A dated snapshot and its undated alias are neither: the alias does not record which snapshot answered, so the +role is named on stderr as not verified, and `--strict` exits 1 on it. A baseline saved before a default-model +change is such a baseline. A panel +artifact (`eval panel-score`) is compared by its judge models: the panels must have the same number of judges +and the same models, and a judge that matches the other panel's only through such an alias makes the role +not verified. A role either side does not record (an older artifact, or answers with no model) is named on stderr as not compared, and `--strict` exits 1 on it as well, since it cannot vouch for that role. A malformed record in either file (no `question_id`) exits 2, diff --git a/dokimi/dokimi/commands/baseline.py b/dokimi/dokimi/commands/baseline.py index 20769c53d..df67e1361 100644 --- a/dokimi/dokimi/commands/baseline.py +++ b/dokimi/dokimi/commands/baseline.py @@ -8,10 +8,11 @@ import argparse import json import sys +from collections import Counter from pathlib import Path from typing import Any, Dict, List, Optional, Tuple -from dokimi.utils import normalize_model_ref +from dokimi.utils import model_identity, same_model BASELINES_DIR = Path("baselines/eval") @@ -88,8 +89,9 @@ def _recorded_models(eval_data: Dict[str, Any]) -> Dict[str, Optional[Tuple[str, The judge is top-level (``judge_provider``/``judge_model``); the answerer is in ``run_metadata``. A panel (``judge_model: "panel(N)"``) records its judges only in - ``panel_config.models``; it maps to ``("panel", )``. A role - the artifact does not record maps to ``None``. + ``panel_config.models``; it maps to ``("panel", )``, which ``_recorded_verdict`` splits and compares. A role the + artifact does not record maps to ``None``. """ meta = eval_data.get("run_metadata") if not isinstance(meta, dict): @@ -106,8 +108,7 @@ def pair(provider: Any, model: Any) -> Optional[Tuple[str, str]]: models = panel.get("models") if isinstance(panel, dict) else None if not (isinstance(models, list) and all(isinstance(m, str) for m in models)): models = [] - judge = ("panel", ",".join(sorted( - normalize_model_ref(*m.split("/", 1)) if "/" in m else m for m in models))) if models else None + judge = ("panel", ",".join(sorted(models))) if models else None else: judge = pair(eval_data.get("judge_provider") or meta.get("judge_provider"), judge_model) return { @@ -119,9 +120,11 @@ def pair(provider: Any, model: Any) -> Optional[Tuple[str, str]]: def _model_mismatches(baseline_data: Dict[str, Any], eval_data: Dict[str, Any]) -> Tuple[List[str], List[str]]: """Per role (answer, judge): lines for a recorded model that differs between the two - runs, and the roles that could not be compared because a side does not record them. + runs, and the roles that cannot be verified: a side does not record them, or they are a dated + snapshot against an undated alias (see ``_recorded_verdict``). - A Bedrock alias and its inference profile count as one model. + Two model names with the same id, snapshot and route (a Bedrock alias and its profile, one model + behind another route) count as one model. """ base, cur = _recorded_models(baseline_data), _recorded_models(eval_data) lines, unverified = [], [] @@ -129,11 +132,64 @@ def _model_mismatches(baseline_data: Dict[str, Any], b, c = base[role], cur[role] if not (b and c): unverified.append(role) - elif normalize_model_ref(*b) != normalize_model_ref(*c): + continue + verdict = _recorded_verdict(b, c) + if verdict == "mismatch": lines.append(f"{role} model: baseline {b[0]}/{b[1]}, current {c[0]}/{c[1]}") + elif verdict == "unverified": + unverified.append(role) return lines, unverified +def _same_identity(x: str, y: str) -> bool: + """Whether two model names have the same id, snapshot date and route-free identity (``model_identity``).""" + ix = model_identity(x) + return bool(ix[0]) and ix == model_identity(y) + + +def _pairs_up(xs: List[str], ys: List[str]) -> bool: + """Whether every name in ``xs`` can be paired with a different name in ``ys`` that ``same_model`` matches. + ``same_model`` is not transitive (a snapshot matches its alias, two snapshots do not match each other), + so the pairing uses augmenting paths rather than greedy first-match.""" + partner: dict[int, int] = {} # index in ys -> index in xs + + def place(i: int, tried: set[int]) -> bool: + for j, y in enumerate(ys): + if j in tried or not same_model(xs[i], y): + continue + tried.add(j) + if j not in partner or place(partner[j], tried): + partner[j] = i + return True + return False + + return all(place(i, set()) for i in range(len(xs))) + + +def _recorded_verdict(b: Tuple[str, str], c: Tuple[str, str]) -> str: + """How two recorded (provider, model) pairs compare: ``"match"``, ``"unverified"`` or ``"mismatch"``. + + ``match``: the same id and snapshot (a panel: the same number of judges, and the same multiset of + identities). ``unverified``: ``same_model`` links them, but they are not exactly equal: a dated snapshot + against its undated alias, which does not record which snapshot answered, so ``--strict`` cannot + verify it. Anything else is a ``mismatch``. + """ + if (b[0] == "panel") != (c[0] == "panel"): + return "mismatch" + if b[0] != "panel": + if _same_identity(b[1], c[1]): + return "match" + return "unverified" if same_model(b[1], c[1]) else "mismatch" + xs, ys = b[1].split(","), c[1].split(",") + if len(xs) != len(ys): + return "mismatch" + if not all(model_identity(x)[0] for x in xs + ys): # a judge with no id is no model: no pair of them is the same + return "mismatch" + if Counter(model_identity(x) for x in xs) == Counter(model_identity(y) for y in ys): + return "match" + return "unverified" if _pairs_up(xs, ys) else "mismatch" + + def cmd_baseline_save(args: argparse.Namespace) -> None: """Save an eval result as a named baseline.""" from datetime import datetime, timezone @@ -218,9 +274,14 @@ def _pick_entry(args: argparse.Namespace, baselines: list, eval_data: dict) -> d def _report_model_checks(args: argparse.Namespace, entry: dict, baseline_data: dict, eval_data: dict) -> None: mismatches, unverified = _model_mismatches(baseline_data, eval_data) strict = getattr(args, "strict", False) + base_rec, cur_rec = _recorded_models(baseline_data), _recorded_models(eval_data) for role in unverified: - sides = [name for name, data in (("baseline", baseline_data), ("current eval", eval_data)) - if _recorded_models(data)[role] is None] + b, c = base_rec[role], cur_rec[role] + if b and c: + print(f"⚠ {role} model not verified: baseline {b[0]}/{b[1]}, current {c[0]}/{c[1]} " + f"(an undated alias does not say which snapshot answered)", file=sys.stderr) + continue + sides = [name for name, rec in (("baseline", base_rec), ("current eval", cur_rec)) if rec[role] is None] print(f"⚠ {role} model not recorded in {' and '.join(sides)}; not compared", file=sys.stderr) if unverified and strict: print(f"❌ --strict cannot verify the {' and '.join(unverified)} model against " diff --git a/dokimi/dokimi/commands/evaluate.py b/dokimi/dokimi/commands/evaluate.py index 9d3602f31..c0cef62fd 100644 --- a/dokimi/dokimi/commands/evaluate.py +++ b/dokimi/dokimi/commands/evaluate.py @@ -20,7 +20,7 @@ ) from dokimi.eval.judge import JUDGE_PROVIDERS from dokimi.eval.judge_validity import DEFAULT_MIN_ACCURACY, NoUsablePairs -from dokimi.utils import normalize_model_ref, write_text_or_exit +from dokimi.utils import same_model, write_text_or_exit def _warn_judge_independence( @@ -30,8 +30,8 @@ def _warn_judge_independence( judge_model: str, context: str = "run", ) -> bool: - """Emit a non-fatal warning when answer and judge use the same model/provider.""" - same = normalize_model_ref(answer_provider, answer_model) == normalize_model_ref(judge_provider, judge_model) + """Emit a non-fatal warning when answer and judge are the same model (on any provider route).""" + same = same_model(answer_model, judge_model) if same: print( "⚠ Evaluator independence warning " diff --git a/dokimi/dokimi/orchestrator/pipeline.py b/dokimi/dokimi/orchestrator/pipeline.py index 99cd48029..a8f2dc4fe 100644 --- a/dokimi/dokimi/orchestrator/pipeline.py +++ b/dokimi/dokimi/orchestrator/pipeline.py @@ -12,6 +12,7 @@ from dokimi.defaults import DEFAULT_ANSWER_MODEL, DEFAULT_ANSWER_PROVIDER, DEFAULT_JUDGE_MODEL, DEFAULT_JUDGE_PROVIDER, DEFAULT_RETRIEVAL_TOP_K from dokimi.eval.stats import paired_summary from dokimi.mcp.factory import create_doc_source_client +from dokimi.utils import same_model logger = logging.getLogger(__name__) @@ -669,10 +670,7 @@ def _evaluate_answers(self, answers: List[Dict[str, Any]] | Dict[str, Any], "judge_model": self.judge_model, "judge_provider": self.judge_provider, "question_set_hash": question_set_hash, - "evaluator_independence_warning": ( - str(self.model).strip().lower() == str(self.judge_model).strip().lower() - and str(self.provider).strip().lower() == str(self.judge_provider).strip().lower() - ), + "evaluator_independence_warning": same_model(str(self.model), str(self.judge_model)), }, ) diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index 5307cf0be..7ca66fd78 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -84,9 +84,10 @@ from dokimi.context.consistency import Run, order_distance, pass_hat, run_from_events from dokimi.eval.stats import adjust_pvalues, bootstrap_ci, mean_ci +from dokimi.utils import same_model from .process_counts import as_measure, process_order, process_values -from .report import (load_cells, mismatch_warnings, model_matches, model_mismatch, reading_cells, resolved_models, +from .report import (load_cells, mismatch_warnings, model_mismatch, reading_cells, resolved_models, unmeasured_note, uptake_counts) #: Largest number of non-zero deltas whose sign-flip distribution is enumerated (2**16 patterns). @@ -216,23 +217,24 @@ def _provenance(batch_dirs: list[Path], cells: list[dict[str, Any]], baseline: s def _models_equivalent(base_models: set[str], treat_models: set[str]) -> bool: - """Whether base and treatment model sets are equivalent under model_matches() logic. + """Whether base and treatment model sets are equivalent under same_model() logic. - Two resolved model sets are equivalent if they differ only by provider prefixes or - version suffixes that model_matches() considers equivalent. For each base model, + Two resolved model sets are equivalent if each model in one matches a model in the other + under same_model(): provider prefixes, regional profiles, revision suffixes, case and + undated snapshot aliases do not count as different models. For each base model, there must be a matching treatment model and the reverse; the set sizes may differ, since one arm can hold several spellings of the same model across repetitions. """ if base_models == treat_models: return True # Check if each base model has an equivalent in treatment models - # (model_matches is already bidirectional, so one call per pair suffices) + # (same_model is symmetric, so one call per pair suffices) for base_model in base_models: - if not any(model_matches(base_model, treat_model) for treat_model in treat_models): + if not any(same_model(base_model, treat_model) for treat_model in treat_models): return False # Check the reverse: every treatment model matches some base model for treat_model in treat_models: - if not any(model_matches(treat_model, base_model) for base_model in base_models): + if not any(same_model(treat_model, base_model) for base_model in base_models): return False return True diff --git a/dokimi/dokimi/repos/report.py b/dokimi/dokimi/repos/report.py index 3244f035b..f9a9ebac4 100644 --- a/dokimi/dokimi/repos/report.py +++ b/dokimi/dokimi/repos/report.py @@ -36,7 +36,6 @@ from __future__ import annotations import json -import re from collections import defaultdict from collections.abc import Iterator from contextlib import contextmanager @@ -45,6 +44,8 @@ import math +from dokimi.utils import same_model + from .grading import SCORE_BOUND from .process_counts import process_line, process_means from .runner import UNSUPPORTED_FILE, slug @@ -331,47 +332,11 @@ def resolved_models(rec: dict[str, Any]) -> list[str]: return [found] if isinstance(found, str) else [str(m) for m in found] if isinstance(found, list) else [] -def _normalize_model_id(model_id: str) -> str: - """Strip Bedrock's revision suffix: any ``-vN:M`` (``-v1:0``, ``-v2:0``) counts as the same model. - A bare ``-v2`` is part of the id: deepseek-v2 is not v3.""" - return re.sub(r"-v\d+:\d+$", "", model_id) - - -# Bedrock cross-region inference profile prefixes: the same model served from another region. -_REGION_PREFIXES = ("us-gov.", "global.", "apac.", "us.", "eu.", "jp.", "au.", "ca.") - - -def _strip_region(model_id: str) -> str: - for prefix in _REGION_PREFIXES: - if model_id.startswith(prefix): - return model_id[len(prefix):] - return model_id - - -def _model_prefix_matches(base: str, candidate: str) -> bool: - """Check if candidate is base optionally preceded by provider prefix (., /).""" - # base could be the suffix of candidate, with optional prefix - if base == candidate: - return True - # Check if candidate ends with . + base or / + base - return candidate.endswith(("." + base, "/" + base)) - - -def model_matches(requested: str, resolved: str) -> bool: - """Whether ``resolved`` is the ``requested`` model: the same id, that id pinned to a snapshot - date, or either behind a provider prefix (Bedrock's ``us.anthropic.claude-x`` answers as - ``claude-x``, with any ``-vN:M`` revision too).""" - # Vertex's @20250929 is the same snapshot as -20250929; an undated side matches any snapshot of it - req, res = (re.sub(r"@(?=\d{8}$)", "-", _strip_region(_normalize_model_id(m))) for m in (requested, resolved)) - pairs = [(req, res), (re.sub(r"-\d{8}$", "", req), res), (req, re.sub(r"-\d{8}$", "", res))] - return any(_model_prefix_matches(a, b) or _model_prefix_matches(b, a) for a, b in pairs) - - def model_mismatch(cells: list[dict[str, Any]]) -> dict[str, list[str]]: """Requested model -> the other models that answered its cells.""" out: dict[str, set[str]] = defaultdict(set) for rec in cells: - out[rec["model"]] |= {m for m in resolved_models(rec) if not model_matches(rec["model"], m)} + out[rec["model"]] |= {m for m in resolved_models(rec) if not same_model(rec["model"], m)} return {k: sorted(v) for k, v in sorted(out.items()) if v} diff --git a/dokimi/dokimi/utils.py b/dokimi/dokimi/utils.py index cda2ff68c..3d803b043 100644 --- a/dokimi/dokimi/utils.py +++ b/dokimi/dokimi/utils.py @@ -3,10 +3,11 @@ import json import os +import re from pathlib import Path from typing import Tuple, Dict, Any -from dokimi.defaults import BEDROCK_MODEL_IDS, bedrock_profile +from dokimi.defaults import bedrock_profile def parse_frontmatter(text: str) -> "Tuple[Dict[str, Any], str]": @@ -113,15 +114,72 @@ def retry_writable(func: Any, path: str, _exc: Any) -> None: pass -def normalize_model_ref(provider: str, model: str) -> str: - """Normalize provider and model names for comparison. +# Bedrock cross-region inference profiles: the same model served from another region. They are dropped +# from any provider's name, not only Bedrock's: no non-Bedrock id is known to start with one of them. +_REGION_PROFILES = ("us-gov.", "global.", "apac.", "us.", "eu.", "jp.", "au.", "ca.") - On Bedrock a bare known name and its inference profile are the same model. + +# Leading segments that name a route (provider or gateway: ``anthropic/``, ``openrouter/``), not a model; any other +# segment (an HF org such as ``meta-llama/``) is part of the id. Holds ``llm.PROVIDERS`` (test-pinned) + litellm routes. +_ROUTES = frozenset({ + "openai", "openai-codex", "anthropic", "google", "gemini", "google-vertex", "openrouter", "amazon-bedrock", + "bedrock", "converse", "vertex_ai", "azure", "azure_ai", "models", "groq", "ollama", "mistral", "deepseek", + "together_ai", "fireworks_ai", "xai", "cohere", "huggingface", +}) + + +def _model_core(model: str) -> tuple[str, str | None]: + """A model name reduced to ``(id, snapshot date or None)``. A snapshot is ``-YYYYMMDD`` or OpenAI's ``-YYYY-MM-DD``. + + The route goes (``openrouter/anthropic/``, ``bedrock/``, ``vertex_ai/``, an ARN's + ``inference-profile/``), and so do the case, the region profile (``eu.``) and a Bedrock + revision (``-v1:0`` or ``-1:0``). Vertex's ``@20250929`` is the snapshot ``-20250929``. The organization of an + ``org/name`` id stays, so a re-upload under another organization is another model. + """ + name = (model or "").strip().lower() + for marker in ("inference-profile/", "foundation-model/"): # the tail of a Bedrock ARN + if marker in name: + name = name.rsplit(marker, 1)[-1] + parts = name.split("/") + while len(parts) > 1 and parts[0] in _ROUTES: + parts.pop(0) + name = "/".join(parts) + name = re.sub(r"-v?\d+:\d+$", "", name) + for prefix in _REGION_PROFILES: + if name.startswith(prefix): + name = name[len(prefix):] + break + name = re.sub(r"@(?=\d{8}$)", "-", name) + snapshot = re.search(r"-(\d{8}|\d{4}-\d{2}-\d{2})$", name) + return (name[:snapshot.start()], snapshot.group(1).replace("-", "")) if snapshot else (name, None) + + +def model_identity(model: str) -> tuple[str | None, str | None]: + """``(id, snapshot date or None)`` of a model name with its route, region profile and vendor prefix dropped; + the id is ``None`` for a name with no id. Two names with equal identities are one model and one snapshot.""" + base, date = _model_core(model) + if not base: + return None, date + # a lowercase word before the first dot is taken as a vendor prefix (``anthropic.``, ``amazon.``) and + # dropped; a dot after a digit or a dash (``gpt-4.1``, ``qwen2.5``) is part of the name + ident = re.sub(r"^[a-z]+\.", "", base) + return (ident if re.search(r"[a-z0-9]", ident) else None), date # separators alone name no model + + +def same_model(a: str, b: str) -> bool: + """Whether two model names are one model. + + The same id matches, and so does a dated snapshot and its undated alias (the alias runs + some snapshot of it), or the same id behind another route or region profile. Two dated + ids match only on the same date, and a bare family name (``haiku``) is not a full id. + + The provider is not part of the identity: the same weights on Bedrock and on the direct + API are still one model for the self-judge warning and for a baseline's model check. """ - prov, name = (provider or "").strip().lower(), (model or "").strip() - if prov == "amazon-bedrock": - name = BEDROCK_MODEL_IDS.get(name, name) - return f"{prov}/{name.lower()}" + (base_a, date_a), (base_b, date_b) = model_identity(a), model_identity(b) + if base_a is None or base_b is None or (date_a and date_b and date_a != date_b): + return False + return base_a == base_b def _read_key(value: str) -> str: diff --git a/dokimi/tests/test_baseline_compare.py b/dokimi/tests/test_baseline_compare.py index 72ffc3733..16500ca0d 100644 --- a/dokimi/tests/test_baseline_compare.py +++ b/dokimi/tests/test_baseline_compare.py @@ -128,6 +128,91 @@ def test_compare_treats_a_bedrock_alias_and_its_profile_as_one_judge(compare, ca assert capsys.readouterr().err == "" +def test_one_judge_id_and_snapshot_behind_another_route_is_one_judge_model(): + dated = _eval(judge=("anthropic", "claude-sonnet-4-5-20250929")) + assert baseline_mod._model_mismatches(dated, _eval(judge=("amazon-bedrock", "us.anthropic.claude-sonnet-4-5-20250929"))) == ( + [], []) + alias = _eval(judge=("anthropic", "claude-sonnet-4-5")) + assert baseline_mod._model_mismatches(alias, _eval(judge=("amazon-bedrock", "us.anthropic.claude-sonnet-4-5"))) == ( + [], []) + + +def test_a_dated_snapshot_and_its_undated_alias_are_not_verified_in_either_direction(): + """The alias does not record which snapshot answered: a role that was not measured, so it is named, not matched.""" + dated = _eval(judge=("anthropic", "claude-sonnet-4-5-20250929")) + alias = _eval(judge=("anthropic", "claude-sonnet-4-5")) + assert baseline_mod._model_mismatches(dated, alias) == ([], ["judge"]) + assert baseline_mod._model_mismatches(alias, dated) == ([], ["judge"]) + + +def test_a_panel_is_the_same_only_when_its_judges_pair_one_to_one(): + verdict = baseline_mod._recorded_verdict + assert verdict(("panel", "anthropic/m,anthropic/m"), ("panel", "m,anthropic/m")) == "match" + assert verdict(("panel", "anthropic/m,anthropic/m,anthropic/m"), ("panel", "anthropic/m,anthropic/m")) == "mismatch" + assert verdict(("panel", "m-20240101,m-20240101"), ("panel", "m,m-20250101")) == "mismatch" + assert verdict(("panel", "m"), ("anthropic", "m")) == "mismatch" + + +def test_a_panel_pairs_its_judges_by_augmenting_paths_not_first_match(): + """``m`` matches both snapshots and ``m-20240101`` only its own: a first-match pairing gives ``m`` the + judge ``m-20240101`` and strands the second baseline judge, though a valid pairing exists. The pairing + alone is only "not verified" (a dated judge against an undated one), never a match.""" + verdict = baseline_mod._recorded_verdict + assert verdict(("panel", "m,m-20240101"), ("panel", "m-20240101,m-20250101")) == "unverified" + assert verdict(("panel", "m-20240101,m-20250101"), ("panel", "m,m-20240101")) == "unverified" # the reverse side + assert verdict(("panel", "m-20240101,m"), ("panel", "m-20240101,m-20250101")) == "unverified" + assert verdict(("panel", "m-20240101,m-20240101"), ("panel", "m,m-20250101")) == "mismatch" + assert verdict(("panel", "m,m-20240101"), ("panel", "m-20240101,m")) == "match" # the same judges, reordered + + +def test_a_panel_of_names_with_no_id_is_a_mismatch_and_digit_ids_still_match(): + """A judge name with no id (separators only) is no model, so two such panels are not the same panel; + a digit-only id is an id, as it was on this branch before the round's first fix.""" + verdict = baseline_mod._recorded_verdict + assert verdict(("panel", "/,/"), ("panel", "-,-")) == "mismatch" + assert verdict(("panel", "/,/"), ("panel", "/,/")) == "mismatch" + assert verdict(("anthropic", "8"), ("anthropic", "8")) == "match" + assert verdict(("panel", "anthropic/8,anthropic/9"), ("panel", "openai/9,openai/8")) == "match" + + +def test_a_panel_that_gained_a_judge_is_a_mismatch(): + """Every judge of the baseline panel can pair with one of the current, and a third one is added: + pairing alone would pass it, so the sizes are compared first.""" + verdict = baseline_mod._recorded_verdict + assert verdict(("panel", "m,m-20240101"), ("panel", "m,m-20240101,m")) == "mismatch" + assert verdict(("panel", "m,m-20240101,m"), ("panel", "m,m-20240101")) == "mismatch" + + +def test_two_snapshots_of_one_judge_model_are_a_mismatch(): + lines, _ = baseline_mod._model_mismatches( + _eval(judge=("anthropic", "claude-sonnet-4-5-20240620")), + _eval(judge=("anthropic", "claude-sonnet-4-5-20250929"))) + assert lines == ["judge model: baseline anthropic/claude-sonnet-4-5-20240620, " + "current anthropic/claude-sonnet-4-5-20250929"] + + +@pytest.mark.parametrize("base_judge, cur_judge", [ + ("claude-sonnet-4-5-20240620", "claude-sonnet-4-5"), # the baseline dated, the current an undated alias + ("claude-sonnet-4-5", "claude-sonnet-4-5-20240620"), # and the other way round +]) +def test_compare_strict_refuses_a_dated_judge_against_its_undated_alias(compare, capsys, base_judge, cur_judge): + """The alias does not say which snapshot answered, so --strict cannot verify that the judge did not change.""" + with pytest.raises(SystemExit) as exc: + compare(_eval(judge=("anthropic", base_judge)), _eval(judge=("anthropic", cur_judge)), "--strict") + assert exc.value.code == 1 + out, err = capsys.readouterr() + assert "--strict cannot verify the judge model" in err + assert "Avg delta" not in out + + +def test_compare_warns_that_a_dated_judge_against_its_undated_alias_is_not_verified(compare, capsys): + compare(_eval(judge=("anthropic", "claude-sonnet-4-5-20240620")), _eval(judge=("anthropic", "claude-sonnet-4-5"))) + out, err = capsys.readouterr() + assert "judge model" in err and "not verified" in err + assert "Baseline model warning" not in err and "judge model: baseline" not in err + assert "Avg delta" in out + + def test_compare_names_a_role_the_baseline_does_not_record(compare, capsys): old = _eval() del old["run_metadata"] diff --git a/dokimi/tests/test_model_defaults.py b/dokimi/tests/test_model_defaults.py index 300644e76..07c7d1e01 100644 --- a/dokimi/tests/test_model_defaults.py +++ b/dokimi/tests/test_model_defaults.py @@ -152,8 +152,18 @@ def test_independence_warning_sees_a_bedrock_alias_and_its_profile_as_one_model( assert _warn_judge_independence("amazon-bedrock", profile, "amazon-bedrock", bare) assert "independence warning" in capsys.readouterr().err assert not _warn_judge_independence("amazon-bedrock", "claude-haiku-5-5", "amazon-bedrock", bare) - # the alias only exists on Bedrock: the same strings on another provider stay distinct - assert not _warn_judge_independence("anthropic", profile, "anthropic", bare) + # the same model answering and judging is the same bias on any route, so it warns on the direct API too + capsys.readouterr() + assert _warn_judge_independence("anthropic", profile, "anthropic", bare) + assert "independence warning" in capsys.readouterr().err + assert _warn_judge_independence("anthropic", "claude-sonnet-5-5", "amazon-bedrock", "claude-sonnet-5-5") + assert "independence warning" in capsys.readouterr().err + assert not _warn_judge_independence("anthropic", "claude-sonnet-5-5", "anthropic", "claude-haiku-5-5") + assert capsys.readouterr().err == "" + # an unset model is not evidence of sameness + assert not _warn_judge_independence("anthropic", "", "anthropic", "") + assert not _warn_judge_independence("anthropic", "claude-sonnet-5-5", "anthropic", "") + assert capsys.readouterr().err == "" @pytest.mark.parametrize("model, provider", [ diff --git a/dokimi/tests/test_pipeline_independence_flag.py b/dokimi/tests/test_pipeline_independence_flag.py new file mode 100644 index 000000000..ac17fd288 --- /dev/null +++ b/dokimi/tests/test_pipeline_independence_flag.py @@ -0,0 +1,43 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""``dokimi run`` stores the same ``evaluator_independence_warning`` that ``dokimi eval`` does.""" + +from __future__ import annotations + +import types + +import pytest + +from dokimi.orchestrator.pipeline import EvaluationPipeline + + +@pytest.mark.parametrize("provider, model, judge_provider, judge_model, expected", [ + ("anthropic", "claude-sonnet-5-5", "amazon-bedrock", "claude-sonnet-5-5", True), # one model, two routes + ("anthropic", "claude-sonnet-5-5", "anthropic", "claude-sonnet-5-5", True), + ("amazon-bedrock", "us.anthropic.claude-sonnet-5-5-v1:0", "amazon-bedrock", "claude-sonnet-5-5", True), + ("anthropic", "claude-sonnet-5-5", "anthropic", "claude-haiku-5-5", False), +]) +def test_the_stored_flag_follows_the_same_model_rule(tmp_path, monkeypatch, provider, model, judge_provider, + judge_model, expected): + seen = {} + + class FakeJudge: + def __init__(self, **kwargs): + seen.update(kwargs["run_metadata"]) + + def evaluate_answers(self, *args, **kwargs): + return [] + + def save_evaluations(self, *args, **kwargs): + pass + + monkeypatch.setattr("dokimi.eval.Judge", FakeJudge) + monkeypatch.setattr("dokimi.records.read", lambda *a, **k: types.SimpleNamespace(records=[{"id": "q1"}])) + pipeline = EvaluationPipeline(product="oneTBB", output_dir=tmp_path, description="d", model=model, + provider=provider, + judge_model=judge_model, judge_provider=judge_provider) + + pipeline._evaluate_answers([{"id": "q1"}]) + + assert seen["evaluator_independence_warning"] is expected diff --git a/dokimi/tests/test_repo_compare.py b/dokimi/tests/test_repo_compare.py index 7cc208e22..205f31f61 100644 --- a/dokimi/tests/test_repo_compare.py +++ b/dokimi/tests/test_repo_compare.py @@ -884,7 +884,7 @@ def test_model_warnings_follow_the_selected_model(tmp_path): @pytest.mark.outcome("cjm-model-answered") -def test_model_matches_equivalence_does_not_warn(tmp_path): +def test_same_model_equivalence_does_not_warn(tmp_path): """Equivalent model IDs (Bedrock provider-prefix variants) do not trigger a warning.""" b = tmp_path / "b1" # Request: Bedrock-prefixed, baseline resolve: shorter form, treatment resolve: base form without version @@ -893,7 +893,7 @@ def test_model_matches_equivalence_does_not_warn(tmp_path): _cell(b, "fix_a", "guidance:raw", 1, model="us.anthropic.claude-haiku-4-5-20251001-v1:0", model_resolved="claude-haiku-4-5-20251001") s = compare([b], "guidance:none") - # No warning about different resolved models—they are equivalent under model_matches() + # No warning about different resolved models—they are equivalent under same_model() assert [w for w in s["warnings"] if "was answered by" in w and "baseline" in w and "treatment" in w] == [] # But cells mismatch still warns (the actual mismatch, not the compare mismatch) @@ -922,7 +922,7 @@ def test_compare_models_equivalent_pins_symmetric_coverage(tmp_path): m = "us.anthropic.claude-haiku-4-5-20251001-v1:0" # Baseline: full Bedrock-prefixed model _cell(b, "fix_a", "guidance:none", 1, model=m, model_resolved="us.anthropic.claude-haiku-4-5-20251001-v1:0") - # Treatment: shorter form (treated as equivalent by model_matches) + # Treatment: shorter form (treated as equivalent by same_model) _cell(b, "fix_a", "guidance:raw", 1, model=m, model_resolved="claude-haiku-4-5-20251001") # No warning because both forms are equivalent assert [w for w in compare([b], "guidance:none")["warnings"] if "was answered by" in w] == [] diff --git a/dokimi/tests/test_repo_run.py b/dokimi/tests/test_repo_run.py index bc1eb25ad..c63b9878d 100644 --- a/dokimi/tests/test_repo_run.py +++ b/dokimi/tests/test_repo_run.py @@ -15,7 +15,8 @@ from dokimi.cli import build_parser from dokimi.repos.container import CONTENT_DIR, HARNESS_DIR, RUN_DIR, SOURCE_DIR, WORKSPACE -from dokimi.repos.report import format_report, model_matches, summarize, wilson +from dokimi.repos.report import format_report, summarize, wilson +from dokimi.utils import same_model from repo_eval_fixture import FakeRunner, fake_harness, git, make_repo @@ -833,7 +834,7 @@ def test_report_warns_when_another_model_answered(setup, capsys): ("m-v1:0", "m-v3:0", True), ("gpt-4.1", "gpt-4.2", False), ("us.anthropic.claude-x", "eu.anthropic.claude-y", False), ("vertex_ai/claude-x", "claude-x", True), ("claude-sonnet-4-5", "claude-sonnet-4-5-1", False)]) def test_a_provider_prefix_is_not_another_model(requested, resolved, same): - assert model_matches(requested, resolved) is same + assert same_model(requested, resolved) is same @pytest.mark.parametrize("key, value", [("budget_usd", float("nan")), ("timeout_s", float("inf")), diff --git a/dokimi/tests/test_same_model.py b/dokimi/tests/test_same_model.py new file mode 100644 index 000000000..808f3668d --- /dev/null +++ b/dokimi/tests/test_same_model.py @@ -0,0 +1,104 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""One rule for "same model": the self-judge warning, `baseline compare` and `repo compare` all use it.""" + +from __future__ import annotations + +import pytest + +from dokimi.defaults import BEDROCK_MODEL_IDS +from dokimi.utils import model_identity, same_model + +# (a, b, same) with a route-qualified name ("provider/model") where the route is part of the input. +CASES = [ + ("claude-sonnet-4-5", "claude-sonnet-4-5", True), + ("Claude-Sonnet-4-5", "claude-sonnet-4-5", True), + (" claude-sonnet-4-5 ", "claude-sonnet-4-5", True), + ("claude-sonnet-4-5-20250929", "claude-sonnet-4-5", True), # a snapshot and its undated alias + ("claude-3-5-sonnet-20240620", "claude-3-5-sonnet-20241022", False), # two snapshots + ("claude-sonnet-4-5@20250929", "claude-sonnet-4-5-20250929", True), # Vertex date form + ("claude-sonnet-4-5", "claude-opus-4-5-20251101", False), + ("claude-sonnet-4-5", "claude-sonnet-4-5-1", False), + ("haiku", "claude-haiku-3", False), # a family name is not a full id + ("x.claude-haiku", "haiku", False), + ("gpt-4.1", "1", False), # a dot after a digit or dash is not a vendor prefix + ("", "", False), # an unset model is not evidence of sameness + ("claude-sonnet-4-5", "", False), + ("deepseek-v2", "deepseek-v3", False), # a bare -v2 is part of the id + ("us.anthropic.claude-sonnet-4-5", "eu.anthropic.claude-sonnet-4-5", True), # regional profiles + ("global.anthropic.claude-sonnet-4-5", "claude-sonnet-4-5", True), + ("us.anthropic.claude-x", "eu.anthropic.claude-y", False), + ("us.anthropic.claude-haiku-4-5-20251001-v1:0", "claude-haiku-4-5-20251001", True), + ("eu.anthropic.claude-sonnet-4-5-20250929-v1:0", "us.anthropic.claude-sonnet-4-5", True), + ("m-v1:0", "m-v3:0", True), # Bedrock revision suffix + ("openai.gpt-oss-120b-1:0", "gpt-oss-120b", True), # a plain -N:M revision + ("openai.gpt-oss-120b-1:0", "openai.gpt-oss-120b-2:0", True), + ("openai.gpt-oss-120b-1:0", "gpt-oss-20b", False), + ("claude-sonnet-4-5", "us.anthropic.claude-sonnet-4-5-20250929-v1:0", True), # alias vs full Bedrock id + ("claude-x-20250929", "claude-x-20241022", False), + ("gpt-4o-2024-08-06", "gpt-4o", True), # OpenAI's dashed snapshot and its alias + ("gpt-4o-2024-08-06", "gpt-4o-2024-05-13", False), + ("gpt-4o-2024-08-06", "gpt-4o-20240806", True), + ("anthropic/claude-sonnet-4-5", "amazon-bedrock/claude-sonnet-4-5", True), # same model, another route + ("anthropic/claude-x", "claude-x", True), + ("vertex_ai/claude-x", "claude-x", True), + ("vertex_ai/claude-x@20250929", "vertex_ai/claude-x", True), + ("bedrock/us.anthropic.claude-x", "us.anthropic.claude-x", True), + ("amazon-bedrock/us.anthropic.claude-sonnet-4-5", "amazon-bedrock/eu.anthropic.claude-sonnet-4-5", True), + ("openrouter/anthropic/claude-sonnet-4-5", "anthropic/claude-sonnet-4-5", True), + ("anthropic/claude-sonnet-5-5", "anthropic/us.anthropic.claude-sonnet-5-5", True), + # the organization is part of a Hugging Face style id; only a route is dropped + ("meta-llama/Llama-3.1-8B-Instruct", "myorg/Llama-3.1-8B-Instruct", False), + ("meta-llama/Llama-3.1-8B-Instruct", "meta-llama/Llama-3.1-8B-Instruct", True), + ("openrouter/meta-llama/llama-3.1-8b-instruct", "meta-llama/Llama-3.1-8B-Instruct", True), + ("openrouter/meta-llama/llama-3.1-8b-instruct", "openrouter/myorg/llama-3.1-8b-instruct", False), + ("meta-llama/Llama-3.1-8B-Instruct", "Llama-3.1-8B-Instruct", False), + ("arn:aws:bedrock:us-east-1:123456789012:inference-profile/us.anthropic.claude-sonnet-4-5-20250929-v1:0", + "claude-sonnet-4-5-20250929", True), + ("bedrock/converse/openai.gpt-oss-120b", "openai/gpt-oss-120b", True), + ("azure/gpt-4o", "gpt-4o", True), # a litellm route that is not one of dokimi's providers + ("groq/llama-3.1-8b", "llama-3.1-8b", True), + ("together_ai/meta-llama/llama-3-8b", "meta-llama/llama-3-8b", True), + ("models/gemini-2.0-flash", "gemini-2.0-flash", True), + ("arn:aws:bedrock:us-east-1::foundation-model/anthropic.claude-sonnet-4-5-20250929-v1:0", + "claude-sonnet-4-5-20250929", True), +] + + +@pytest.mark.parametrize("a, b, same", CASES) +def test_same_model_table(a, b, same): + assert same_model(a, b) is same + + +@pytest.mark.parametrize("a, b, same", CASES) +def test_same_model_is_symmetric(a, b, same): + assert same_model(b, a) is same + + +@pytest.mark.parametrize("bare, profile", sorted(BEDROCK_MODEL_IDS.items())) +def test_a_bedrock_alias_and_its_profile_are_one_model(bare, profile): + assert same_model(bare, profile) + assert same_model(f"amazon-bedrock/{bare}", f"amazon-bedrock/{profile}") + + +@pytest.mark.parametrize("a, b", [ + ("anthropic.", "openai."), # a vendor prefix with no id after it + ("anthropic.", "anthropic."), + ("us.anthropic.", "us.anthropic."), + ("openai/", "openai/"), # a route with no id after it + ("/", "/"), # separators only + (".", "."), + ("eu./", "/"), # a region profile with no id after it + ("-", "-"), +]) +def test_a_name_with_no_id_is_no_model(a, b): + assert same_model(a, b) is False + assert model_identity(a)[0] is None + + +def test_every_provider_dokimi_accepts_is_a_route_not_an_organization(): + from dokimi.llm import PROVIDER_ALIASES, PROVIDERS + + for provider in [*PROVIDERS, *PROVIDER_ALIASES]: + assert same_model(f"{provider}/gpt-4o", "gpt-4o"), provider