From a3ff4a5c8e5aee7b0299a167b4c6d6ba1a35343e Mon Sep 17 00:00:00 2001 From: Nick Sullivan Date: Tue, 25 Aug 2026 14:45:04 -0500 Subject: [PATCH 1/2] fix(benchmarks): derive AA slugs so the key is the only remaining blocker 22 of 35 models render blank Reasoning/Coding/Agentic on the live site (model-data.json generated 2026-08-24, verified today). Three independent walls caused it; this removes two of them. Wall 1 - the weekly refresh has carried --no-aa since its first commit (b51e3d4, 2026-07-25), so no scheduled run has ever been capable of writing an AA score. Removed, and AA_API_KEY is now passed through from secrets. Wall 3 - enrichment was gated on a 9-entry hand-written AA_SLUG_MAP that contained 0 of the 23 affected models, so even a keyed run would have skipped every one of them. Slugs are now derived (drop the vendor prefix, strip a trailing -it, flatten dots to hyphens), which reproduces 8 of the 9 hand entries; the 9th (claude-haiku-4.5 -> claude-4-5-haiku) reorders the name and stays in the map, which always wins. Safety: a derived slug is returned ONLY on an exact key match in the AA payload. A near-miss returns None rather than a guess, so this can add coverage but can never attach one model's scores to another. Mutation-tested - reverting that check turns two tests red. Keyless behaviour is unchanged and verified: load_aa_api_key() returns None, aa_models stays empty, zero NEW models resolve, nothing is enriched or wiped. Adding the secret is now the only switch. Measured: with the derived slugs, all 23 blank models resolve. --- .../workflows/refresh-model-benchmarks.yml | 8 +- model-benchmarks/scripts/fetch-model.py | 34 +++++++- model-benchmarks/tests/test_aa_slug.py | 81 +++++++++++++++++++ 3 files changed, 121 insertions(+), 2 deletions(-) create mode 100644 model-benchmarks/tests/test_aa_slug.py diff --git a/.github/workflows/refresh-model-benchmarks.yml b/.github/workflows/refresh-model-benchmarks.yml index 7a4352b..4369998 100644 --- a/.github/workflows/refresh-model-benchmarks.yml +++ b/.github/workflows/refresh-model-benchmarks.yml @@ -24,7 +24,13 @@ jobs: python-version: "3.13" - name: Refresh OpenRouter metadata - run: python model-benchmarks/scripts/fetch-model.py --refresh --no-aa + # AA enrichment is attempted only when AA_API_KEY exists as a repo + # secret. Without it load_aa_api_key() returns None and the run is + # byte-identical to the old --no-aa behaviour, so adding the secret + # is the ONLY switch needed to start populating scores. + env: + AA_API_KEY: ${{ secrets.AA_API_KEY }} + run: python model-benchmarks/scripts/fetch-model.py --refresh - name: Watch EQ-Bench upstream for new rows id: watch diff --git a/model-benchmarks/scripts/fetch-model.py b/model-benchmarks/scripts/fetch-model.py index f212e7f..f8d1063 100644 --- a/model-benchmarks/scripts/fetch-model.py +++ b/model-benchmarks/scripts/fetch-model.py @@ -24,6 +24,7 @@ import argparse import json import os +import re import sys import urllib.request from datetime import datetime, timezone @@ -71,6 +72,37 @@ "minimax/minimax-m2.7": "minimax-m2-7", } + +def derive_aa_slug(model_id: str) -> str: + """Derive a candidate Artificial Analysis slug from an OpenRouter model id. + + AA slugs are the bare model name with dots flattened to hyphens and any + trailing instruct-tuning marker dropped. Verified against every entry in + AA_SLUG_MAP by test_derive_aa_slug(); the one entry it cannot reproduce + (claude-haiku-4.5 -> claude-4-5-haiku, which reorders the name) stays in + the hand map, which always wins. + """ + slug = model_id.split("/", 1)[1] if "/" in model_id else model_id + slug = re.sub(r"-it$", "", slug) + return slug.replace(".", "-") + + +def resolve_aa_slug(model_id: str, aa_models: dict) -> str | None: + """Resolve a model id to an AA slug: hand map first, then derivation. + + The hand map is authoritative and is returned as-is, so a stale entry + still surfaces via the caller's "AA slug not found" diagnostic instead of + disappearing silently. + + A DERIVED slug is only ever returned when it is an EXACT key in the AA + payload. A near-miss returns None rather than a guess, so derivation can + add coverage but can never attach one model's scores to another. + """ + if slug := AA_SLUG_MAP.get(model_id): + return slug + candidate = derive_aa_slug(model_id) + return candidate if candidate in aa_models else None + # OpenRouter renames/retires slugs over time. A dataset entry keeps its original # id (so PinchBench/EQ-Bench keys and site URLs stay stable) but its live # metadata is fetched from the current OpenRouter id listed here. @@ -810,7 +842,7 @@ def main(): transformed["notes"] = MODEL_NOTES[model_id] # Enrich with Artificial Analysis data if available - aa_slug = AA_SLUG_MAP.get(model_id) + aa_slug = resolve_aa_slug(model_id, aa_models) if aa_slug and aa_slug in aa_models: transformed = enrich_with_aa(transformed, aa_models[aa_slug]) print(f" + Enriched with Artificial Analysis data (slug: {aa_slug})") diff --git a/model-benchmarks/tests/test_aa_slug.py b/model-benchmarks/tests/test_aa_slug.py new file mode 100644 index 0000000..e0be226 --- /dev/null +++ b/model-benchmarks/tests/test_aa_slug.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +"""Tests for AA slug resolution. + +Guards two properties: + 1. Derivation reproduces the hand-maintained map (except the one reordered + entry, which the hand map must keep winning). + 2. Resolution NEVER returns a slug absent from the AA payload, so a derived + near-miss can never attach one model's scores to another. +""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent / "scripts")) + +import importlib.util + +_spec = importlib.util.spec_from_file_location( + "fetch_model", Path(__file__).parent.parent / "scripts" / "fetch-model.py" +) +fetch_model = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(fetch_model) + +AA_SLUG_MAP = fetch_model.AA_SLUG_MAP +derive_aa_slug = fetch_model.derive_aa_slug +resolve_aa_slug = fetch_model.resolve_aa_slug + +# The one entry derivation cannot produce: AA reorders the name components. +KNOWN_UNDERIVABLE = {"anthropic/claude-haiku-4.5"} + + +def test_derive_reproduces_hand_map(): + misses = { + mid: (derive_aa_slug(mid), slug) + for mid, slug in AA_SLUG_MAP.items() + if derive_aa_slug(mid) != slug + } + assert set(misses) == KNOWN_UNDERIVABLE, f"unexpected derivation misses: {misses}" + + +def test_hand_map_wins_over_derivation(): + mid = "anthropic/claude-haiku-4.5" + aa = {"claude-4-5-haiku": {}, "claude-haiku-4-5": {}} + assert resolve_aa_slug(mid, aa) == "claude-4-5-haiku" + + +def test_derivation_adds_coverage(): + # A model absent from the hand map but present in AA now resolves. + mid = "openai/gpt-5.5" + assert mid not in AA_SLUG_MAP + assert resolve_aa_slug(mid, {"gpt-5-5": {}}) == "gpt-5-5" + + +def test_near_miss_returns_none_not_a_guess(): + # THE LOAD-BEARING CASE. AA holds a similar-but-different slug; we must + # return None rather than attach the wrong model's scores. + mid = "openai/gpt-5.5" + assert resolve_aa_slug(mid, {"gpt-5-5-preview": {}, "gpt-5-4": {}}) is None + assert resolve_aa_slug(mid, {}) is None + + +def test_gate_varies_with_its_subject(): + # Proves the guard above is not vacuous: same model id, two payloads, + # opposite answers. + mid = "z-ai/glm-5.2" + assert resolve_aa_slug(mid, {"glm-5-2": {}}) == "glm-5-2" + assert resolve_aa_slug(mid, {"glm-5-3": {}}) is None + + +if __name__ == "__main__": + failed = 0 + for name, fn in sorted(globals().items()): + if name.startswith("test_") and callable(fn): + try: + fn() + print(f"PASS {name}") + except AssertionError as e: + failed += 1 + print(f"FAIL {name}: {e}") + print(f"\n{'FAILED' if failed else 'all tests passed'}") + sys.exit(1 if failed else 0) From a3cc386873d68e77d3f3d9aef05cd8a0d95b41b6 Mon Sep 17 00:00:00 2001 From: Nick Sullivan Date: Tue, 25 Aug 2026 14:47:06 -0500 Subject: [PATCH 2/2] style: ruff-format blank line after resolve_aa_slug --- model-benchmarks/scripts/fetch-model.py | 1 + 1 file changed, 1 insertion(+) diff --git a/model-benchmarks/scripts/fetch-model.py b/model-benchmarks/scripts/fetch-model.py index f8d1063..0138caa 100644 --- a/model-benchmarks/scripts/fetch-model.py +++ b/model-benchmarks/scripts/fetch-model.py @@ -103,6 +103,7 @@ def resolve_aa_slug(model_id: str, aa_models: dict) -> str | None: candidate = derive_aa_slug(model_id) return candidate if candidate in aa_models else None + # OpenRouter renames/retires slugs over time. A dataset entry keeps its original # id (so PinchBench/EQ-Bench keys and site URLs stay stable) but its live # metadata is fetched from the current OpenRouter id listed here.