fix(benchmarks): derive AA slugs so the API key is the only remaining blocker

.github/workflows/refresh-model-benchmarks.yml

@@ -24,7 +24,13 @@ jobs:
           python-version: "3.13"
 
       - name: Refresh OpenRouter metadata
-        run: python model-benchmarks/scripts/fetch-model.py --refresh --no-aa
+        # AA enrichment is attempted only when AA_API_KEY exists as a repo
+        # secret. Without it load_aa_api_key() returns None and the run is
+        # byte-identical to the old --no-aa behaviour, so adding the secret
+        # is the ONLY switch needed to start populating scores.
+        env:
+          AA_API_KEY: ${{ secrets.AA_API_KEY }}
+        run: python model-benchmarks/scripts/fetch-model.py --refresh
 
       - name: Watch EQ-Bench upstream for new rows
         id: watch

model-benchmarks/scripts/fetch-model.py

@@ -24,6 +24,7 @@
 import argparse
 import json
 import os
+import re
 import sys
 import urllib.request
 from datetime import datetime, timezone
@@ -71,6 +72,38 @@
     "minimax/minimax-m2.7": "minimax-m2-7",
 }
 
+
+def derive_aa_slug(model_id: str) -> str:
+    """Derive a candidate Artificial Analysis slug from an OpenRouter model id.
+
+    AA slugs are the bare model name with dots flattened to hyphens and any
+    trailing instruct-tuning marker dropped. Verified against every entry in
+    AA_SLUG_MAP by test_derive_aa_slug(); the one entry it cannot reproduce
+    (claude-haiku-4.5 -> claude-4-5-haiku, which reorders the name) stays in
+    the hand map, which always wins.
+    """
+    slug = model_id.split("/", 1)[1] if "/" in model_id else model_id
+    slug = re.sub(r"-it$", "", slug)
+    return slug.replace(".", "-")
+
+
+def resolve_aa_slug(model_id: str, aa_models: dict) -> str | None:
+    """Resolve a model id to an AA slug: hand map first, then derivation.
+
+    The hand map is authoritative and is returned as-is, so a stale entry
+    still surfaces via the caller's "AA slug not found" diagnostic instead of
+    disappearing silently.
+
+    A DERIVED slug is only ever returned when it is an EXACT key in the AA
+    payload. A near-miss returns None rather than a guess, so derivation can
+    add coverage but can never attach one model's scores to another.
+    """
+    if slug := AA_SLUG_MAP.get(model_id):
+        return slug
+    candidate = derive_aa_slug(model_id)
+    return candidate if candidate in aa_models else None
+
+
 # OpenRouter renames/retires slugs over time. A dataset entry keeps its original
 # id (so PinchBench/EQ-Bench keys and site URLs stay stable) but its live
 # metadata is fetched from the current OpenRouter id listed here.
@@ -810,7 +843,7 @@ def main():
             transformed["notes"] = MODEL_NOTES[model_id]
 
         # Enrich with Artificial Analysis data if available
-        aa_slug = AA_SLUG_MAP.get(model_id)
+        aa_slug = resolve_aa_slug(model_id, aa_models)
         if aa_slug and aa_slug in aa_models:
             transformed = enrich_with_aa(transformed, aa_models[aa_slug])
             print(f"    + Enriched with Artificial Analysis data (slug: {aa_slug})")

model-benchmarks/tests/test_aa_slug.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+"""Tests for AA slug resolution.
+
+Guards two properties:
+  1. Derivation reproduces the hand-maintained map (except the one reordered
+     entry, which the hand map must keep winning).
+  2. Resolution NEVER returns a slug absent from the AA payload, so a derived
+     near-miss can never attach one model's scores to another.
+"""
+
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
+
+import importlib.util
+
+_spec = importlib.util.spec_from_file_location(
+    "fetch_model", Path(__file__).parent.parent / "scripts" / "fetch-model.py"
+)
+fetch_model = importlib.util.module_from_spec(_spec)
+_spec.loader.exec_module(fetch_model)
+
+AA_SLUG_MAP = fetch_model.AA_SLUG_MAP
+derive_aa_slug = fetch_model.derive_aa_slug
+resolve_aa_slug = fetch_model.resolve_aa_slug
+
+# The one entry derivation cannot produce: AA reorders the name components.
+KNOWN_UNDERIVABLE = {"anthropic/claude-haiku-4.5"}
+
+
+def test_derive_reproduces_hand_map():
+    misses = {
+        mid: (derive_aa_slug(mid), slug)
+        for mid, slug in AA_SLUG_MAP.items()
+        if derive_aa_slug(mid) != slug
+    }
+    assert set(misses) == KNOWN_UNDERIVABLE, f"unexpected derivation misses: {misses}"
+
+
+def test_hand_map_wins_over_derivation():
+    mid = "anthropic/claude-haiku-4.5"
+    aa = {"claude-4-5-haiku": {}, "claude-haiku-4-5": {}}
+    assert resolve_aa_slug(mid, aa) == "claude-4-5-haiku"
+
+
+def test_derivation_adds_coverage():
+    # A model absent from the hand map but present in AA now resolves.
+    mid = "openai/gpt-5.5"
+    assert mid not in AA_SLUG_MAP
+    assert resolve_aa_slug(mid, {"gpt-5-5": {}}) == "gpt-5-5"
+
+
+def test_near_miss_returns_none_not_a_guess():
+    # THE LOAD-BEARING CASE. AA holds a similar-but-different slug; we must
+    # return None rather than attach the wrong model's scores.
+    mid = "openai/gpt-5.5"
+    assert resolve_aa_slug(mid, {"gpt-5-5-preview": {}, "gpt-5-4": {}}) is None
+    assert resolve_aa_slug(mid, {}) is None
+
+
+def test_gate_varies_with_its_subject():
+    # Proves the guard above is not vacuous: same model id, two payloads,
+    # opposite answers.
+    mid = "z-ai/glm-5.2"
+    assert resolve_aa_slug(mid, {"glm-5-2": {}}) == "glm-5-2"
+    assert resolve_aa_slug(mid, {"glm-5-3": {}}) is None
+
+
+if __name__ == "__main__":
+    failed = 0
+    for name, fn in sorted(globals().items()):
+        if name.startswith("test_") and callable(fn):
+            try:
+                fn()
+                print(f"PASS {name}")
+            except AssertionError as e:
+                failed += 1
+                print(f"FAIL {name}: {e}")
+    print(f"\n{'FAILED' if failed else 'all tests passed'}")
+    sys.exit(1 if failed else 0)