From be8607a5cdd0193117be60be24315dc05bd6bef1 Mon Sep 17 00:00:00 2001
From: Hex Sullivan <hex@technick.ai>
Date: Tue, 28 Jul 2026 11:40:27 +0000
Subject: [PATCH] fix(benchmarks): plug three eq_bench_public data-integrity
 gaps
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- merge_model: preserve sources.eq_bench_public alongside
  artificial_analysis and eq_bench so weekly --refresh doesn't silently
  drop the provenance flag while keeping the public scores

- apply_public_eq: clear stale PUBLIC_EQ_FIELDS + source flag when
  public_eq_block() returns None, preventing outdated data from being
  published after a mapped leaderboard row disappears or is renamed

- --discover: auto-apply public EQ scores for newly discovered models;
  discovery is already gated on EQBENCH_PUBLIC_MAP so every found model
  has a verified mapping — requiring a separate --eq-public pass was an
  unnecessary footgun

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 model-benchmarks/scripts/fetch-model.py | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/model-benchmarks/scripts/fetch-model.py b/model-benchmarks/scripts/fetch-model.py
index ebd81cd..6fe5d44 100644
--- a/model-benchmarks/scripts/fetch-model.py
+++ b/model-benchmarks/scripts/fetch-model.py
@@ -452,6 +452,13 @@ def apply_public_eq(data: dict, only_ids: list[str] | None = None) -> tuple[int,
         block = eqbench_public.public_eq_block(m["id"], leaderboard, traits)
         if block is None:
             missing += 1
+            # Clear stale public fields so outdated data isn't published after a
+            # mapped row disappears or is renamed upstream.
+            eq = m.get("benchmarks", {}).get("eq_bench", {})
+            if any(eq.get(f) is not None for f in eqbench_public.PUBLIC_EQ_FIELDS):
+                for f in eqbench_public.PUBLIC_EQ_FIELDS:
+                    eq.pop(f, None)
+                m.get("sources", {}).pop("eq_bench_public", None)
             continue
         eq = m.setdefault("benchmarks", {}).setdefault("eq_bench", {})
         before = {k: eq.get(k) for k in eqbench_public.PUBLIC_EQ_FIELDS}
@@ -529,7 +536,7 @@ def merge_model(existing_data: dict, new_model: dict) -> dict:
             if not new_model.get("speed") and m.get("speed"):
                 new_model["speed"] = m["speed"]
             # Preserve source flags for non-openrouter sources
-            for src_key in ("artificial_analysis", "eq_bench"):
+            for src_key in ("artificial_analysis", "eq_bench", "eq_bench_public"):
                 if m.get("sources", {}).get(src_key) and not new_model["sources"].get(
                     src_key
                 ):
@@ -746,6 +753,7 @@ def main():
 
     data = load_model_data()
 
+    discovered = []
     if args.discover:
         discovered = discover_new_models(data, all_models)
         if discovered:
@@ -822,6 +830,15 @@ def main():
             f"  Public EQ applied to {updated} model(s); "
             f"{missing} have no hand-verified public entry (left empty)."
         )
+    elif discovered:
+        # --discover gates on EQ-Bench map entries, so every discovered model
+        # already has a verified mapping — fetch their public scores automatically.
+        print(f"Fetching public EQ-Bench scores for {len(discovered)} discovered model(s)...")
+        updated, missing = apply_public_eq(data, only_ids=set(discovered))
+        print(
+            f"  Public EQ applied to {updated} model(s); "
+            f"{missing} have no hand-verified public entry (left empty)."
+        )
 
     if not args.dry_run:
         save_model_data(data)