Discover the whole OpenRouter catalog for BYO SaaS tenants

saas_routes.py

@@ -69,7 +69,13 @@ def choices(host):
                     "tools": bool((c.get("capabilities") or {}).get("supports_tools")),
                     "price_in": finite(c.get("raw_price_in", c.get("price_in"))),
                     "price_out": finite(c.get("raw_price_out", c.get("price_out")))}
-    return sorted(out.values(), key=lambda c: (c["provider"], c["family"]))
+    # A curated family and its marketplace twin (`openrouter` / `openrouter_market`)
+    # are one choice for the user: keep the curated identity, which carries the
+    # benchmark ranking, and drop the duplicate label.
+    for key, row in list(out.items()):
+        if row["provider"] == "openrouter_market" and f"openrouter|{row['family']}" in out:
+            del out[key]
+    return sorted(out.values(), key=lambda c: (label(c["provider"]), c["family"]))
 
 
 def finite(value):
@@ -163,7 +169,8 @@ def preview(host, intent):
             explanation = "Not available through your connected provider accounts."
         else:
             explanation = "Does not meet the mandatory requirements."
-        exclusions.append({"id": target, "label": target.replace("|", " · "), "reason": explanation})
+        pid, _, family = target.partition("|")
+        exclusions.append({"id": target, "label": f"{family} · {label(pid)}", "reason": explanation})
     warnings = []
     if len(rows) == 1:
         warnings.append("Only one model qualifies. There is no fallback if it fails.")

sources/openrouter.py

@@ -28,11 +28,15 @@ class OpenRouterSource:
 
     def __init__(self, catalog: dict, env_get=os.environ.get,
                  client: Any = None, base_url: str = BASE_URL,
-                 route_stats=None):
+                 route_stats=None, endpoint_details: bool = True):
         self._env_get = env_get
         self._base_url = base_url.rstrip("/")
         self._client = client  # injected in tests; lazy httpx otherwise
         self._route_stats = route_stats or host_store.route_stats
+        # One detail request per model (~1000) is affordable on the hourly
+        # operator refresh, not on an on-demand SaaS catalog request. Without
+        # details every listed model stays routable (see _model_available).
+        self._endpoint_details = endpoint_details
         # /models snapshot cached by the async pricing() refresh so the SYNC
         # discover hook (offers_sync, called inside rank) never blocks on HTTP.
         self._models_snapshot: list[dict] = []
@@ -159,7 +163,8 @@ async def pricing(self) -> list[Price]:
         # cache for the sync offers_sync()/market_book() (whole-catalog discovery)
         merged = {m['id']: m for m in (body.get("data") or []) + self._decision_snapshot if m.get('id')}
         self._models_snapshot = list(merged.values())
-        await self._refresh_endpoint_availability(self._models_snapshot)
+        if self._endpoint_details:
+            await self._refresh_endpoint_availability(self._models_snapshot)
         # Live, full model-level traits (benchmarks/modalities/caps + ranks) for
         # EVERY model, keyed by raw id, ranked across the whole OpenRouter
         # catalog. Discovered families carry these inline so they rank on real

tenant_providers.py

@@ -100,6 +100,67 @@ async def _aws(host):
             await source._client.aclose()
 
 
+# OpenRouter's /models listing is public and identical for every tenant: one
+# shared snapshot, refreshed on demand by at most one request at a time. It is
+# used only for tenants with their own OpenRouter key; offers carry no operator
+# latency/health, and calls still authenticate with the tenant's key.
+OPENROUTER_CATALOG_TTL_S = 600
+OPENROUTER_CATALOG_STALE_S = 3600
+OPENROUTER_CATALOG_RETRY_S = 60  # after a failed refresh, do not call OpenRouter again for this long
+_openrouter_public = {'at': 0.0, 'value': None, 'failed_at': None}
+_openrouter_lock = None
+
+
+async def _openrouter_catalog(catalog):
+    global _openrouter_lock
+    loop = asyncio.get_running_loop()
+    if _openrouter_lock is None or _openrouter_lock[0] is not loop:
+        _openrouter_lock = (loop, asyncio.Lock())
+    async with _openrouter_lock[1]:
+        now = time.monotonic()
+        age = now - _openrouter_public['at']
+        stale = _openrouter_public['value'] is not None and age < OPENROUTER_CATALOG_STALE_S
+        if _openrouter_public['value'] is not None and age < OPENROUTER_CATALOG_TTL_S:
+            return _openrouter_public['value']
+        failed_at = _openrouter_public['failed_at']
+        if failed_at is not None and now - failed_at < OPENROUTER_CATALOG_RETRY_S:
+            # Serialized callers must not each wait for another upstream timeout.
+            if stale:
+                return _openrouter_public['value']
+            raise RuntimeError('OpenRouter catalog recently unavailable')
+        from sources.openrouter import OpenRouterSource
+        source = OpenRouterSource(catalog, env_get=lambda _key: None, route_stats=lambda: {},
+                                  endpoint_details=False)
+        try:
+            prices = await source.pricing()
+            value = ({pid: source.offers_sync(pid) for pid in source.provider_ids if pid != 'openrouter'}, prices)
+        except Exception:
+            _openrouter_public['failed_at'] = time.monotonic()
+            if stale:
+                return _openrouter_public['value']  # public data: a short outage keeps the last list
+            raise
+        finally:
+            if source._client is not None:
+                await source._client.aclose()
+        _openrouter_public.update(at=time.monotonic(), value=value, failed_at=None)
+        return value
+
+
+async def _openrouter(host):
+    from sources import push_prices
+    market = [pid for pid, p in host.catalog()['providers'].items()
+              if p.get('discovery_id') == 'openrouter_market']
+    host._tenant_offers.update({pid: [] for pid in market})
+    try:
+        offers, prices = await asyncio.wait_for(_openrouter_catalog(host.catalog()), timeout=35)
+        host._tenant_offers.update({pid: offers.get(pid, []) for pid in market})
+        # Curated families keep the static `openrouter` provider; this only fills
+        # their current prices, in this request-local engine.
+        push_prices(host, host.catalog(), prices)
+    except Exception:
+        host._connection_errors['openrouter'] = 'Could not load the OpenRouter model list. Try again shortly.'
+
+
 async def prepare(host):
     configs = getattr(host, '_tenant_connections', {})
     host._connection_errors = {}
@@ -133,4 +194,6 @@ async def one(name, load, ids):
         tasks.append(one('bedrock', _aws, ids))
     if configs.get('antseed') and host._env.get('ANTSEED_BYO_URL'):
         tasks.append(one('antseed', _buyer, ['antseed']))
+    if host._env.get('OPENROUTER_API_KEY') and 'openrouter_market' in (host._tenant_allowed or ()):
+        tasks.append(_openrouter(host))
     await asyncio.gather(*tasks)

tests/fixtures/openrouter_byo.lua

@@ -0,0 +1,11 @@
+return {
+ providers = {
+  openrouter = {discovery='static',base_url='https://openrouter.ai/api/v1',api_kind='openai_compatible',auth_env='OPENROUTER_API_KEY',source='openrouter',tier='partner'},
+  openrouter_market = {discovery='marketplace',discovery_id='openrouter_market',base_url='https://openrouter.ai/api/v1',api_kind='openai_compatible',auth_env='OPENROUTER_API_KEY',source='openrouter',tier='marketplace',market_price_cap={input=1000,output=1000}},
+ },
+ models = {
+  ['curated-model'] = {served_by={{provider='openrouter',provider_model_id='vendor/curated-model'}},capabilities={context=128000,supports_tools=true,supports_json_mode=true}},
+ },
+ profiles={default={scorer={'zero'}}},
+ policy_envelope={'and',{'meets_req'},{'not',{'is','disabled'}}},
+}

tests/test_saas_openrouter_discovery.py

@@ -0,0 +1,157 @@
+"""BYO OpenRouter: the whole live catalog, discovered on demand without operator refreshers."""
+import asyncio
+from pathlib import Path
+
+import httpx
+import pytest
+
+from llm_router_host import LLMRouterHost
+from saas_routes import choices
+import tenant_providers as tp
+
+ROOT = Path(__file__).resolve().parents[1]
+
+MODELS = [
+    {"id": "vendor/curated-model", "pricing": {"prompt": "0.000002", "completion": "0.000008"},
+     "context_length": 200000, "supported_parameters": ["tools"],
+     "links": {"details": "/api/v1/models/vendor/curated-model/endpoints"}},
+    {"id": "newlab/brand-new-model", "pricing": {"prompt": "0.0000001", "completion": "0.0000004"},
+     "context_length": 131072, "supported_parameters": ["tools", "response_format"],
+     "architecture": {"input_modalities": ["text", "image"]},
+     "links": {"details": "/api/v1/models/newlab/brand-new-model/endpoints"}},
+    {"id": "other/variable-price", "pricing": {"prompt": "-1", "completion": "-1"}, "context_length": 8192},
+]
+
+
+@pytest.fixture
+def base(host_store_clean):
+    tp._cache.clear()
+    tp._openrouter_public.update(at=0.0, value=None, failed_at=None)
+    host = LLMRouterHost(ROOT/'core/router.lua', ROOT/'tests/fixtures/openrouter_byo.lua',
+        discover=lambda _: {'ok': True, 'offers': [{'model_family': 'OPERATOR-ONLY'}]})
+    host.init()
+    return host
+
+
+@pytest.fixture
+def openrouter(monkeypatch):
+    """Mock OpenRouter's public API; records every request path."""
+    state = {"paths": [], "fail": False, "models": MODELS}
+    real = httpx.AsyncClient
+
+    def network(request):
+        state["paths"].append(request.url.path + ("?" + request.url.query.decode() if request.url.query else ""))
+        assert request.url.host == "openrouter.ai"
+        assert "authorization" not in request.headers  # the listing needs no tenant credential
+        if state["fail"]:
+            return httpx.Response(503)
+        if request.url.query:
+            return httpx.Response(200, json={"data": []})
+        return httpx.Response(200, json={"data": state["models"]})
+
+    monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **k: real(transport=httpx.MockTransport(network)))
+    return state
+
+
+def tenant(base, tenant_id, key="sk-or-tenant"):
+    return base.for_tenant(tenant_id, {"OPENROUTER_API_KEY": key} if key else {})
+
+
+def test_byo_openrouter_lists_live_catalog_once_with_curated_prices(base, openrouter):
+    child = tenant(base, 1)
+    asyncio.run(tp.prepare(child))
+    rows = {row["id"]: row for row in choices(child)}
+    assert set(rows) == {"openrouter|curated-model", "openrouter_market|brand-new-model"}
+    assert {row["label"].split(" · ")[1] for row in rows.values()} == {"OpenRouter"}
+    assert (rows["openrouter|curated-model"]["price_in"], rows["openrouter|curated-model"]["price_out"]) == pytest.approx((2.0, 8.0))
+    new = rows["openrouter_market|brand-new-model"]
+    assert (new["price_in"], new["price_out"]) == pytest.approx((0.1, 0.4))
+    assert new["tools"] is True
+    assert child._connection_errors == {}
+    # Only the listing (+ decision listing): no per-model endpoint detail requests.
+    assert not any("/endpoints" in path for path in openrouter["paths"])
+
+
+def test_new_models_appear_after_the_cache_expires(base, openrouter, monkeypatch):
+    asyncio.run(tp.prepare(tenant(base, 1)))
+    openrouter["models"] = MODELS + [{"id": "lab/released-today", "context_length": 32768,
+                                      "pricing": {"prompt": "0.0000003", "completion": "0.0000009"}}]
+    child = tenant(base, 2)
+    asyncio.run(tp.prepare(child))
+    assert "openrouter_market|released-today" not in {r["id"] for r in choices(child)}  # within TTL: shared snapshot
+    monkeypatch.setattr(tp, "OPENROUTER_CATALOG_TTL_S", 0)
+    child = tenant(base, 2)
+    asyncio.run(tp.prepare(child))
+    assert "openrouter_market|released-today" in {r["id"] for r in choices(child)}
+
+
+def test_listing_is_shared_across_tenants_and_fetched_once_per_ttl(base, openrouter):
+    async def run():
+        await asyncio.gather(*(tp.prepare(tenant(base, tid)) for tid in (1, 2, 3)))
+    asyncio.run(run())
+    assert [p for p in openrouter["paths"] if p.endswith("/models")] == ["/api/v1/models"]
+
+
+def test_without_openrouter_key_nothing_is_fetched_and_operator_discovery_is_not_used(base, openrouter):
+    child = tenant(base, 1, key=None)
+    asyncio.run(tp.prepare(child))
+    assert openrouter["paths"] == []
+    assert choices(child) == []
+
+
+def test_failure_reports_error_and_never_falls_back_to_operator(base, openrouter):
+    openrouter["fail"] = True
+    child = tenant(base, 1)
+    asyncio.run(tp.prepare(child))
+    ids = {r["id"] for r in choices(child)}
+    assert not any("OPERATOR-ONLY" in i for i in ids)
+    assert not any(i.startswith("openrouter_market|") for i in ids)
+    assert "sk-or" not in str(child._connection_errors)
+    assert child._connection_errors["openrouter"]
+
+
+def test_short_outage_serves_the_last_public_listing(base, openrouter, monkeypatch):
+    asyncio.run(tp.prepare(tenant(base, 1)))
+    monkeypatch.setattr(tp, "OPENROUTER_CATALOG_TTL_S", 0)
+    openrouter["fail"] = True
+    child = tenant(base, 2)
+    asyncio.run(tp.prepare(child))
+    assert "openrouter_market|brand-new-model" in {r["id"] for r in choices(child)}
+    assert child._connection_errors == {}
+
+
+
+def test_preview_ranks_long_tail_models_and_names_exclusions_by_public_provider(base, openrouter):
+    from saas_routes import preview
+    child = tenant(base, 1)
+    asyncio.run(tp.prepare(child))
+    result = preview(child, {"goal": "cost", "workload": "agent", "timeout_seconds": 8,
+                             "targets": ["openrouter|curated-model", "openrouter_market|brand-new-model",
+                                         "openrouter_market|retired-model"]})
+    assert [row["id"] for row in result["ranked"]] == ["openrouter_market|brand-new-model", "openrouter|curated-model"]
+    assert [row["label"] for row in result["excluded"]] == ["retired-model · OpenRouter"]
+
+
+def test_failed_refresh_is_not_retried_by_every_caller(base, openrouter, monkeypatch):
+    asyncio.run(tp.prepare(tenant(base, 1)))
+    monkeypatch.setattr(tp, "OPENROUTER_CATALOG_TTL_S", 0)
+    openrouter["fail"] = True
+    for tid in (2, 3, 4):
+        child = tenant(base, tid)
+        asyncio.run(tp.prepare(child))
+        assert "openrouter_market|brand-new-model" in {r["id"] for r in choices(child)}
+    # One listing, one failed refresh; later callers are served the last list without upstream calls.
+    assert [p for p in openrouter["paths"] if p.endswith("/models")] == ["/api/v1/models", "/api/v1/models"]
+    monkeypatch.setattr(tp, "OPENROUTER_CATALOG_RETRY_S", 0)
+    openrouter["fail"] = False
+    asyncio.run(tp.prepare(tenant(base, 5)))
+    assert tp._openrouter_public["failed_at"] is None
+
+
+def test_first_failure_without_a_listing_fails_fast_for_the_next_caller(base, openrouter):
+    openrouter["fail"] = True
+    asyncio.run(tp.prepare(tenant(base, 1)))
+    child = tenant(base, 2)
+    asyncio.run(tp.prepare(child))
+    assert child._connection_errors["openrouter"]
+    assert len([p for p in openrouter["paths"] if p.endswith("/models")]) == 1