From cace35d3d433995c5186b58734e8d4fec93673df Mon Sep 17 00:00:00 2001 From: jmlago Date: Wed, 23 Sep 2026 13:34:30 +0100 Subject: [PATCH 1/3] Discover the whole OpenRouter catalog for BYO SaaS tenants Cloud tenants with their own OpenRouter key only saw the ~24 curated families: the openrouter_market long tail and live prices come from the global source refreshers, which the SaaS router keeps disabled, and on-demand tenant discovery covered only Bedrock and AntSeed. - tenant_providers: OpenRouter discovery for tenants with OPENROUTER_API_KEY. The public /models listing is shared across tenants (10 min TTL, one refresh at a time, last listing kept up to 1 h during an outage). Offers carry no operator latency/health; calls keep using the tenant key. Curated family prices are pushed into the request-local engine only. - sources/openrouter: endpoint_details=False skips the per-model detail requests (~1000) on this on-demand path. - saas_routes.choices: a curated family and its marketplace twin are one choice (curated identity kept); list sorted by provider label. Co-Authored-By: Claude Opus 5.5 (1M context) --- saas_routes.py | 8 +- sources/openrouter.py | 9 +- tenant_providers.py | 53 +++++++++++ tests/fixtures/openrouter_byo.lua | 11 +++ tests/test_saas_openrouter_discovery.py | 120 ++++++++++++++++++++++++ 5 files changed, 198 insertions(+), 3 deletions(-) create mode 100644 tests/fixtures/openrouter_byo.lua create mode 100644 tests/test_saas_openrouter_discovery.py diff --git a/saas_routes.py b/saas_routes.py index edd91bb..bbb2bcf 100644 --- a/saas_routes.py +++ b/saas_routes.py @@ -69,7 +69,13 @@ def choices(host): "tools": bool((c.get("capabilities") or {}).get("supports_tools")), "price_in": finite(c.get("raw_price_in", c.get("price_in"))), "price_out": finite(c.get("raw_price_out", c.get("price_out")))} - return sorted(out.values(), key=lambda c: (c["provider"], c["family"])) + # A curated family and its marketplace twin (`openrouter` / `openrouter_market`) + # are one choice for the user: keep the curated identity, which carries the + # benchmark ranking, and drop the duplicate label. + for key, row in list(out.items()): + if row["provider"] == "openrouter_market" and f"openrouter|{row['family']}" in out: + del out[key] + return sorted(out.values(), key=lambda c: (label(c["provider"]), c["family"])) def finite(value): diff --git a/sources/openrouter.py b/sources/openrouter.py index a43f1d7..7697863 100644 --- a/sources/openrouter.py +++ b/sources/openrouter.py @@ -28,11 +28,15 @@ class OpenRouterSource: def __init__(self, catalog: dict, env_get=os.environ.get, client: Any = None, base_url: str = BASE_URL, - route_stats=None): + route_stats=None, endpoint_details: bool = True): self._env_get = env_get self._base_url = base_url.rstrip("/") self._client = client # injected in tests; lazy httpx otherwise self._route_stats = route_stats or host_store.route_stats + # One detail request per model (~1000) is affordable on the hourly + # operator refresh, not on an on-demand SaaS catalog request. Without + # details every listed model stays routable (see _model_available). + self._endpoint_details = endpoint_details # /models snapshot cached by the async pricing() refresh so the SYNC # discover hook (offers_sync, called inside rank) never blocks on HTTP. self._models_snapshot: list[dict] = [] @@ -159,7 +163,8 @@ async def pricing(self) -> list[Price]: # cache for the sync offers_sync()/market_book() (whole-catalog discovery) merged = {m['id']: m for m in (body.get("data") or []) + self._decision_snapshot if m.get('id')} self._models_snapshot = list(merged.values()) - await self._refresh_endpoint_availability(self._models_snapshot) + if self._endpoint_details: + await self._refresh_endpoint_availability(self._models_snapshot) # Live, full model-level traits (benchmarks/modalities/caps + ranks) for # EVERY model, keyed by raw id, ranked across the whole OpenRouter # catalog. Discovered families carry these inline so they rank on real diff --git a/tenant_providers.py b/tenant_providers.py index 3113eab..6fa95a2 100644 --- a/tenant_providers.py +++ b/tenant_providers.py @@ -100,6 +100,57 @@ async def _aws(host): await source._client.aclose() +# OpenRouter's /models listing is public and identical for every tenant: one +# shared snapshot, refreshed on demand by at most one request at a time. It is +# used only for tenants with their own OpenRouter key; offers carry no operator +# latency/health, and calls still authenticate with the tenant's key. +OPENROUTER_CATALOG_TTL_S = 600 +OPENROUTER_CATALOG_STALE_S = 3600 +_openrouter_public = {'at': 0.0, 'value': None} +_openrouter_lock = None + + +async def _openrouter_catalog(catalog): + global _openrouter_lock + loop = asyncio.get_running_loop() + if _openrouter_lock is None or _openrouter_lock[0] is not loop: + _openrouter_lock = (loop, asyncio.Lock()) + async with _openrouter_lock[1]: + age = time.monotonic() - _openrouter_public['at'] + if _openrouter_public['value'] is not None and age < OPENROUTER_CATALOG_TTL_S: + return _openrouter_public['value'] + from sources.openrouter import OpenRouterSource + source = OpenRouterSource(catalog, env_get=lambda _key: None, route_stats=lambda: {}, + endpoint_details=False) + try: + prices = await source.pricing() + value = ({pid: source.offers_sync(pid) for pid in source.provider_ids if pid != 'openrouter'}, prices) + except Exception: + if _openrouter_public['value'] is not None and age < OPENROUTER_CATALOG_STALE_S: + return _openrouter_public['value'] # public data: a short outage keeps the last list + raise + finally: + if source._client is not None: + await source._client.aclose() + _openrouter_public.update(at=time.monotonic(), value=value) + return value + + +async def _openrouter(host): + from sources import push_prices + market = [pid for pid, p in host.catalog()['providers'].items() + if p.get('discovery_id') == 'openrouter_market'] + host._tenant_offers.update({pid: [] for pid in market}) + try: + offers, prices = await asyncio.wait_for(_openrouter_catalog(host.catalog()), timeout=35) + host._tenant_offers.update({pid: offers.get(pid, []) for pid in market}) + # Curated families keep the static `openrouter` provider; this only fills + # their current prices, in this request-local engine. + push_prices(host, host.catalog(), prices) + except Exception: + host._connection_errors['openrouter'] = 'Could not load the OpenRouter model list. Try again shortly.' + + async def prepare(host): configs = getattr(host, '_tenant_connections', {}) host._connection_errors = {} @@ -133,4 +184,6 @@ async def one(name, load, ids): tasks.append(one('bedrock', _aws, ids)) if configs.get('antseed') and host._env.get('ANTSEED_BYO_URL'): tasks.append(one('antseed', _buyer, ['antseed'])) + if host._env.get('OPENROUTER_API_KEY') and 'openrouter_market' in (host._tenant_allowed or ()): + tasks.append(_openrouter(host)) await asyncio.gather(*tasks) diff --git a/tests/fixtures/openrouter_byo.lua b/tests/fixtures/openrouter_byo.lua new file mode 100644 index 0000000..f13027f --- /dev/null +++ b/tests/fixtures/openrouter_byo.lua @@ -0,0 +1,11 @@ +return { + providers = { + openrouter = {discovery='static',base_url='https://openrouter.ai/api/v1',api_kind='openai_compatible',auth_env='OPENROUTER_API_KEY',source='openrouter',tier='partner'}, + openrouter_market = {discovery='marketplace',discovery_id='openrouter_market',base_url='https://openrouter.ai/api/v1',api_kind='openai_compatible',auth_env='OPENROUTER_API_KEY',source='openrouter',tier='marketplace',market_price_cap={input=1000,output=1000}}, + }, + models = { + ['curated-model'] = {served_by={{provider='openrouter',provider_model_id='vendor/curated-model'}},capabilities={context=128000,supports_tools=true,supports_json_mode=true}}, + }, + profiles={default={scorer={'zero'}}}, + policy_envelope={'and',{'meets_req'},{'not',{'is','disabled'}}}, +} diff --git a/tests/test_saas_openrouter_discovery.py b/tests/test_saas_openrouter_discovery.py new file mode 100644 index 0000000..12b2b9c --- /dev/null +++ b/tests/test_saas_openrouter_discovery.py @@ -0,0 +1,120 @@ +"""BYO OpenRouter: the whole live catalog, discovered on demand without operator refreshers.""" +import asyncio +from pathlib import Path + +import httpx +import pytest + +from llm_router_host import LLMRouterHost +from saas_routes import choices +import tenant_providers as tp + +ROOT = Path(__file__).resolve().parents[1] + +MODELS = [ + {"id": "vendor/curated-model", "pricing": {"prompt": "0.000002", "completion": "0.000008"}, + "context_length": 200000, "supported_parameters": ["tools"], + "links": {"details": "/api/v1/models/vendor/curated-model/endpoints"}}, + {"id": "newlab/brand-new-model", "pricing": {"prompt": "0.0000001", "completion": "0.0000004"}, + "context_length": 131072, "supported_parameters": ["tools", "response_format"], + "architecture": {"input_modalities": ["text", "image"]}, + "links": {"details": "/api/v1/models/newlab/brand-new-model/endpoints"}}, + {"id": "other/variable-price", "pricing": {"prompt": "-1", "completion": "-1"}, "context_length": 8192}, +] + + +@pytest.fixture +def base(host_store_clean): + tp._cache.clear() + tp._openrouter_public.update(at=0.0, value=None) + host = LLMRouterHost(ROOT/'core/router.lua', ROOT/'tests/fixtures/openrouter_byo.lua', + discover=lambda _: {'ok': True, 'offers': [{'model_family': 'OPERATOR-ONLY'}]}) + host.init() + return host + + +@pytest.fixture +def openrouter(monkeypatch): + """Mock OpenRouter's public API; records every request path.""" + state = {"paths": [], "fail": False, "models": MODELS} + real = httpx.AsyncClient + + def network(request): + state["paths"].append(request.url.path + ("?" + request.url.query.decode() if request.url.query else "")) + assert request.url.host == "openrouter.ai" + assert "authorization" not in request.headers # the listing needs no tenant credential + if state["fail"]: + return httpx.Response(503) + if request.url.query: + return httpx.Response(200, json={"data": []}) + return httpx.Response(200, json={"data": state["models"]}) + + monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **k: real(transport=httpx.MockTransport(network))) + return state + + +def tenant(base, tenant_id, key="sk-or-tenant"): + return base.for_tenant(tenant_id, {"OPENROUTER_API_KEY": key} if key else {}) + + +def test_byo_openrouter_lists_live_catalog_once_with_curated_prices(base, openrouter): + child = tenant(base, 1) + asyncio.run(tp.prepare(child)) + rows = {row["id"]: row for row in choices(child)} + assert set(rows) == {"openrouter|curated-model", "openrouter_market|brand-new-model"} + assert {row["label"].split(" · ")[1] for row in rows.values()} == {"OpenRouter"} + assert (rows["openrouter|curated-model"]["price_in"], rows["openrouter|curated-model"]["price_out"]) == pytest.approx((2.0, 8.0)) + new = rows["openrouter_market|brand-new-model"] + assert (new["price_in"], new["price_out"]) == pytest.approx((0.1, 0.4)) + assert new["tools"] is True + assert child._connection_errors == {} + # Only the listing (+ decision listing): no per-model endpoint detail requests. + assert not any("/endpoints" in path for path in openrouter["paths"]) + + +def test_new_models_appear_after_the_cache_expires(base, openrouter, monkeypatch): + asyncio.run(tp.prepare(tenant(base, 1))) + openrouter["models"] = MODELS + [{"id": "lab/released-today", "context_length": 32768, + "pricing": {"prompt": "0.0000003", "completion": "0.0000009"}}] + child = tenant(base, 2) + asyncio.run(tp.prepare(child)) + assert "openrouter_market|released-today" not in {r["id"] for r in choices(child)} # within TTL: shared snapshot + monkeypatch.setattr(tp, "OPENROUTER_CATALOG_TTL_S", 0) + child = tenant(base, 2) + asyncio.run(tp.prepare(child)) + assert "openrouter_market|released-today" in {r["id"] for r in choices(child)} + + +def test_listing_is_shared_across_tenants_and_fetched_once_per_ttl(base, openrouter): + async def run(): + await asyncio.gather(*(tp.prepare(tenant(base, tid)) for tid in (1, 2, 3))) + asyncio.run(run()) + assert [p for p in openrouter["paths"] if p.endswith("/models")] == ["/api/v1/models"] + + +def test_without_openrouter_key_nothing_is_fetched_and_operator_discovery_is_not_used(base, openrouter): + child = tenant(base, 1, key=None) + asyncio.run(tp.prepare(child)) + assert openrouter["paths"] == [] + assert choices(child) == [] + + +def test_failure_reports_error_and_never_falls_back_to_operator(base, openrouter): + openrouter["fail"] = True + child = tenant(base, 1) + asyncio.run(tp.prepare(child)) + ids = {r["id"] for r in choices(child)} + assert not any("OPERATOR-ONLY" in i for i in ids) + assert not any(i.startswith("openrouter_market|") for i in ids) + assert "sk-or" not in str(child._connection_errors) + assert child._connection_errors["openrouter"] + + +def test_short_outage_serves_the_last_public_listing(base, openrouter, monkeypatch): + asyncio.run(tp.prepare(tenant(base, 1))) + monkeypatch.setattr(tp, "OPENROUTER_CATALOG_TTL_S", 0) + openrouter["fail"] = True + child = tenant(base, 2) + asyncio.run(tp.prepare(child)) + assert "openrouter_market|brand-new-model" in {r["id"] for r in choices(child)} + assert child._connection_errors == {} From c0f83c2e5d1c3380fc101afffdc6d68a75c1a132 Mon Sep 17 00:00:00 2001 From: jmlago Date: Wed, 23 Sep 2026 13:36:20 +0100 Subject: [PATCH 2/3] Name excluded preview models by their public provider MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Excluded targets were labelled with the raw provider id (openrouter_market). Use the same family · provider label as ranked rows. Co-Authored-By: Claude Opus 5.5 (1M context) --- saas_routes.py | 3 ++- tests/test_saas_openrouter_discovery.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/saas_routes.py b/saas_routes.py index bbb2bcf..65d3408 100644 --- a/saas_routes.py +++ b/saas_routes.py @@ -169,7 +169,8 @@ def preview(host, intent): explanation = "Not available through your connected provider accounts." else: explanation = "Does not meet the mandatory requirements." - exclusions.append({"id": target, "label": target.replace("|", " · "), "reason": explanation}) + pid, _, family = target.partition("|") + exclusions.append({"id": target, "label": f"{family} · {label(pid)}", "reason": explanation}) warnings = [] if len(rows) == 1: warnings.append("Only one model qualifies. There is no fallback if it fails.") diff --git a/tests/test_saas_openrouter_discovery.py b/tests/test_saas_openrouter_discovery.py index 12b2b9c..01b0a3e 100644 --- a/tests/test_saas_openrouter_discovery.py +++ b/tests/test_saas_openrouter_discovery.py @@ -118,3 +118,15 @@ def test_short_outage_serves_the_last_public_listing(base, openrouter, monkeypat asyncio.run(tp.prepare(child)) assert "openrouter_market|brand-new-model" in {r["id"] for r in choices(child)} assert child._connection_errors == {} + + + +def test_preview_ranks_long_tail_models_and_names_exclusions_by_public_provider(base, openrouter): + from saas_routes import preview + child = tenant(base, 1) + asyncio.run(tp.prepare(child)) + result = preview(child, {"goal": "cost", "workload": "agent", "timeout_seconds": 8, + "targets": ["openrouter|curated-model", "openrouter_market|brand-new-model", + "openrouter_market|retired-model"]}) + assert [row["id"] for row in result["ranked"]] == ["openrouter_market|brand-new-model", "openrouter|curated-model"] + assert [row["label"] for row in result["excluded"]] == ["retired-model · OpenRouter"] From 925af022849aee002abf3185934e40c69fa32cdd Mon Sep 17 00:00:00 2001 From: jmlago Date: Wed, 23 Sep 2026 13:51:45 +0100 Subject: [PATCH 3/3] Back off after a failed OpenRouter catalog refresh During an outage every serialized caller retried /models and could wait for another upstream timeout. For 60 s after a failure, serve the last listing (or fail fast when there is none) without calling OpenRouter. Co-Authored-By: Claude Opus 5.5 (1M context) --- tenant_providers.py | 18 +++++++++++++---- tests/test_saas_openrouter_discovery.py | 27 ++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/tenant_providers.py b/tenant_providers.py index 6fa95a2..5e2b8bc 100644 --- a/tenant_providers.py +++ b/tenant_providers.py @@ -106,7 +106,8 @@ async def _aws(host): # latency/health, and calls still authenticate with the tenant's key. OPENROUTER_CATALOG_TTL_S = 600 OPENROUTER_CATALOG_STALE_S = 3600 -_openrouter_public = {'at': 0.0, 'value': None} +OPENROUTER_CATALOG_RETRY_S = 60 # after a failed refresh, do not call OpenRouter again for this long +_openrouter_public = {'at': 0.0, 'value': None, 'failed_at': None} _openrouter_lock = None @@ -116,9 +117,17 @@ async def _openrouter_catalog(catalog): if _openrouter_lock is None or _openrouter_lock[0] is not loop: _openrouter_lock = (loop, asyncio.Lock()) async with _openrouter_lock[1]: - age = time.monotonic() - _openrouter_public['at'] + now = time.monotonic() + age = now - _openrouter_public['at'] + stale = _openrouter_public['value'] is not None and age < OPENROUTER_CATALOG_STALE_S if _openrouter_public['value'] is not None and age < OPENROUTER_CATALOG_TTL_S: return _openrouter_public['value'] + failed_at = _openrouter_public['failed_at'] + if failed_at is not None and now - failed_at < OPENROUTER_CATALOG_RETRY_S: + # Serialized callers must not each wait for another upstream timeout. + if stale: + return _openrouter_public['value'] + raise RuntimeError('OpenRouter catalog recently unavailable') from sources.openrouter import OpenRouterSource source = OpenRouterSource(catalog, env_get=lambda _key: None, route_stats=lambda: {}, endpoint_details=False) @@ -126,13 +135,14 @@ async def _openrouter_catalog(catalog): prices = await source.pricing() value = ({pid: source.offers_sync(pid) for pid in source.provider_ids if pid != 'openrouter'}, prices) except Exception: - if _openrouter_public['value'] is not None and age < OPENROUTER_CATALOG_STALE_S: + _openrouter_public['failed_at'] = time.monotonic() + if stale: return _openrouter_public['value'] # public data: a short outage keeps the last list raise finally: if source._client is not None: await source._client.aclose() - _openrouter_public.update(at=time.monotonic(), value=value) + _openrouter_public.update(at=time.monotonic(), value=value, failed_at=None) return value diff --git a/tests/test_saas_openrouter_discovery.py b/tests/test_saas_openrouter_discovery.py index 01b0a3e..888a8d1 100644 --- a/tests/test_saas_openrouter_discovery.py +++ b/tests/test_saas_openrouter_discovery.py @@ -26,7 +26,7 @@ @pytest.fixture def base(host_store_clean): tp._cache.clear() - tp._openrouter_public.update(at=0.0, value=None) + tp._openrouter_public.update(at=0.0, value=None, failed_at=None) host = LLMRouterHost(ROOT/'core/router.lua', ROOT/'tests/fixtures/openrouter_byo.lua', discover=lambda _: {'ok': True, 'offers': [{'model_family': 'OPERATOR-ONLY'}]}) host.init() @@ -130,3 +130,28 @@ def test_preview_ranks_long_tail_models_and_names_exclusions_by_public_provider( "openrouter_market|retired-model"]}) assert [row["id"] for row in result["ranked"]] == ["openrouter_market|brand-new-model", "openrouter|curated-model"] assert [row["label"] for row in result["excluded"]] == ["retired-model · OpenRouter"] + + +def test_failed_refresh_is_not_retried_by_every_caller(base, openrouter, monkeypatch): + asyncio.run(tp.prepare(tenant(base, 1))) + monkeypatch.setattr(tp, "OPENROUTER_CATALOG_TTL_S", 0) + openrouter["fail"] = True + for tid in (2, 3, 4): + child = tenant(base, tid) + asyncio.run(tp.prepare(child)) + assert "openrouter_market|brand-new-model" in {r["id"] for r in choices(child)} + # One listing, one failed refresh; later callers are served the last list without upstream calls. + assert [p for p in openrouter["paths"] if p.endswith("/models")] == ["/api/v1/models", "/api/v1/models"] + monkeypatch.setattr(tp, "OPENROUTER_CATALOG_RETRY_S", 0) + openrouter["fail"] = False + asyncio.run(tp.prepare(tenant(base, 5))) + assert tp._openrouter_public["failed_at"] is None + + +def test_first_failure_without_a_listing_fails_fast_for_the_next_caller(base, openrouter): + openrouter["fail"] = True + asyncio.run(tp.prepare(tenant(base, 1))) + child = tenant(base, 2) + asyncio.run(tp.prepare(child)) + assert child._connection_errors["openrouter"] + assert len([p for p in openrouter["paths"] if p.endswith("/models")]) == 1