diff --git a/saas_routes.py b/saas_routes.py index edd91bb..65d3408 100644 --- a/saas_routes.py +++ b/saas_routes.py @@ -69,7 +69,13 @@ def choices(host): "tools": bool((c.get("capabilities") or {}).get("supports_tools")), "price_in": finite(c.get("raw_price_in", c.get("price_in"))), "price_out": finite(c.get("raw_price_out", c.get("price_out")))} - return sorted(out.values(), key=lambda c: (c["provider"], c["family"])) + # A curated family and its marketplace twin (`openrouter` / `openrouter_market`) + # are one choice for the user: keep the curated identity, which carries the + # benchmark ranking, and drop the duplicate label. + for key, row in list(out.items()): + if row["provider"] == "openrouter_market" and f"openrouter|{row['family']}" in out: + del out[key] + return sorted(out.values(), key=lambda c: (label(c["provider"]), c["family"])) def finite(value): @@ -163,7 +169,8 @@ def preview(host, intent): explanation = "Not available through your connected provider accounts." else: explanation = "Does not meet the mandatory requirements." - exclusions.append({"id": target, "label": target.replace("|", " · "), "reason": explanation}) + pid, _, family = target.partition("|") + exclusions.append({"id": target, "label": f"{family} · {label(pid)}", "reason": explanation}) warnings = [] if len(rows) == 1: warnings.append("Only one model qualifies. There is no fallback if it fails.") diff --git a/sources/openrouter.py b/sources/openrouter.py index a43f1d7..7697863 100644 --- a/sources/openrouter.py +++ b/sources/openrouter.py @@ -28,11 +28,15 @@ class OpenRouterSource: def __init__(self, catalog: dict, env_get=os.environ.get, client: Any = None, base_url: str = BASE_URL, - route_stats=None): + route_stats=None, endpoint_details: bool = True): self._env_get = env_get self._base_url = base_url.rstrip("/") self._client = client # injected in tests; lazy httpx otherwise self._route_stats = route_stats or host_store.route_stats + # One detail request per model (~1000) is affordable on the hourly + # operator refresh, not on an on-demand SaaS catalog request. Without + # details every listed model stays routable (see _model_available). + self._endpoint_details = endpoint_details # /models snapshot cached by the async pricing() refresh so the SYNC # discover hook (offers_sync, called inside rank) never blocks on HTTP. self._models_snapshot: list[dict] = [] @@ -159,7 +163,8 @@ async def pricing(self) -> list[Price]: # cache for the sync offers_sync()/market_book() (whole-catalog discovery) merged = {m['id']: m for m in (body.get("data") or []) + self._decision_snapshot if m.get('id')} self._models_snapshot = list(merged.values()) - await self._refresh_endpoint_availability(self._models_snapshot) + if self._endpoint_details: + await self._refresh_endpoint_availability(self._models_snapshot) # Live, full model-level traits (benchmarks/modalities/caps + ranks) for # EVERY model, keyed by raw id, ranked across the whole OpenRouter # catalog. Discovered families carry these inline so they rank on real diff --git a/tenant_providers.py b/tenant_providers.py index 3113eab..5e2b8bc 100644 --- a/tenant_providers.py +++ b/tenant_providers.py @@ -100,6 +100,67 @@ async def _aws(host): await source._client.aclose() +# OpenRouter's /models listing is public and identical for every tenant: one +# shared snapshot, refreshed on demand by at most one request at a time. It is +# used only for tenants with their own OpenRouter key; offers carry no operator +# latency/health, and calls still authenticate with the tenant's key. +OPENROUTER_CATALOG_TTL_S = 600 +OPENROUTER_CATALOG_STALE_S = 3600 +OPENROUTER_CATALOG_RETRY_S = 60 # after a failed refresh, do not call OpenRouter again for this long +_openrouter_public = {'at': 0.0, 'value': None, 'failed_at': None} +_openrouter_lock = None + + +async def _openrouter_catalog(catalog): + global _openrouter_lock + loop = asyncio.get_running_loop() + if _openrouter_lock is None or _openrouter_lock[0] is not loop: + _openrouter_lock = (loop, asyncio.Lock()) + async with _openrouter_lock[1]: + now = time.monotonic() + age = now - _openrouter_public['at'] + stale = _openrouter_public['value'] is not None and age < OPENROUTER_CATALOG_STALE_S + if _openrouter_public['value'] is not None and age < OPENROUTER_CATALOG_TTL_S: + return _openrouter_public['value'] + failed_at = _openrouter_public['failed_at'] + if failed_at is not None and now - failed_at < OPENROUTER_CATALOG_RETRY_S: + # Serialized callers must not each wait for another upstream timeout. + if stale: + return _openrouter_public['value'] + raise RuntimeError('OpenRouter catalog recently unavailable') + from sources.openrouter import OpenRouterSource + source = OpenRouterSource(catalog, env_get=lambda _key: None, route_stats=lambda: {}, + endpoint_details=False) + try: + prices = await source.pricing() + value = ({pid: source.offers_sync(pid) for pid in source.provider_ids if pid != 'openrouter'}, prices) + except Exception: + _openrouter_public['failed_at'] = time.monotonic() + if stale: + return _openrouter_public['value'] # public data: a short outage keeps the last list + raise + finally: + if source._client is not None: + await source._client.aclose() + _openrouter_public.update(at=time.monotonic(), value=value, failed_at=None) + return value + + +async def _openrouter(host): + from sources import push_prices + market = [pid for pid, p in host.catalog()['providers'].items() + if p.get('discovery_id') == 'openrouter_market'] + host._tenant_offers.update({pid: [] for pid in market}) + try: + offers, prices = await asyncio.wait_for(_openrouter_catalog(host.catalog()), timeout=35) + host._tenant_offers.update({pid: offers.get(pid, []) for pid in market}) + # Curated families keep the static `openrouter` provider; this only fills + # their current prices, in this request-local engine. + push_prices(host, host.catalog(), prices) + except Exception: + host._connection_errors['openrouter'] = 'Could not load the OpenRouter model list. Try again shortly.' + + async def prepare(host): configs = getattr(host, '_tenant_connections', {}) host._connection_errors = {} @@ -133,4 +194,6 @@ async def one(name, load, ids): tasks.append(one('bedrock', _aws, ids)) if configs.get('antseed') and host._env.get('ANTSEED_BYO_URL'): tasks.append(one('antseed', _buyer, ['antseed'])) + if host._env.get('OPENROUTER_API_KEY') and 'openrouter_market' in (host._tenant_allowed or ()): + tasks.append(_openrouter(host)) await asyncio.gather(*tasks) diff --git a/tests/fixtures/openrouter_byo.lua b/tests/fixtures/openrouter_byo.lua new file mode 100644 index 0000000..f13027f --- /dev/null +++ b/tests/fixtures/openrouter_byo.lua @@ -0,0 +1,11 @@ +return { + providers = { + openrouter = {discovery='static',base_url='https://openrouter.ai/api/v1',api_kind='openai_compatible',auth_env='OPENROUTER_API_KEY',source='openrouter',tier='partner'}, + openrouter_market = {discovery='marketplace',discovery_id='openrouter_market',base_url='https://openrouter.ai/api/v1',api_kind='openai_compatible',auth_env='OPENROUTER_API_KEY',source='openrouter',tier='marketplace',market_price_cap={input=1000,output=1000}}, + }, + models = { + ['curated-model'] = {served_by={{provider='openrouter',provider_model_id='vendor/curated-model'}},capabilities={context=128000,supports_tools=true,supports_json_mode=true}}, + }, + profiles={default={scorer={'zero'}}}, + policy_envelope={'and',{'meets_req'},{'not',{'is','disabled'}}}, +} diff --git a/tests/test_saas_openrouter_discovery.py b/tests/test_saas_openrouter_discovery.py new file mode 100644 index 0000000..888a8d1 --- /dev/null +++ b/tests/test_saas_openrouter_discovery.py @@ -0,0 +1,157 @@ +"""BYO OpenRouter: the whole live catalog, discovered on demand without operator refreshers.""" +import asyncio +from pathlib import Path + +import httpx +import pytest + +from llm_router_host import LLMRouterHost +from saas_routes import choices +import tenant_providers as tp + +ROOT = Path(__file__).resolve().parents[1] + +MODELS = [ + {"id": "vendor/curated-model", "pricing": {"prompt": "0.000002", "completion": "0.000008"}, + "context_length": 200000, "supported_parameters": ["tools"], + "links": {"details": "/api/v1/models/vendor/curated-model/endpoints"}}, + {"id": "newlab/brand-new-model", "pricing": {"prompt": "0.0000001", "completion": "0.0000004"}, + "context_length": 131072, "supported_parameters": ["tools", "response_format"], + "architecture": {"input_modalities": ["text", "image"]}, + "links": {"details": "/api/v1/models/newlab/brand-new-model/endpoints"}}, + {"id": "other/variable-price", "pricing": {"prompt": "-1", "completion": "-1"}, "context_length": 8192}, +] + + +@pytest.fixture +def base(host_store_clean): + tp._cache.clear() + tp._openrouter_public.update(at=0.0, value=None, failed_at=None) + host = LLMRouterHost(ROOT/'core/router.lua', ROOT/'tests/fixtures/openrouter_byo.lua', + discover=lambda _: {'ok': True, 'offers': [{'model_family': 'OPERATOR-ONLY'}]}) + host.init() + return host + + +@pytest.fixture +def openrouter(monkeypatch): + """Mock OpenRouter's public API; records every request path.""" + state = {"paths": [], "fail": False, "models": MODELS} + real = httpx.AsyncClient + + def network(request): + state["paths"].append(request.url.path + ("?" + request.url.query.decode() if request.url.query else "")) + assert request.url.host == "openrouter.ai" + assert "authorization" not in request.headers # the listing needs no tenant credential + if state["fail"]: + return httpx.Response(503) + if request.url.query: + return httpx.Response(200, json={"data": []}) + return httpx.Response(200, json={"data": state["models"]}) + + monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **k: real(transport=httpx.MockTransport(network))) + return state + + +def tenant(base, tenant_id, key="sk-or-tenant"): + return base.for_tenant(tenant_id, {"OPENROUTER_API_KEY": key} if key else {}) + + +def test_byo_openrouter_lists_live_catalog_once_with_curated_prices(base, openrouter): + child = tenant(base, 1) + asyncio.run(tp.prepare(child)) + rows = {row["id"]: row for row in choices(child)} + assert set(rows) == {"openrouter|curated-model", "openrouter_market|brand-new-model"} + assert {row["label"].split(" · ")[1] for row in rows.values()} == {"OpenRouter"} + assert (rows["openrouter|curated-model"]["price_in"], rows["openrouter|curated-model"]["price_out"]) == pytest.approx((2.0, 8.0)) + new = rows["openrouter_market|brand-new-model"] + assert (new["price_in"], new["price_out"]) == pytest.approx((0.1, 0.4)) + assert new["tools"] is True + assert child._connection_errors == {} + # Only the listing (+ decision listing): no per-model endpoint detail requests. + assert not any("/endpoints" in path for path in openrouter["paths"]) + + +def test_new_models_appear_after_the_cache_expires(base, openrouter, monkeypatch): + asyncio.run(tp.prepare(tenant(base, 1))) + openrouter["models"] = MODELS + [{"id": "lab/released-today", "context_length": 32768, + "pricing": {"prompt": "0.0000003", "completion": "0.0000009"}}] + child = tenant(base, 2) + asyncio.run(tp.prepare(child)) + assert "openrouter_market|released-today" not in {r["id"] for r in choices(child)} # within TTL: shared snapshot + monkeypatch.setattr(tp, "OPENROUTER_CATALOG_TTL_S", 0) + child = tenant(base, 2) + asyncio.run(tp.prepare(child)) + assert "openrouter_market|released-today" in {r["id"] for r in choices(child)} + + +def test_listing_is_shared_across_tenants_and_fetched_once_per_ttl(base, openrouter): + async def run(): + await asyncio.gather(*(tp.prepare(tenant(base, tid)) for tid in (1, 2, 3))) + asyncio.run(run()) + assert [p for p in openrouter["paths"] if p.endswith("/models")] == ["/api/v1/models"] + + +def test_without_openrouter_key_nothing_is_fetched_and_operator_discovery_is_not_used(base, openrouter): + child = tenant(base, 1, key=None) + asyncio.run(tp.prepare(child)) + assert openrouter["paths"] == [] + assert choices(child) == [] + + +def test_failure_reports_error_and_never_falls_back_to_operator(base, openrouter): + openrouter["fail"] = True + child = tenant(base, 1) + asyncio.run(tp.prepare(child)) + ids = {r["id"] for r in choices(child)} + assert not any("OPERATOR-ONLY" in i for i in ids) + assert not any(i.startswith("openrouter_market|") for i in ids) + assert "sk-or" not in str(child._connection_errors) + assert child._connection_errors["openrouter"] + + +def test_short_outage_serves_the_last_public_listing(base, openrouter, monkeypatch): + asyncio.run(tp.prepare(tenant(base, 1))) + monkeypatch.setattr(tp, "OPENROUTER_CATALOG_TTL_S", 0) + openrouter["fail"] = True + child = tenant(base, 2) + asyncio.run(tp.prepare(child)) + assert "openrouter_market|brand-new-model" in {r["id"] for r in choices(child)} + assert child._connection_errors == {} + + + +def test_preview_ranks_long_tail_models_and_names_exclusions_by_public_provider(base, openrouter): + from saas_routes import preview + child = tenant(base, 1) + asyncio.run(tp.prepare(child)) + result = preview(child, {"goal": "cost", "workload": "agent", "timeout_seconds": 8, + "targets": ["openrouter|curated-model", "openrouter_market|brand-new-model", + "openrouter_market|retired-model"]}) + assert [row["id"] for row in result["ranked"]] == ["openrouter_market|brand-new-model", "openrouter|curated-model"] + assert [row["label"] for row in result["excluded"]] == ["retired-model · OpenRouter"] + + +def test_failed_refresh_is_not_retried_by_every_caller(base, openrouter, monkeypatch): + asyncio.run(tp.prepare(tenant(base, 1))) + monkeypatch.setattr(tp, "OPENROUTER_CATALOG_TTL_S", 0) + openrouter["fail"] = True + for tid in (2, 3, 4): + child = tenant(base, tid) + asyncio.run(tp.prepare(child)) + assert "openrouter_market|brand-new-model" in {r["id"] for r in choices(child)} + # One listing, one failed refresh; later callers are served the last list without upstream calls. + assert [p for p in openrouter["paths"] if p.endswith("/models")] == ["/api/v1/models", "/api/v1/models"] + monkeypatch.setattr(tp, "OPENROUTER_CATALOG_RETRY_S", 0) + openrouter["fail"] = False + asyncio.run(tp.prepare(tenant(base, 5))) + assert tp._openrouter_public["failed_at"] is None + + +def test_first_failure_without_a_listing_fails_fast_for_the_next_caller(base, openrouter): + openrouter["fail"] = True + asyncio.run(tp.prepare(tenant(base, 1))) + child = tenant(base, 2) + asyncio.run(tp.prepare(child)) + assert child._connection_errors["openrouter"] + assert len([p for p in openrouter["paths"] if p.endswith("/models")]) == 1