From d68d8e7a722a341dcaade4de8b31f78bfe5fd061 Mon Sep 17 00:00:00 2001 From: aliasunder <32620895+aliasunder@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:33:38 -0500 Subject: [PATCH 01/20] chore(openrouter): add @openrouter/agent and a temporary Responses probe The probe workflow captures the raw accepted, context-overflow and bad-key payloads from the Responses API for every slug the consumers run, so the client's new fixtures come from the wire rather than from guesses. Both probe files are removed before merge. Co-Authored-By: Claude Fable 5.1 --- .github/scripts/probe-responses.ts | 179 ++++++++++++++++++++++++++ .github/workflows/probe_responses.yml | 40 ++++++ package-lock.json | 29 +++++ package.json | 1 + 4 files changed, 249 insertions(+) create mode 100644 .github/scripts/probe-responses.ts create mode 100644 .github/workflows/probe_responses.yml diff --git a/.github/scripts/probe-responses.ts b/.github/scripts/probe-responses.ts new file mode 100644 index 00000000..716c2d61 --- /dev/null +++ b/.github/scripts/probe-responses.ts @@ -0,0 +1,179 @@ +// Temporary probe for the Agent SDK migration. It captures the raw Responses +// API payloads (accepted, context overflow, bad key) that become the client's +// fixtures, printing each HTTP body between markers from the SDK's response +// hooks. It runs once from the probe workflow on the migration branch under +// bun (which resolves the src import), then both files are removed. +import { OpenRouter } from "@openrouter/agent" +import envVar from "env-var" +import { reviewResponseJsonSchema } from "../../src/review/finding.js" + +const apiKey = envVar.from(process.env).get("OPENROUTER_KEY").required().asString() + +/** Every primary and fallback slug a consumer runs (repo variables read 2026-11-11). */ +const consumerSlugs = [ + "deepseek/deepseek-v4.1-flash:online", + "z-ai/glm-5.3-flash:online", + "deepseek/deepseek-v4.1-flash-1017:online", + "deepseek/deepseek-v4-flash-1017:online", + "deepseek/deepseek-v4-flash-0731:online", + "minimax/minimax-m3:online", + "deepseek/deepseek-v4-flash-1017", + "z-ai/glm-5.3-flash", +] + +/** The ceiling the client sends on every attempt except a fitted retry. */ +const PRODUCTION_MAX_OUTPUT_TOKENS = 128_000 + +/** The overflow target has an 8,192-token window and endpoints that cap output + * at 7,372 tokens, so a 2K prompt plus a 7,000 ceiling overflows the window + * while staying under the output cap (public endpoints API, 2026-11-11). */ +const OVERFLOW_MODEL = "sao10k/l3-lunaris-8b" +const OVERFLOW_MAX_OUTPUT_TOKENS = 7_000 + +const REQUEST_TIMEOUT_MS = 120_000 + +/** Prints one captured payload between markers the session greps for. */ +const printCapture = (label: string, payload: string): void => { + console.log(`<<>>`) + console.log(payload) + console.log(`<<>>`) +} + +/** Reads a cloned HTTP body without disturbing the SDK's own read. */ +const captureBody = async (label: string, response: Response): Promise => { + const text = await response.clone().text() + printCapture(`${label} status=${response.status}`, text) +} + +const makeClient = (key: string, label: string): OpenRouter => { + return new OpenRouter({ + apiKey: key, + hooks: [ + { + afterSuccess: async (_context: unknown, response: Response): Promise => { + await captureBody(`${label} raw`, response) + return response + }, + afterError: async ( + _context: unknown, + response: Response | null, + error: unknown, + ): Promise<{ response: Response | null; error: unknown }> => { + if (response) await captureBody(`${label} raw-error`, response) + return { response, error } + }, + }, + ], + }) +} + +const describeError = (error: unknown): string => { + if (!(error instanceof Error)) return JSON.stringify({ thrown: String(error) }) + const statusCode = "statusCode" in error ? error.statusCode : undefined + const body = + "body" in error && typeof error.body === "string" ? error.body.slice(0, 4000) : undefined + return JSON.stringify({ + name: error.name, + constructor: error.constructor.name, + statusCode, + message: error.message, + body, + }) +} + +const reviewInstructions = + "You are a code reviewer. Reply with JSON that matches the schema: an analysis string and a findings array." +const reviewInput = + 'Review this one-line change and report no findings: `export const greeting = "hello"`.' + +const runAccepted = async (model: string): Promise => { + const client = makeClient(apiKey, `accepted ${model}`) + try { + const result = client.callModel( + { + model, + instructions: reviewInstructions, + input: reviewInput, + maxOutputTokens: PRODUCTION_MAX_OUTPUT_TOKENS, + text: { + format: { + type: "json_schema", + name: "review_response", + strict: true, + schema: reviewResponseJsonSchema, + }, + }, + }, + { retries: { strategy: "none" }, timeoutMs: REQUEST_TIMEOUT_MS }, + ) + const response = await result.getResponse() + printCapture(`accepted ${model} parsed`, JSON.stringify(response)) + printCapture(`accepted ${model} text`, await result.getText()) + } catch (error) { + printCapture(`accepted ${model} threw`, describeError(error)) + } +} + +const runOverflow = async (): Promise => { + const client = makeClient(apiKey, "overflow") + const filler = "The quick brown fox jumps over the lazy dog. ".repeat(400) + try { + const result = client.callModel( + { + model: OVERFLOW_MODEL, + instructions: reviewInstructions, + input: `${filler}\n${reviewInput}`, + maxOutputTokens: OVERFLOW_MAX_OUTPUT_TOKENS, + // Endpoints with an 8K window compress the prompt by default, which + // would hide the overflow; the probe needs the rejection itself + plugins: [{ id: "context-compression", enabled: false }], + text: { + format: { + type: "json_schema", + name: "review_response", + strict: true, + schema: reviewResponseJsonSchema, + }, + }, + }, + { retries: { strategy: "none" }, timeoutMs: REQUEST_TIMEOUT_MS }, + ) + const response = await result.getResponse() + printCapture("overflow returned", JSON.stringify(response)) + } catch (error) { + printCapture("overflow threw", describeError(error)) + } +} + +const runBadKey = async (): Promise => { + const client = makeClient("sk-or-v1-invalid-probe-key", "bad-key") + try { + const result = client.callModel( + { + model: "deepseek/deepseek-v4.1-flash", + instructions: reviewInstructions, + input: reviewInput, + maxOutputTokens: PRODUCTION_MAX_OUTPUT_TOKENS, + text: { + format: { + type: "json_schema", + name: "review_response", + strict: true, + schema: reviewResponseJsonSchema, + }, + }, + }, + { retries: { strategy: "none" }, timeoutMs: REQUEST_TIMEOUT_MS }, + ) + const response = await result.getResponse() + printCapture("bad-key returned", JSON.stringify(response)) + } catch (error) { + printCapture("bad-key threw", describeError(error)) + } +} + +for (const slug of consumerSlugs) { + await runAccepted(slug) +} +await runOverflow() +await runBadKey() diff --git a/.github/workflows/probe_responses.yml b/.github/workflows/probe_responses.yml new file mode 100644 index 00000000..5fa10ed8 --- /dev/null +++ b/.github/workflows/probe_responses.yml @@ -0,0 +1,40 @@ +# Temporary workflow for the Agent SDK migration. It captures raw Responses +# API payloads for the client's fixtures and is removed before the PR merges. +name: Probe Responses + +on: + pull_request: + types: [opened, synchronize, reopened] + +permissions: + contents: read + +jobs: + probe: + runs-on: ubuntu-latest + # The probe sends real requests with the repo's key, so a single run is + # enough to capture every payload + timeout-minutes: 10 + # Only the owner's same-repo PRs run secrets-bearing jobs + if: >- + github.event.pull_request.user.login == github.repository_owner && + github.event.pull_request.head.repo.full_name == github.repository + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version-file: .nvmrc + cache: npm + + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + + - name: Install dependencies + run: npm ci + + - name: Capture Responses payloads + env: + OPENROUTER_KEY: ${{ secrets.OPENROUTER_KEY }} + run: bun .github/scripts/probe-responses.ts diff --git a/package-lock.json b/package-lock.json index 1f379b50..8b3f3634 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "dependencies": { "@actions/core": "3.0.1", "@actions/github": "9.1.1", + "@openrouter/agent": "1.1.0", "@openrouter/sdk": "1.5.33", "env-var": "7.5.0", "luxon": "3.7.2", @@ -502,6 +503,34 @@ "@octokit/openapi-types": "^27.0.0" } }, + "node_modules/@openrouter/agent": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@openrouter/agent/-/agent-1.1.0.tgz", + "integrity": "sha512-oIBn4AxeHZIUVYYWf2wPRNximYovlD6nnuU9afKhojAmcs8YKVUVs4NFwQh1dhnEO0iVUg7VmW94eT4Y9u0dfg==", + "license": "Apache-2.0", + "dependencies": { + "@openrouter/sdk": "^0.13.7", + "zod": "^4.0.0" + }, + "peerDependencies": { + "@modelcontextprotocol/client": "^2.0.0" + }, + "peerDependenciesMeta": { + "@modelcontextprotocol/client": { + "optional": true + } + } + }, + "node_modules/@openrouter/agent/node_modules/@openrouter/sdk": { + "version": "0.13.67", + "resolved": "https://registry.npmjs.org/@openrouter/sdk/-/sdk-0.13.67.tgz", + "integrity": "sha512-kUVaFYA9Xehq67ndTk98i+tRLfYH/F6xSf/WuC8dsukBzcgBvYQydoeKw4Xe+peXG/wOVtDxukWt5SqzhbloKw==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "zod": "^3.25.0 || ^4.0.0" + } + }, "node_modules/@openrouter/sdk": { "version": "1.5.33", "resolved": "https://registry.npmjs.org/@openrouter/sdk/-/sdk-1.5.33.tgz", diff --git a/package.json b/package.json index dbc4701d..c24edf9a 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "dependencies": { "@actions/core": "3.0.1", "@actions/github": "9.1.1", + "@openrouter/agent": "1.1.0", "@openrouter/sdk": "1.5.33", "env-var": "7.5.0", "luxon": "3.7.2", From a3d47276b914da584d65a909a29ec3e6624504b7 Mon Sep 17 00:00:00 2001 From: aliasunder <32620895+aliasunder@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:48:23 -0500 Subject: [PATCH 02/20] refactor(openrouter): send review requests through the Agent SDK The review client calls callModel on @openrouter/agent, one streaming Responses API request per attempt with structured output through text.format, and reads the text the SDK extracts. The retry ladder keeps its policy; @openrouter/sdk stays for the generation cost lookup. Tests run on raw Responses payloads captured from live requests, transformed by the SDK's own public helper, and build the SDK's error classes from captured error bodies. Co-Authored-By: Claude Fable 5.1 --- fixtures/openrouter.chat-result.json | 15 - fixtures/openrouter.invalid-json.json | 15 - fixtures/openrouter.responses-accepted.json | 168 +++ .../openrouter.responses-bad-key-body.json | 6 + .../openrouter.responses-invalid-json.json | 168 +++ .../openrouter.responses-overflow-body.json | 9 + .../openrouter.responses-schema-mismatch.json | 168 +++ fixtures/openrouter.schema-mismatch.json | 15 - src/__tests__/orchestrate.test.ts | 86 +- src/main.ts | 2 + src/openrouter/__tests__/client.test.ts | 1039 ++++++++++------- src/openrouter/client.ts | 132 ++- 12 files changed, 1245 insertions(+), 578 deletions(-) delete mode 100644 fixtures/openrouter.chat-result.json delete mode 100644 fixtures/openrouter.invalid-json.json create mode 100644 fixtures/openrouter.responses-accepted.json create mode 100644 fixtures/openrouter.responses-bad-key-body.json create mode 100644 fixtures/openrouter.responses-invalid-json.json create mode 100644 fixtures/openrouter.responses-overflow-body.json create mode 100644 fixtures/openrouter.responses-schema-mismatch.json delete mode 100644 fixtures/openrouter.schema-mismatch.json diff --git a/fixtures/openrouter.chat-result.json b/fixtures/openrouter.chat-result.json deleted file mode 100644 index e8c005e4..00000000 --- a/fixtures/openrouter.chat-result.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "id": "gen-fixture-0001", - "object": "chat.completion", - "model": "openai/gpt-5-mini", - "choices": [ - { - "finishReason": "stop", - "message": { - "role": "assistant", - "content": "{\"analysis\":\"src/greeter.ts: verified the empty-key guard and traced register() callers.\",\"findings\":[]}" - } - } - ], - "usage": { "promptTokens": 12000, "completionTokens": 800, "cost": 0.0421 } -} diff --git a/fixtures/openrouter.invalid-json.json b/fixtures/openrouter.invalid-json.json deleted file mode 100644 index 4f617a61..00000000 --- a/fixtures/openrouter.invalid-json.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "id": "gen-fixture-0002", - "object": "chat.completion", - "model": "openai/gpt-5-mini", - "choices": [ - { - "finishReason": "length", - "message": { - "role": "assistant", - "content": "The model rambled instead of emitting JSON {" - } - } - ], - "usage": { "promptTokens": 11800, "completionTokens": 42, "cost": null } -} diff --git a/fixtures/openrouter.responses-accepted.json b/fixtures/openrouter.responses-accepted.json new file mode 100644 index 00000000..e93ce3a4 --- /dev/null +++ b/fixtures/openrouter.responses-accepted.json @@ -0,0 +1,168 @@ +{ + "id": "gen-1794440085-n7DSIWy7fBrBUcqMEvla", + "object": "response", + "created_at": 1794440085, + "model": "z-ai/glm-5.3-flash", + "status": "completed", + "completed_at": 1794440086, + "output": [ + { + "id": "rs_tmp_viau8k9rg1", + "type": "reasoning", + "status": "completed", + "content": [ + { + "type": "reasoning_text", + "text": "User asks to report no findings. Provide JSON with analysis and empty findings array." + } + ], + "summary": [], + "format": "unknown" + }, + { + "id": "msg_tmp_o67rrerum0m", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": "{\n \"analysis\": \"Reviewed the single-line change `export const greeting = \\\"hello\\\"`. This is a simple, valid module-level constant export. The syntax is correct, the value is a static string with no side effects, and naming follows common conventions. No security, correctness, style, or maintainability concerns were identified. As requested, there are no findings to report.\",\n \"findings\": []\n}", + "annotations": [], + "logprobs": [] + } + ] + } + ], + "error": null, + "incomplete_details": null, + "access_programs": null, + "tools": [], + "tool_choice": "auto", + "parallel_tool_calls": true, + "max_output_tokens": 128000, + "temperature": 1, + "top_p": 1, + "presence_penalty": 0, + "frequency_penalty": 0, + "top_logprobs": 0, + "max_tool_calls": null, + "metadata": {}, + "background": false, + "previous_response_id": null, + "service_tier": "auto", + "truncation": "disabled", + "store": false, + "instructions": "You are a code reviewer. Reply with JSON that matches the schema: an analysis string and a findings array.", + "text": { + "format": { + "type": "json_schema", + "name": "review_response", + "strict": true, + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "analysis": { + "type": "string" + }, + "findings": { + "type": "array", + "items": { + "type": "object", + "properties": { + "file": { + "type": "string", + "minLength": 1 + }, + "line": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "end_line": { + "anyOf": [ + { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + { + "type": "null" + } + ] + }, + "category": { + "type": "string", + "enum": ["correctness", "security", "conventions", "tests", "subtle_bugs", "ci"] + }, + "severity": { + "type": "string", + "enum": ["critical", "high", "medium", "low"] + }, + "confidence": { + "type": "string", + "enum": ["high", "medium", "low"] + }, + "title": { + "type": "string", + "minLength": 1 + }, + "description": { + "type": "string", + "minLength": 1 + }, + "suggestion": { + "type": ["string", "null"] + }, + "failure_scenario": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "file", + "line", + "end_line", + "category", + "severity", + "confidence", + "title", + "description", + "suggestion", + "failure_scenario" + ], + "additionalProperties": false + } + } + }, + "required": ["analysis", "findings"], + "additionalProperties": false + } + } + }, + "reasoning": { + "effort": "max" + }, + "safety_identifier": null, + "prompt_cache_key": null, + "prompt_cache_options": null, + "usage": { + "input_tokens": 54, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 102, + "output_tokens_details": { + "reasoning_tokens": 17 + }, + "total_tokens": 156, + "cost": 0.000021036, + "is_byok": false, + "cost_details": { + "upstream_inference_cost": 0.000021036, + "upstream_inference_input_cost": 0.000002268, + "upstream_inference_output_cost": 0.000018768 + } + } +} diff --git a/fixtures/openrouter.responses-bad-key-body.json b/fixtures/openrouter.responses-bad-key-body.json new file mode 100644 index 00000000..9e0cd438 --- /dev/null +++ b/fixtures/openrouter.responses-bad-key-body.json @@ -0,0 +1,6 @@ +{ + "error": { + "message": "User not found.", + "code": 401 + } +} diff --git a/fixtures/openrouter.responses-invalid-json.json b/fixtures/openrouter.responses-invalid-json.json new file mode 100644 index 00000000..5f8e59ec --- /dev/null +++ b/fixtures/openrouter.responses-invalid-json.json @@ -0,0 +1,168 @@ +{ + "id": "gen-1794440085-n7DSIWy7fBrBUcqMEvla", + "object": "response", + "created_at": 1794440085, + "model": "z-ai/glm-5.3-flash", + "status": "completed", + "completed_at": 1794440086, + "output": [ + { + "id": "rs_tmp_viau8k9rg1", + "type": "reasoning", + "status": "completed", + "content": [ + { + "type": "reasoning_text", + "text": "User asks to report no findings. Provide JSON with analysis and empty findings array." + } + ], + "summary": [], + "format": "unknown" + }, + { + "id": "msg_tmp_o67rrerum0m", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": "not json", + "annotations": [], + "logprobs": [] + } + ] + } + ], + "error": null, + "incomplete_details": null, + "access_programs": null, + "tools": [], + "tool_choice": "auto", + "parallel_tool_calls": true, + "max_output_tokens": 128000, + "temperature": 1, + "top_p": 1, + "presence_penalty": 0, + "frequency_penalty": 0, + "top_logprobs": 0, + "max_tool_calls": null, + "metadata": {}, + "background": false, + "previous_response_id": null, + "service_tier": "auto", + "truncation": "disabled", + "store": false, + "instructions": "You are a code reviewer. Reply with JSON that matches the schema: an analysis string and a findings array.", + "text": { + "format": { + "type": "json_schema", + "name": "review_response", + "strict": true, + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "analysis": { + "type": "string" + }, + "findings": { + "type": "array", + "items": { + "type": "object", + "properties": { + "file": { + "type": "string", + "minLength": 1 + }, + "line": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "end_line": { + "anyOf": [ + { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + { + "type": "null" + } + ] + }, + "category": { + "type": "string", + "enum": ["correctness", "security", "conventions", "tests", "subtle_bugs", "ci"] + }, + "severity": { + "type": "string", + "enum": ["critical", "high", "medium", "low"] + }, + "confidence": { + "type": "string", + "enum": ["high", "medium", "low"] + }, + "title": { + "type": "string", + "minLength": 1 + }, + "description": { + "type": "string", + "minLength": 1 + }, + "suggestion": { + "type": ["string", "null"] + }, + "failure_scenario": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "file", + "line", + "end_line", + "category", + "severity", + "confidence", + "title", + "description", + "suggestion", + "failure_scenario" + ], + "additionalProperties": false + } + } + }, + "required": ["analysis", "findings"], + "additionalProperties": false + } + } + }, + "reasoning": { + "effort": "max" + }, + "safety_identifier": null, + "prompt_cache_key": null, + "prompt_cache_options": null, + "usage": { + "input_tokens": 54, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 102, + "output_tokens_details": { + "reasoning_tokens": 17 + }, + "total_tokens": 156, + "cost": 0.000021036, + "is_byok": false, + "cost_details": { + "upstream_inference_cost": 0.000021036, + "upstream_inference_input_cost": 0.000002268, + "upstream_inference_output_cost": 0.000018768 + } + } +} diff --git a/fixtures/openrouter.responses-overflow-body.json b/fixtures/openrouter.responses-overflow-body.json new file mode 100644 index 00000000..8eb6e77e --- /dev/null +++ b/fixtures/openrouter.responses-overflow-body.json @@ -0,0 +1,9 @@ +{ + "error": { + "message": "This endpoint's maximum context length is 8192 tokens. However, you requested about 11593 tokens (4593 of text input, 7000 in the output). Please reduce the length of either one, or use the context-compression plugin to compress your prompt automatically.", + "code": 400, + "metadata": { + "provider_name": null + } + } +} diff --git a/fixtures/openrouter.responses-schema-mismatch.json b/fixtures/openrouter.responses-schema-mismatch.json new file mode 100644 index 00000000..4ea12472 --- /dev/null +++ b/fixtures/openrouter.responses-schema-mismatch.json @@ -0,0 +1,168 @@ +{ + "id": "gen-1794440085-n7DSIWy7fBrBUcqMEvla", + "object": "response", + "created_at": 1794440085, + "model": "z-ai/glm-5.3-flash", + "status": "completed", + "completed_at": 1794440086, + "output": [ + { + "id": "rs_tmp_viau8k9rg1", + "type": "reasoning", + "status": "completed", + "content": [ + { + "type": "reasoning_text", + "text": "User asks to report no findings. Provide JSON with analysis and empty findings array." + } + ], + "summary": [], + "format": "unknown" + }, + { + "id": "msg_tmp_o67rrerum0m", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": "{\"verdict\":\"lgtm\"}", + "annotations": [], + "logprobs": [] + } + ] + } + ], + "error": null, + "incomplete_details": null, + "access_programs": null, + "tools": [], + "tool_choice": "auto", + "parallel_tool_calls": true, + "max_output_tokens": 128000, + "temperature": 1, + "top_p": 1, + "presence_penalty": 0, + "frequency_penalty": 0, + "top_logprobs": 0, + "max_tool_calls": null, + "metadata": {}, + "background": false, + "previous_response_id": null, + "service_tier": "auto", + "truncation": "disabled", + "store": false, + "instructions": "You are a code reviewer. Reply with JSON that matches the schema: an analysis string and a findings array.", + "text": { + "format": { + "type": "json_schema", + "name": "review_response", + "strict": true, + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "analysis": { + "type": "string" + }, + "findings": { + "type": "array", + "items": { + "type": "object", + "properties": { + "file": { + "type": "string", + "minLength": 1 + }, + "line": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "end_line": { + "anyOf": [ + { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + { + "type": "null" + } + ] + }, + "category": { + "type": "string", + "enum": ["correctness", "security", "conventions", "tests", "subtle_bugs", "ci"] + }, + "severity": { + "type": "string", + "enum": ["critical", "high", "medium", "low"] + }, + "confidence": { + "type": "string", + "enum": ["high", "medium", "low"] + }, + "title": { + "type": "string", + "minLength": 1 + }, + "description": { + "type": "string", + "minLength": 1 + }, + "suggestion": { + "type": ["string", "null"] + }, + "failure_scenario": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "file", + "line", + "end_line", + "category", + "severity", + "confidence", + "title", + "description", + "suggestion", + "failure_scenario" + ], + "additionalProperties": false + } + } + }, + "required": ["analysis", "findings"], + "additionalProperties": false + } + } + }, + "reasoning": { + "effort": "max" + }, + "safety_identifier": null, + "prompt_cache_key": null, + "prompt_cache_options": null, + "usage": { + "input_tokens": 54, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 102, + "output_tokens_details": { + "reasoning_tokens": 17 + }, + "total_tokens": 156, + "cost": 0.000021036, + "is_byok": false, + "cost_details": { + "upstream_inference_cost": 0.000021036, + "upstream_inference_input_cost": 0.000002268, + "upstream_inference_output_cost": 0.000018768 + } + } +} diff --git a/fixtures/openrouter.schema-mismatch.json b/fixtures/openrouter.schema-mismatch.json deleted file mode 100644 index 98591dfc..00000000 --- a/fixtures/openrouter.schema-mismatch.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "id": "gen-fixture-0003", - "object": "chat.completion", - "model": "openai/gpt-5-mini", - "choices": [ - { - "finishReason": "stop", - "message": { - "role": "assistant", - "content": "{\"verdict\":\"lgtm\"}" - } - } - ], - "usage": { "promptTokens": 11900, "completionTokens": 12, "cost": 0.001 } -} diff --git a/src/__tests__/orchestrate.test.ts b/src/__tests__/orchestrate.test.ts index cae33f8f..e0b8eea2 100644 --- a/src/__tests__/orchestrate.test.ts +++ b/src/__tests__/orchestrate.test.ts @@ -8,9 +8,9 @@ import type { ContextReader } from "../context/workspace.js" import { ReviewRequestError, createOpenRouterClient, - type ChatRequestSubset, type ModelAttempt, type OpenRouterClient, + type ReviewRequest, type StructuredReviewResult, } from "../openrouter/client.js" import { buildUserPrompt, estimateTokens, type PromptFile } from "../review/prompt.js" @@ -61,6 +61,31 @@ const fixtureReviewResponse: ReviewResponse = JSON.parse( readFileSync(new URL("../../fixtures/openrouter.response.json", import.meta.url), "utf8"), ) +/** The pair a `callModel` stub hands back once its request settles: the + * materialized response the client parses and the text the SDK extracts. */ +type ModelCallResult = { response: unknown; text: string } + +/** A `callModel` result that settles with the given promise. Both reads await + * the one promise, the way the SDK's reads share one materialized stream. */ +const makePendingModelResultStub = ( + settled: Promise, +): { getResponse(): Promise; getText(): Promise } => ({ + getResponse: async () => (await settled).response, + getText: async () => (await settled).text, +}) + +/** A `callModel` result that resolves at once with an accepted-shaped response + * carrying the fixture review as its text. */ +const makeModelResultStub = (response: { + id: string + model: string + usage: { inputTokens: number; outputTokens: number; cost?: number } +}): { getResponse(): Promise; getText(): Promise } => { + return makePendingModelResultStub( + Promise.resolve({ response, text: JSON.stringify(fixtureReviewResponse) }), + ) +} + const fixturePrContext: PrContext = { prNumber: 7, title: "feat: trim names before greeting", @@ -4691,14 +4716,16 @@ describe("staged phases", () => { const remainingReviewMs = () => deadline - Date.now() const costResponse = Promise.withResolvers() const getGeneration = vi.fn(() => costResponse.promise) - const send = vi.fn(async () => ({ - id: "accepted-before-deadline", - model: "test/model", - choices: [{ message: { content: JSON.stringify(fixtureReviewResponse) } }], - usage: { promptTokens: 10, completionTokens: 20 }, - })) + const callModel = vi.fn(() => { + return makeModelResultStub({ + id: "accepted-before-deadline", + model: "test/model", + usage: { inputTokens: 10, outputTokens: 20 }, + }) + }) const client = createOpenRouterClient({ - sdk: { chat: { send }, generations: { getGeneration } }, + agent: { callModel }, + sdk: { generations: { getGeneration } }, requestTimeoutMs: 900_000, remainingReviewMs, }) @@ -4731,7 +4758,7 @@ describe("staged phases", () => { modelUsed: "test/model", }) expect(remainingReviewMs()).toBe(0) - expect(send).toHaveBeenCalledTimes(1) + expect(callModel).toHaveBeenCalledTimes(1) expect(getGeneration).toHaveBeenCalledExactlyOnceWith( { id: "accepted-before-deadline" }, { retries: { strategy: "none" }, signal: expect.any(AbortSignal) }, @@ -4776,20 +4803,20 @@ describe("staged phases", () => { try { const deadline = Date.now() + 1000 const remainingReviewMs = () => deadline - Date.now() - const lateResponse = Promise.withResolvers() + const lateResponse = Promise.withResolvers() const response = { id: "completed", model: "test/model", - choices: [{ message: { content: JSON.stringify(fixtureReviewResponse) } }], - usage: { promptTokens: 10, completionTokens: 20, cost: 0.01 }, + usage: { inputTokens: 10, outputTokens: 20, cost: 0.01 }, } - const send = vi + const callModel = vi .fn() - .mockResolvedValueOnce(response) - .mockResolvedValueOnce(response) - .mockImplementation(() => lateResponse.promise) + .mockReturnValueOnce(makeModelResultStub(response)) + .mockReturnValueOnce(makeModelResultStub(response)) + .mockImplementation(() => makePendingModelResultStub(lateResponse.promise)) const client = createOpenRouterClient({ - sdk: { chat: { send }, generations: { getGeneration: vi.fn() } }, + agent: { callModel }, + sdk: { generations: { getGeneration: vi.fn() } }, requestTimeoutMs: 900_000, remainingReviewMs, }) @@ -4808,7 +4835,7 @@ describe("staged phases", () => { const pending = orchestrate(stubs.deps, createTestLogger()) await vi.advanceTimersByTimeAsync(1000) const result = await pending - expect(send).toHaveBeenCalledTimes(3) + expect(callModel).toHaveBeenCalledTimes(3) expect(result.phases).toEqual([ { phase: "correctness-security", status: "completed" }, { phase: "conventions-tests", status: "completed" }, @@ -4858,7 +4885,7 @@ describe("staged phases", () => { reviews: stubs.postFindingsReviewCalls, result, }) - lateResponse.resolve(response) + lateResponse.resolve({ response, text: JSON.stringify(fixtureReviewResponse) }) await vi.runAllTimersAsync() expect({ checks: stubs.updateCheckRunCalls, @@ -4875,8 +4902,7 @@ describe("staged phases", () => { const response = { id: "completed", model: "test/model", - choices: [{ message: { content: JSON.stringify(fixtureReviewResponse) } }], - usage: { promptTokens: 10, completionTokens: 20, cost: 0.01 }, + usage: { inputTokens: 10, outputTokens: 20, cost: 0.01 }, } // correctness-security is dispatched first, so a logger shared across the @@ -4884,19 +4910,21 @@ describe("staged phases", () => { // The failure is keyed on that phase's prompt text, not on call order const correctnessSecurityPassScope = "this pass covers correctness & security" const failedPrompts = new Set() - const send = vi.fn(async ({ chatRequest }: { chatRequest: ChatRequestSubset }) => { - const systemPrompt = first(chatRequest.messages).content + const callModel = vi.fn(({ instructions }: ReviewRequest) => { const isFirstCorrectnessSecurityRequest = - systemPrompt.includes(correctnessSecurityPassScope) && !failedPrompts.has(systemPrompt) + instructions.includes(correctnessSecurityPassScope) && !failedPrompts.has(instructions) if (isFirstCorrectnessSecurityRequest) { - failedPrompts.add(systemPrompt) - throw Object.assign(new Error("HTTP 500"), { statusCode: 500 }) + failedPrompts.add(instructions) + return makePendingModelResultStub( + Promise.reject(Object.assign(new Error("HTTP 500"), { statusCode: 500 })), + ) } - return response + return makeModelResultStub(response) }) const client = createOpenRouterClient({ - sdk: { chat: { send }, generations: { getGeneration: vi.fn() } }, + agent: { callModel }, + sdk: { generations: { getGeneration: vi.fn() } }, requestTimeoutMs: 900_000, remainingReviewMs: () => Infinity, retryDelayMs: 0, @@ -4934,7 +4962,7 @@ describe("staged phases", () => { { phase: "conventions-tests", status: "completed" }, { phase: "subtle-bugs", status: "completed" }, ]) - expect(send).toHaveBeenCalledTimes(4) + expect(callModel).toHaveBeenCalledTimes(4) expect(logsWithMessage(generateLogger, "review attempt failed")).toEqual([ { level: "warn", diff --git a/src/main.ts b/src/main.ts index 8cdb469d..6030de66 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,5 +1,6 @@ import * as core from "@actions/core" import { context, getOctokit } from "@actions/github" +import { OpenRouter as OpenRouterAgent } from "@openrouter/agent" import { OpenRouter } from "@openrouter/sdk" import envVar from "env-var" import { parseConfig, type RawInputs } from "./config.js" @@ -136,6 +137,7 @@ try { // createOpenRouterClient takes no logger because each review phase // passes its own phase-tagged logger to requestReview openrouterClient: createOpenRouterClient({ + agent: new OpenRouterAgent({ apiKey: config.openrouterApiKey }), sdk: new OpenRouter({ apiKey: config.openrouterApiKey }), requestTimeoutMs: config.requestTimeoutSeconds * 1000, remainingReviewMs, diff --git a/src/openrouter/__tests__/client.test.ts b/src/openrouter/__tests__/client.test.ts index 7e61c5b8..de687167 100644 --- a/src/openrouter/__tests__/client.test.ts +++ b/src/openrouter/__tests__/client.test.ts @@ -1,112 +1,196 @@ import { readFileSync } from "node:fs" +import { openResponsesResultFromJSON } from "@openrouter/sdk/models" +import { + BadRequestResponseError, + OpenRouterError, + UnauthorizedResponseError, +} from "@openrouter/sdk/models/errors" import { describe, expect, it, vi } from "vitest" +import { z } from "zod" import { createTestLogger, logsWithMessage } from "../../__tests__/test-logger.js" import { reviewResponseJsonSchema } from "../../review/finding.js" import { createOpenRouterClient, ReviewRequestError, - type ChatRequestSubset, - type OpenRouterLike, + type AgentLike, + type GenerationsLike, + type ReviewRequest, } from "../client.js" const readJsonFixture = (name: string): unknown => { return JSON.parse(readFileSync(new URL(`../../../fixtures/${name}`, import.meta.url), "utf8")) } -const acceptedChatResult = readJsonFixture("openrouter.chat-result.json") -const invalidJsonChatResult = readJsonFixture("openrouter.invalid-json.json") -const schemaMismatchChatResult = readJsonFixture("openrouter.schema-mismatch.json") +// Raw Responses API payloads captured from live requests. The two variants +// are the accepted payload with only its message text edited +const acceptedWirePayload = readJsonFixture("openrouter.responses-accepted.json") +const invalidJsonWirePayload = readJsonFixture("openrouter.responses-invalid-json.json") +const schemaMismatchWirePayload = readJsonFixture("openrouter.responses-schema-mismatch.json") +const overflowErrorBody = readJsonFixture("openrouter.responses-overflow-body.json") +const badKeyErrorBody = readJsonFixture("openrouter.responses-bad-key-body.json") + +/** The generation id and usage the accepted wire payload carries. */ +const ACCEPTED_GENERATION_ID = "gen-1794440085-n7DSIWy7fBrBUcqMEvla" +const ACCEPTED_USAGE = { promptTokens: 54, completionTokens: 102, costUsd: 0.000021036 } -/** The review JSON embedded in the accepted chat-result fixture, parsed. */ +/** The review JSON in the accepted wire payload's message text, parsed. */ const acceptedReview = { - analysis: "src/greeter.ts: verified the empty-key guard and traced register() callers.", + analysis: + 'Reviewed the single-line change `export const greeting = "hello"`. This is a simple, valid module-level constant export. The syntax is correct, the value is a static string with no side effects, and naming follows common conventions. No security, correctness, style, or maintainability concerns were identified. As requested, there are no findings to report.', findings: [], } -/** Accepted-shape chat result whose usage carries no cost — the case that - * routes the client to the generation cost lookup. */ -const makeNoCostChatResult = (): unknown => ({ - id: "gen-no-cost", - model: "openai/gpt-5-mini", - choices: [{ message: { content: JSON.stringify(acceptedReview) } }], - usage: { promptTokens: 12000, completionTokens: 800, cost: null }, +/** The output items of a wire payload; every other field passes through. */ +const wirePayloadSchema = z.looseObject({ + output: z.array( + z.looseObject({ + type: z.string(), + content: z.array(z.looseObject({ type: z.string(), text: z.string() })).optional(), + }), + ), }) -type StubOptions = { signal?: AbortSignal } | undefined +/** The accepted payload also carries a priced usage block the variants edit. */ +const acceptedPayloadSchema = wirePayloadSchema.extend({ + usage: z.looseObject({ cost: z.number() }), +}) + +const parseAcceptedPayload = (): z.infer => { + return acceptedPayloadSchema.parse(acceptedWirePayload) +} + +/** The text the Agent SDK extracts from a payload: the first message item's + * output text, or "" when the output holds no message item. */ +const extractMessageText = (payload: unknown): string => { + const message = wirePayloadSchema.parse(payload).output.find((item) => item.type === "message") + const outputText = message?.content?.find((part) => part.type === "output_text") + + if (!outputText) return "" + return outputText.text +} + +/** What the Agent SDK hands the client for a wire payload: the payload run + * through the SDK's own transform. The repo's @openrouter/sdk copy runs it + * here as a drift guard on hand-edited fixtures; the Agent SDK's bundled + * copy parses every field the client reads identically. */ +const transformWirePayload = (payload: unknown): unknown => { + const transformed = openResponsesResultFromJSON(JSON.stringify(payload)) + + if (!transformed.ok) throw transformed.error + return transformed.value +} + +/** What one `callModel` stub call resolves: the transformed response and the + * SDK-extracted text, as the client reads them. */ +type ModelCallResult = { response: unknown; text: string } + +const toModelCallResult = (payload: unknown): ModelCallResult => ({ + response: transformWirePayload(payload), + text: extractMessageText(payload), +}) -/** `pending` hands the call's options to a custom promise — for a request +const acceptedResult = toModelCallResult(acceptedWirePayload) +const invalidJsonResult = toModelCallResult(invalidJsonWirePayload) +const schemaMismatchResult = toModelCallResult(schemaMismatchWirePayload) + +/** The accepted payload with its usage cost removed — the case that routes + * the client to the generation cost lookup. */ +const makeNoCostResult = (): ModelCallResult => { + const { usage, ...payload } = parseAcceptedPayload() + const usageWithoutCost = Object.fromEntries( + Object.entries(usage).filter(([key]) => key !== "cost"), + ) + return toModelCallResult({ ...payload, usage: usageWithoutCost }) +} + +/** The accepted payload with no usage block at all. */ +const makeNoUsageResult = (): ModelCallResult => { + return toModelCallResult({ ...parseAcceptedPayload(), usage: null }) +} + +/** The accepted payload with only its reasoning item, so the SDK extracts no text. */ +const makeReasoningOnlyResult = (): ModelCallResult => { + const payload = parseAcceptedPayload() + const reasoningItems = payload.output.filter((item) => item.type === "reasoning") + return toModelCallResult({ ...payload, output: reasoningItems }) +} + +/** `pending` hands the call's request to a custom promise — for a request * that never settles, settles late, or honours the abort signal. */ -type StubResponse = +type StubModelResult = + | { value: ModelCallResult } + | { error: unknown } + | { pending: (request: { signal: AbortSignal }) => Promise } + +type StubOptions = { signal?: AbortSignal } | undefined + +type StubGenerationResponse = { value: unknown } | { error: unknown } | { pending: (options: StubOptions) => Promise } -const makeSdkStub = ({ - sendResponses, +const makeClientStubs = ({ + modelResults, generationResponses = [], }: { - sendResponses: StubResponse[] - generationResponses?: StubResponse[] + modelResults: StubModelResult[] + generationResponses?: StubGenerationResponse[] }) => { - const sendCalls: { - chatRequest: ChatRequestSubset - options: StubOptions + const modelCalls: { + request: ReviewRequest & { signal: AbortSignal } + options: { retries?: { strategy: "backoff" | "none" } } | undefined }[] = [] const generationCalls: { id: string options: StubOptions }[] = [] - const takeNext = ({ - queue, - callCount, - operation, - options, - }: { - queue: StubResponse[] - callCount: number - operation: string - options: StubOptions - }): unknown => { - const next = queue[callCount - 1] + const takeNextModelResult = (request: { signal: AbortSignal }): Promise => { + const next = modelResults.at(modelCalls.length - 1) if (!next) { - throw new Error(`stub: unexpected ${operation} call #${callCount}`) + throw new Error(`stub: unexpected callModel call #${modelCalls.length}`) } if ("error" in next) throw next.error - if ("pending" in next) return next.pending(options) - return next.value + if ("pending" in next) return next.pending(request) + return Promise.resolve(next.value) } - const sdk: OpenRouterLike = { - chat: { - send: async (request, options) => { - sendCalls.push({ ...request, options }) - return takeNext({ - queue: sendResponses, - callCount: sendCalls.length, - operation: "chat.send", - options, - }) - }, + const agent: AgentLike = { + callModel: (request, options) => { + modelCalls.push({ request, options }) + + // Promise.try settles a synchronous throw from the queue as a rejection, + // the way the SDK's materialized stream rejects + const settled = Promise.try(takeNextModelResult, request) + return { + getResponse: async () => (await settled).response, + getText: async () => (await settled).text, + } }, + } + + const sdk: GenerationsLike = { generations: { getGeneration: async (request, options) => { generationCalls.push({ ...request, options }) - return takeNext({ - queue: generationResponses, - callCount: generationCalls.length, - operation: "generations.getGeneration", - options, - }) + const next = generationResponses.at(generationCalls.length - 1) + + if (!next) { + throw new Error(`stub: unexpected getGeneration call #${generationCalls.length}`) + } + if ("error" in next) throw next.error + if ("pending" in next) return next.pending(options) + return next.value }, }, } - return { sdk, sendCalls, generationCalls } + return { agent, sdk, modelCalls, generationCalls } } -const makeClient = (stub: { sdk: OpenRouterLike }) => { +const makeClient = (stub: { agent: AgentLike; sdk: GenerationsLike }) => { const logger = createTestLogger() const client = createOpenRouterClient({ + agent: stub.agent, sdk: stub.sdk, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, @@ -115,8 +199,32 @@ const makeClient = (stub: { sdk: OpenRouterLike }) => { return { client, logger } } -const makeStatusError = (statusCode: number): Error => { - return Object.assign(new Error(`HTTP ${statusCode}`), { statusCode }) +const RESPONSES_URL = "https://openrouter.ai/api/v1/responses" + +/** The HTTP metadata the SDK's error classes read their status and body from. */ +const makeHttpMeta = ( + statusCode: number, + body: string, +): { response: Response; request: Request; body: string } => ({ + response: new Response(body, { status: statusCode }), + request: new Request(RESPONSES_URL), + body, +}) + +/** An SDK error of the given status, as the SDK's base class carries it. */ +const makeStatusError = (statusCode: number, message = `HTTP ${statusCode}`): OpenRouterError => { + return new OpenRouterError(message, makeHttpMeta(statusCode, "")) +} + +/** The `{ error: { code, message } }` body OpenRouter sends on a rejected request. */ +const errorBodySchema = z.object({ error: z.object({ code: z.number(), message: z.string() }) }) + +/** The SDK's 400 error built from a captured error body. */ +const makeBadRequestError = (body: unknown): BadRequestResponseError => { + return new BadRequestResponseError( + errorBodySchema.parse(body), + makeHttpMeta(400, JSON.stringify(body)), + ) } /** OpenRouter's 400 when the prompt plus the requested output ceiling exceeds @@ -127,18 +235,18 @@ const makeContextOverflowError = ({ }: { contextLength: number inputTokens: number -}): Error => { +}): BadRequestResponseError => { const message = `This endpoint's maximum context length is ${contextLength} tokens. However, you requested about ${inputTokens + 128_000} tokens (${inputTokens} of text input, 128000 in the output). Please reduce the length of either one, or use the context-compression plugin to compress your prompt automatically.` - return Object.assign(new Error(message), { statusCode: 400 }) + return makeBadRequestError({ error: { code: 400, message } }) } -/** The model and output ceiling of every chat request sent, in order. */ +/** The model and output ceiling of every review request sent, in order. */ const sentCeilings = (stub: { - sendCalls: { chatRequest: ChatRequestSubset }[] + modelCalls: { request: ReviewRequest }[] }): { model: string; maxCompletionTokens: number }[] => { - return stub.sendCalls.map(({ chatRequest }) => ({ - model: chatRequest.model, - maxCompletionTokens: chatRequest.maxCompletionTokens, + return stub.modelCalls.map(({ request }) => ({ + model: request.model, + maxCompletionTokens: request.maxOutputTokens, })) } @@ -157,15 +265,42 @@ const captureRejection = async (request: Promise): Promise => const requestParams = { systemPrompt: "system prompt", userPrompt: "user prompt", - model: "openai/gpt-5-mini", - fallbackModel: "anthropic/claude-haiku-4.5", + model: "z-ai/glm-5.3-flash", + fallbackModel: "deepseek/deepseek-v4.1-flash", } +describe("captured Responses fixtures", () => { + it.each([ + { label: "accepted", payload: acceptedWirePayload }, + { label: "invalid JSON", payload: invalidJsonWirePayload }, + { label: "schema mismatch", payload: schemaMismatchWirePayload }, + ])("the $label wire payload passes the SDK's result transform", ({ payload }) => { + expect(openResponsesResultFromJSON(JSON.stringify(payload)).ok).toBe(true) + }) + + it("the overflow and bad-key bodies build the SDK's error classes with their status and message", () => { + const overflowError = makeBadRequestError(overflowErrorBody) + const badKeyError = new UnauthorizedResponseError( + errorBodySchema.parse(badKeyErrorBody), + makeHttpMeta(401, JSON.stringify(badKeyErrorBody)), + ) + + expect([overflowError, badKeyError].map((error) => [error.statusCode, error.message])).toEqual([ + [ + 400, + "This endpoint's maximum context length is 8192 tokens. However, you requested about 11593 tokens (4593 of text input, 7000 in the output). Please reduce the length of either one, or use the context-compression plugin to compress your prompt automatically.", + ], + [401, "User not found."], + ]) + }) +}) + describe("requestReview", () => { it("does not start an attempt when the review deadline has expired", async () => { - const stub = makeSdkStub({ sendResponses: [] }) + const stub = makeClientStubs({ modelResults: [] }) const logger = createTestLogger() const client = createOpenRouterClient({ + agent: stub.agent, sdk: stub.sdk, requestTimeoutMs: 45_000, remainingReviewMs: () => 0, @@ -184,7 +319,7 @@ describe("requestReview", () => { keyRejected: false, deadlineExceeded: true, }) - expect(stub.sendCalls).toEqual([]) + expect(stub.modelCalls).toEqual([]) expect(stub.generationCalls).toEqual([]) }) @@ -193,9 +328,9 @@ describe("requestReview", () => { async (rung) => { vi.useFakeTimers() try { - const late = Promise.withResolvers() - const stub = makeSdkStub({ - sendResponses: [ + const late = Promise.withResolvers() + const stub = makeClientStubs({ + modelResults: [ ...(rung === "fallback" ? [{ error: makeStatusError(400) }] : []), { pending: () => late.promise }, ], @@ -203,6 +338,7 @@ describe("requestReview", () => { const deadline = performance.now() + 100 const logger = createTestLogger() const client = createOpenRouterClient({ + agent: stub.agent, sdk: stub.sdk, requestTimeoutMs: 45_000, remainingReviewMs: () => deadline - performance.now(), @@ -246,9 +382,9 @@ describe("requestReview", () => { deadlineExceeded: true, }) expect( - stub.sendCalls.map(({ chatRequest, options }) => ({ - model: chatRequest.model, - aborted: options?.signal?.aborted, + stub.modelCalls.map(({ request }) => ({ + model: request.model, + aborted: request.signal.aborted, })), ).toEqual([ ...(rung === "fallback" ? [{ model: requestParams.model, aborted: false }] : []), @@ -257,7 +393,7 @@ describe("requestReview", () => { aborted: true, }, ]) - late.resolve(acceptedChatResult) + late.resolve(acceptedResult) await vi.advanceTimersByTimeAsync(1000) expect(error.attempts).toEqual(expectedAttempts) expect(stub.generationCalls).toEqual([]) @@ -271,21 +407,11 @@ describe("requestReview", () => { it("expires during retry sleep without losing billed usage or starting another attempt", async () => { vi.useFakeTimers() try { - const stub = makeSdkStub({ - sendResponses: [ - { - value: { - id: "invalid", - model: requestParams.model, - choices: [{ message: { content: "not json" } }], - usage: { promptTokens: 10, completionTokens: 20, cost: 0.01 }, - }, - }, - ], - }) + const stub = makeClientStubs({ modelResults: [{ value: invalidJsonResult }] }) const deadline = performance.now() + 100 const logger = createTestLogger() const client = createOpenRouterClient({ + agent: stub.agent, sdk: stub.sdk, requestTimeoutMs: 45_000, remainingReviewMs: () => deadline - performance.now(), @@ -306,16 +432,12 @@ describe("requestReview", () => { { model: requestParams.model, outcome: "invalid_json", - promptTokens: 10, - completionTokens: 20, - costUsd: 0.01, + ...ACCEPTED_USAGE, errorSummary: "response content is not valid JSON", }, ], }) - expect(stub.sendCalls.map(({ chatRequest }) => chatRequest.model)).toEqual([ - requestParams.model, - ]) + expect(stub.modelCalls.map(({ request }) => request.model)).toEqual([requestParams.model]) expect(vi.getTimerCount()).toBe(0) } finally { vi.useRealTimers() @@ -324,18 +446,19 @@ describe("requestReview", () => { it("skips cost lookup when an accepted response exhausts the remaining review time", async () => { const remainingReviewMs = vi.fn(() => 100) - const stub = makeSdkStub({ - sendResponses: [ + const stub = makeClientStubs({ + modelResults: [ { pending: async () => { remainingReviewMs.mockReturnValue(0) - return makeNoCostChatResult() + return makeNoCostResult() }, }, ], }) const logger = createTestLogger() const client = createOpenRouterClient({ + agent: stub.agent, sdk: stub.sdk, requestTimeoutMs: 45_000, remainingReviewMs, @@ -347,16 +470,14 @@ describe("requestReview", () => { { model: requestParams.model, outcome: "accepted", - promptTokens: 12000, - completionTokens: 800, + promptTokens: 54, + completionTokens: 102, costUsd: null, errorSummary: null, }, ], }) - expect(stub.sendCalls.map(({ chatRequest }) => chatRequest.model)).toEqual([ - requestParams.model, - ]) + expect(stub.modelCalls.map(({ request }) => request.model)).toEqual([requestParams.model]) expect(stub.generationCalls).toEqual([]) expect(logsWithMessage(logger, "generation cost lookup skipped")).toEqual([ { @@ -364,7 +485,7 @@ describe("requestReview", () => { message: "generation cost lookup skipped", data: { operation: "generation cost lookup", - generationId: "gen-no-cost", + generationId: ACCEPTED_GENERATION_ID, reason: "review deadline exceeded", }, }, @@ -375,13 +496,14 @@ describe("requestReview", () => { vi.useFakeTimers() try { const late = Promise.withResolvers() - const stub = makeSdkStub({ - sendResponses: [{ value: makeNoCostChatResult() }], + const stub = makeClientStubs({ + modelResults: [{ value: makeNoCostResult() }], generationResponses: [{ pending: () => late.promise }], }) const deadline = performance.now() + 100 const logger = createTestLogger() const client = createOpenRouterClient({ + agent: stub.agent, sdk: stub.sdk, requestTimeoutMs: 45_000, remainingReviewMs: () => deadline - performance.now(), @@ -396,8 +518,8 @@ describe("requestReview", () => { { model: requestParams.model, outcome: "accepted", - promptTokens: 12000, - completionTokens: 800, + promptTokens: 54, + completionTokens: 102, costUsd: null, errorSummary: null, }, @@ -409,68 +531,61 @@ describe("requestReview", () => { id, aborted: options?.signal?.aborted, })), - ).toEqual([{ id: "gen-no-cost", aborted: true }]) + ).toEqual([{ id: ACCEPTED_GENERATION_ID, aborted: true }]) late.resolve({ data: { totalCost: 42 } }) await vi.advanceTimersByTimeAsync(1000) expect(result).toEqual(expected) - expect(stub.sendCalls.map(({ chatRequest }) => chatRequest.model)).toEqual([ - requestParams.model, - ]) + expect(stub.modelCalls.map(({ request }) => request.model)).toEqual([requestParams.model]) expect(vi.getTimerCount()).toBe(0) } finally { vi.useRealTimers() } }) - it("sends the strict json_schema response format with the review schema and per-attempt timeout", async () => { - const stub = makeSdkStub({ sendResponses: [{ value: acceptedChatResult }] }) + it("sends one Responses request with the prompts, the full output ceiling, the strict review schema and an abort signal", async () => { + const stub = makeClientStubs({ modelResults: [{ value: acceptedResult }] }) const { client, logger } = makeClient(stub) await client.requestReview(requestParams, logger) - expect(stub.sendCalls).toEqual([ + expect(stub.modelCalls).toEqual([ { - chatRequest: { - model: "openai/gpt-5-mini", - messages: [ - { role: "system", content: "system prompt" }, - { role: "user", content: "user prompt" }, - ], - maxCompletionTokens: 128_000, - responseFormat: { - type: "json_schema", - jsonSchema: { + request: { + model: "z-ai/glm-5.3-flash", + instructions: "system prompt", + input: "user prompt", + maxOutputTokens: 128_000, + text: { + format: { + type: "json_schema", name: "review_response", strict: true, schema: reviewResponseJsonSchema, }, }, - stream: false, - }, - options: { - retries: { strategy: "none" }, signal: expect.any(AbortSignal), }, + options: { retries: { strategy: "none" } }, }, ]) }) it("returns the parsed review and a single accepted attempt on first success", async () => { - const stub = makeSdkStub({ sendResponses: [{ value: acceptedChatResult }] }) + const stub = makeClientStubs({ modelResults: [{ value: acceptedResult }] }) const { client, logger } = makeClient(stub) const result = await client.requestReview(requestParams, logger) expect(result).toEqual({ review: acceptedReview, - modelUsed: "openai/gpt-5-mini", + modelUsed: "z-ai/glm-5.3-flash", attempts: [ { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "accepted", - promptTokens: 12000, - completionTokens: 800, - costUsd: 0.0421, + promptTokens: 54, + completionTokens: 102, + costUsd: 0.000021036, errorSummary: null, }, ], @@ -478,46 +593,40 @@ describe("requestReview", () => { }) it("retries the same model once on invalid JSON, recording both attempts", async () => { - const stub = makeSdkStub({ - sendResponses: [{ value: invalidJsonChatResult }, { value: acceptedChatResult }], + const stub = makeClientStubs({ + modelResults: [{ value: invalidJsonResult }, { value: acceptedResult }], }) const { client, logger } = makeClient(stub) const result = await client.requestReview(requestParams, logger) - expect(stub.sendCalls.map((sendCall) => sendCall.chatRequest.model)).toEqual([ - "openai/gpt-5-mini", - "openai/gpt-5-mini", + expect(stub.modelCalls.map((modelCall) => modelCall.request.model)).toEqual([ + "z-ai/glm-5.3-flash", + "z-ai/glm-5.3-flash", ]) expect(result.attempts).toEqual([ { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "invalid_json", - promptTokens: 11800, - completionTokens: 42, - costUsd: null, + promptTokens: 54, + completionTokens: 102, + costUsd: 0.000021036, errorSummary: "response content is not valid JSON", }, { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "accepted", - promptTokens: 12000, - completionTokens: 800, - costUsd: 0.0421, + promptTokens: 54, + completionTokens: 102, + costUsd: 0.000021036, errorSummary: null, }, ]) }) - it("retries the same model once on empty content", async () => { - const emptyContentResult = { - id: "gen-empty", - model: "openai/gpt-5-mini", - choices: [{ message: { content: "" } }], - usage: { promptTokens: 10, completionTokens: 0, cost: null }, - } - const stub = makeSdkStub({ - sendResponses: [{ value: emptyContentResult }, { value: acceptedChatResult }], + it("retries the same model once when the output holds no message item", async () => { + const stub = makeClientStubs({ + modelResults: [{ value: makeReasoningOnlyResult() }, { value: acceptedResult }], }) const { client, logger } = makeClient(stub) @@ -525,48 +634,46 @@ describe("requestReview", () => { expect(result.attempts).toEqual([ { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "empty_content", - promptTokens: 10, - completionTokens: 0, - costUsd: null, + ...ACCEPTED_USAGE, errorSummary: "response had no text content", }, { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "accepted", - promptTokens: 12000, - completionTokens: 800, - costUsd: 0.0421, + promptTokens: 54, + completionTokens: 102, + costUsd: 0.000021036, errorSummary: null, }, ]) }) it("advances to the fallback model after two schema mismatches", async () => { - const stub = makeSdkStub({ - sendResponses: [ - { value: schemaMismatchChatResult }, - { value: schemaMismatchChatResult }, - { value: acceptedChatResult }, + const stub = makeClientStubs({ + modelResults: [ + { value: schemaMismatchResult }, + { value: schemaMismatchResult }, + { value: acceptedResult }, ], }) const { client, logger } = makeClient(stub) const result = await client.requestReview(requestParams, logger) - expect(stub.sendCalls.map((sendCall) => sendCall.chatRequest.model)).toEqual([ - "openai/gpt-5-mini", - "openai/gpt-5-mini", - "anthropic/claude-haiku-4.5", + expect(stub.modelCalls.map((modelCall) => modelCall.request.model)).toEqual([ + "z-ai/glm-5.3-flash", + "z-ai/glm-5.3-flash", + "deepseek/deepseek-v4.1-flash", ]) const schemaMismatchAttempt = { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "schema_mismatch", - promptTokens: 11900, - completionTokens: 12, - costUsd: 0.001, + promptTokens: 54, + completionTokens: 102, + costUsd: 0.000021036, errorSummary: "schema mismatch at analysis: Invalid input: expected string, received undefined", } @@ -574,30 +681,30 @@ describe("requestReview", () => { schemaMismatchAttempt, schemaMismatchAttempt, { - model: "anthropic/claude-haiku-4.5", + model: "deepseek/deepseek-v4.1-flash", outcome: "accepted", - promptTokens: 12000, - completionTokens: 800, - costUsd: 0.0421, + promptTokens: 54, + completionTokens: 102, + costUsd: 0.000021036, errorSummary: null, }, ]) }) it("retries the same model once on a 429", async () => { - const stub = makeSdkStub({ - sendResponses: [{ error: makeStatusError(429) }, { value: acceptedChatResult }], + const stub = makeClientStubs({ + modelResults: [{ error: makeStatusError(429) }, { value: acceptedResult }], }) const { client, logger } = makeClient(stub) const result = await client.requestReview(requestParams, logger) - expect(stub.sendCalls.map((sendCall) => sendCall.chatRequest.model)).toEqual([ - "openai/gpt-5-mini", - "openai/gpt-5-mini", + expect(stub.modelCalls.map((modelCall) => modelCall.request.model)).toEqual([ + "z-ai/glm-5.3-flash", + "z-ai/glm-5.3-flash", ]) expect(result.attempts[0]).toEqual({ - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "api_error", promptTokens: null, completionTokens: null, @@ -607,19 +714,19 @@ describe("requestReview", () => { }) it("retries a status-less network error and records its bare message", async () => { - const stub = makeSdkStub({ - sendResponses: [{ error: new Error("socket hang up") }, { value: acceptedChatResult }], + const stub = makeClientStubs({ + modelResults: [{ error: new Error("socket hang up") }, { value: acceptedResult }], }) const { client, logger } = makeClient(stub) const result = await client.requestReview(requestParams, logger) - expect(stub.sendCalls.map((sendCall) => sendCall.chatRequest.model)).toEqual([ - "openai/gpt-5-mini", - "openai/gpt-5-mini", + expect(stub.modelCalls.map((modelCall) => modelCall.request.model)).toEqual([ + "z-ai/glm-5.3-flash", + "z-ai/glm-5.3-flash", ]) expect(result.attempts[0]).toEqual({ - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "api_error", promptTokens: null, completionTokens: null, @@ -631,23 +738,21 @@ describe("requestReview", () => { it("retries an SDK-thrown TimeoutError as an api_error through the action's ladder, not the SDK's", async () => { const timeoutError = new Error("Request timed out") timeoutError.name = "TimeoutError" - const stub = makeSdkStub({ - sendResponses: [{ error: timeoutError }, { value: acceptedChatResult }], + const stub = makeClientStubs({ + modelResults: [{ error: timeoutError }, { value: acceptedResult }], }) const { client, logger } = makeClient(stub) const result = await client.requestReview(requestParams, logger) - expect(stub.sendCalls.map((sendCall) => sendCall.chatRequest.model)).toEqual([ - "openai/gpt-5-mini", - "openai/gpt-5-mini", + expect(stub.modelCalls.map((modelCall) => modelCall.request.model)).toEqual([ + "z-ai/glm-5.3-flash", + "z-ai/glm-5.3-flash", ]) - expect(stub.sendCalls[0]?.options).toEqual({ - retries: { strategy: "none" }, - signal: expect.any(AbortSignal), - }) + expect(stub.modelCalls[0]?.options).toEqual({ retries: { strategy: "none" } }) + expect(stub.modelCalls[0]?.request.signal).toBeInstanceOf(AbortSignal) expect(result.attempts[0]).toEqual({ - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "api_error", promptTokens: null, completionTokens: null, @@ -659,7 +764,7 @@ describe("requestReview", () => { level: "warn", message: "review attempt failed", data: { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", modelAttemptNumber: 1, outcome: "api_error", errorSummary: "Request timed out", @@ -669,15 +774,15 @@ describe("requestReview", () => { }) it("passes a live (non-aborted) AbortSignal to the SDK call", async () => { - const stub = makeSdkStub({ - sendResponses: [{ value: acceptedChatResult }], + const stub = makeClientStubs({ + modelResults: [{ value: acceptedResult }], generationResponses: [{ value: { data: { totalCost: 0.01 } } }], }) const { client, logger } = makeClient(stub) await client.requestReview(requestParams, logger) - const signal = stub.sendCalls[0]?.options?.signal + const signal = stub.modelCalls[0]?.request.signal expect(signal).toBeInstanceOf(AbortSignal) expect(signal?.aborted).toBe(false) }) @@ -687,9 +792,9 @@ describe("requestReview", () => { try { // Shaped like the SDK's rejection: its own error wrapping the // DOMException the aborted fetch raised - const rejectOnAbort = (options: StubOptions): Promise => { + const rejectOnAbort = (request: { signal: AbortSignal }): Promise => { return new Promise((_resolve, reject) => { - options?.signal?.addEventListener("abort", () => { + request.signal.addEventListener("abort", () => { reject( Object.assign(new Error("Request aborted by client"), { name: "RequestAbortedError", @@ -699,11 +804,12 @@ describe("requestReview", () => { }) }) } - const stub = makeSdkStub({ - sendResponses: [{ pending: rejectOnAbort }, { pending: rejectOnAbort }], + const stub = makeClientStubs({ + modelResults: [{ pending: rejectOnAbort }, { pending: rejectOnAbort }], }) const logger = createTestLogger() const client = createOpenRouterClient({ + agent: stub.agent, sdk: stub.sdk, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, @@ -727,10 +833,10 @@ describe("requestReview", () => { await expect(reviewPromise).rejects.toThrow( new ReviewRequestError({ message: - "review request failed after 2 attempt(s): openai/gpt-5-mini: timeout (no response within 45s); openai/gpt-5-mini: timeout (no response within 45s)", + "review request failed after 2 attempt(s): z-ai/glm-5.3-flash: timeout (no response within 45s); z-ai/glm-5.3-flash: timeout (no response within 45s)", attempts: [ { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "timeout", promptTokens: null, completionTokens: null, @@ -738,7 +844,7 @@ describe("requestReview", () => { errorSummary: "no response within 45s", }, { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "timeout", promptTokens: null, completionTokens: null, @@ -749,15 +855,15 @@ describe("requestReview", () => { keyRejected: false, }), ) - expect(stub.sendCalls[0]?.options?.signal?.aborted).toBe(true) + expect(stub.modelCalls[0]?.request.signal.aborted).toBe(true) // One orphan settlement per timed-out attempt const abortedSettlement = { level: "warn", message: "deadline-elapsed request settled", data: { - operation: "chat request", - model: "openai/gpt-5-mini", + operation: "review request", + model: "z-ai/glm-5.3-flash", elapsedMs: 45_000, timeoutMs: 45_000, settledWith: "abort_error", @@ -776,14 +882,12 @@ describe("requestReview", () => { it("advances to the fallback model at the deadline even when the SDK ignores the abort signal", async () => { vi.useFakeTimers() try { - const stub = makeSdkStub({ - sendResponses: [ - { pending: () => new Promise(() => undefined) }, - { value: acceptedChatResult }, - ], + const stub = makeClientStubs({ + modelResults: [{ pending: () => new Promise(() => undefined) }, { value: acceptedResult }], }) const logger = createTestLogger() const client = createOpenRouterClient({ + agent: stub.agent, sdk: stub.sdk, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, @@ -794,13 +898,13 @@ describe("requestReview", () => { await vi.advanceTimersByTimeAsync(45_000) const result = await reviewPromise - expect(stub.sendCalls.map((sendCall) => sendCall.chatRequest.model)).toEqual([ - "openai/gpt-5-mini", - "anthropic/claude-haiku-4.5", + expect(stub.modelCalls.map((modelCall) => modelCall.request.model)).toEqual([ + "z-ai/glm-5.3-flash", + "deepseek/deepseek-v4.1-flash", ]) expect(result.attempts).toEqual([ { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "timeout", promptTokens: null, completionTokens: null, @@ -808,11 +912,11 @@ describe("requestReview", () => { errorSummary: "no response within 45s", }, { - model: "anthropic/claude-haiku-4.5", + model: "deepseek/deepseek-v4.1-flash", outcome: "accepted", - promptTokens: 12000, - completionTokens: 800, - costUsd: 0.0421, + promptTokens: 54, + completionTokens: 102, + costUsd: 0.000021036, errorSummary: null, }, ]) @@ -823,8 +927,8 @@ describe("requestReview", () => { level: "info", message: "advancing to fallback model without same-model retry", data: { - from: "openai/gpt-5-mini", - to: "anthropic/claude-haiku-4.5", + from: "z-ai/glm-5.3-flash", + to: "deepseek/deepseek-v4.1-flash", }, }, ]) @@ -833,8 +937,8 @@ describe("requestReview", () => { level: "warn", message: "request deadline elapsed", data: { - operation: "chat request", - model: "openai/gpt-5-mini", + operation: "review request", + model: "z-ai/glm-5.3-flash", timeoutMs: 45_000, }, }, @@ -844,7 +948,7 @@ describe("requestReview", () => { level: "warn", message: "review attempt failed", data: { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", modelAttemptNumber: 1, outcome: "timeout", errorSummary: "no response within 45s", @@ -859,16 +963,17 @@ describe("requestReview", () => { it("logs a late response when the orphaned request completes after the deadline", async () => { vi.useFakeTimers() try { - const resolveLate = (): Promise => { + const resolveLate = (): Promise => { return new Promise((resolve) => { - setTimeout(() => resolve(acceptedChatResult), 120_000) + setTimeout(() => resolve(acceptedResult), 120_000) }) } - const stub = makeSdkStub({ - sendResponses: [{ pending: resolveLate }, { value: acceptedChatResult }], + const stub = makeClientStubs({ + modelResults: [{ pending: resolveLate }, { value: acceptedResult }], }) const logger = createTestLogger() const client = createOpenRouterClient({ + agent: stub.agent, sdk: stub.sdk, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, @@ -879,8 +984,8 @@ describe("requestReview", () => { await vi.advanceTimersByTimeAsync(45_000) const result = await reviewPromise expect(result.attempts.map((attempt) => [attempt.model, attempt.outcome])).toEqual([ - ["openai/gpt-5-mini", "timeout"], - ["anthropic/claude-haiku-4.5", "accepted"], + ["z-ai/glm-5.3-flash", "timeout"], + ["deepseek/deepseek-v4.1-flash", "accepted"], ]) await vi.advanceTimersByTimeAsync(75_000) @@ -890,8 +995,8 @@ describe("requestReview", () => { level: "warn", message: "deadline-elapsed request settled", data: { - operation: "chat request", - model: "openai/gpt-5-mini", + operation: "review request", + model: "z-ai/glm-5.3-flash", elapsedMs: 120_000, timeoutMs: 45_000, settledWith: "response", @@ -906,16 +1011,17 @@ describe("requestReview", () => { it("tags a late settlement with the caller's logger context after the request has rejected", async () => { vi.useFakeTimers() try { - const resolveLate = (): Promise => { + const resolveLate = (): Promise => { return new Promise((resolve) => { - setTimeout(() => resolve(acceptedChatResult), 120_000) + setTimeout(() => resolve(acceptedResult), 120_000) }) } - const stub = makeSdkStub({ - sendResponses: [{ pending: resolveLate }, { error: makeStatusError(400) }], + const stub = makeClientStubs({ + modelResults: [{ pending: resolveLate }, { error: makeStatusError(400) }], }) const logger = createTestLogger() const client = createOpenRouterClient({ + agent: stub.agent, sdk: stub.sdk, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, @@ -933,7 +1039,7 @@ describe("requestReview", () => { if (!(error instanceof ReviewRequestError)) throw error expect(error.message).toBe( - "review request failed after 2 attempt(s): openai/gpt-5-mini: timeout (no response within 45s); openai/gpt-5-mini: api_error (HTTP 400: HTTP 400)", + "review request failed after 2 attempt(s): z-ai/glm-5.3-flash: timeout (no response within 45s); z-ai/glm-5.3-flash: api_error (HTTP 400: HTTP 400)", ) expect(logsWithMessage(logger, "deadline-elapsed request settled")).toEqual([]) @@ -945,8 +1051,8 @@ describe("requestReview", () => { message: "deadline-elapsed request settled", data: { phase: "subtle-bugs", - operation: "chat request", - model: "openai/gpt-5-mini", + operation: "review request", + model: "z-ai/glm-5.3-flash", elapsedMs: 120_000, timeoutMs: 45_000, settledWith: "response", @@ -961,16 +1067,17 @@ describe("requestReview", () => { it("logs a late non-abort rejection without raising an unhandled rejection", async () => { vi.useFakeTimers() try { - const rejectLate = (): Promise => { + const rejectLate = (): Promise => { return new Promise((_resolve, reject) => { setTimeout(() => reject(new Error("socket hang up")), 120_000) }) } - const stub = makeSdkStub({ - sendResponses: [{ pending: rejectLate }, { value: acceptedChatResult }], + const stub = makeClientStubs({ + modelResults: [{ pending: rejectLate }, { value: acceptedResult }], }) const logger = createTestLogger() const client = createOpenRouterClient({ + agent: stub.agent, sdk: stub.sdk, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, @@ -981,8 +1088,8 @@ describe("requestReview", () => { await vi.advanceTimersByTimeAsync(45_000) const result = await reviewPromise expect(result.attempts.map((attempt) => [attempt.model, attempt.outcome])).toEqual([ - ["openai/gpt-5-mini", "timeout"], - ["anthropic/claude-haiku-4.5", "accepted"], + ["z-ai/glm-5.3-flash", "timeout"], + ["deepseek/deepseek-v4.1-flash", "accepted"], ]) await vi.advanceTimersByTimeAsync(75_000) @@ -991,8 +1098,8 @@ describe("requestReview", () => { level: "warn", message: "deadline-elapsed request settled", data: { - operation: "chat request", - model: "openai/gpt-5-mini", + operation: "review request", + model: "z-ai/glm-5.3-flash", elapsedMs: 120_000, timeoutMs: 45_000, settledWith: "error", @@ -1008,14 +1115,12 @@ describe("requestReview", () => { it("retries a timeout on the same model when no fallback is configured", async () => { vi.useFakeTimers() try { - const stub = makeSdkStub({ - sendResponses: [ - { pending: () => new Promise(() => undefined) }, - { value: acceptedChatResult }, - ], + const stub = makeClientStubs({ + modelResults: [{ pending: () => new Promise(() => undefined) }, { value: acceptedResult }], }) const logger = createTestLogger() const client = createOpenRouterClient({ + agent: stub.agent, sdk: stub.sdk, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, @@ -1034,7 +1139,7 @@ describe("requestReview", () => { expect(result.attempts).toEqual([ { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "timeout", promptTokens: null, completionTokens: null, @@ -1042,11 +1147,11 @@ describe("requestReview", () => { errorSummary: "no response within 45s", }, { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "accepted", - promptTokens: 12000, - completionTokens: 800, - costUsd: 0.0421, + promptTokens: 54, + completionTokens: 102, + costUsd: 0.000021036, errorSummary: null, }, ]) @@ -1061,9 +1166,9 @@ describe("requestReview", () => { it("exhausts the ladder in order when the fallback also times out, retrying only the last rung", async () => { vi.useFakeTimers() try { - const neverSettle = (): Promise => new Promise(() => undefined) - const stub = makeSdkStub({ - sendResponses: [ + const neverSettle = (): Promise => new Promise(() => undefined) + const stub = makeClientStubs({ + modelResults: [ { pending: neverSettle }, { pending: neverSettle }, { pending: neverSettle }, @@ -1071,6 +1176,7 @@ describe("requestReview", () => { }) const logger = createTestLogger() const client = createOpenRouterClient({ + agent: stub.agent, sdk: stub.sdk, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, @@ -1085,10 +1191,10 @@ describe("requestReview", () => { await expect(reviewPromise).rejects.toThrow( new ReviewRequestError({ message: - "review request failed after 3 attempt(s): openai/gpt-5-mini: timeout (no response within 45s); anthropic/claude-haiku-4.5: timeout (no response within 45s); anthropic/claude-haiku-4.5: timeout (no response within 45s)", + "review request failed after 3 attempt(s): z-ai/glm-5.3-flash: timeout (no response within 45s); deepseek/deepseek-v4.1-flash: timeout (no response within 45s); deepseek/deepseek-v4.1-flash: timeout (no response within 45s)", attempts: [ { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "timeout", promptTokens: null, completionTokens: null, @@ -1096,7 +1202,7 @@ describe("requestReview", () => { errorSummary: "no response within 45s", }, { - model: "anthropic/claude-haiku-4.5", + model: "deepseek/deepseek-v4.1-flash", outcome: "timeout", promptTokens: null, completionTokens: null, @@ -1104,7 +1210,7 @@ describe("requestReview", () => { errorSummary: "no response within 45s", }, { - model: "anthropic/claude-haiku-4.5", + model: "deepseek/deepseek-v4.1-flash", outcome: "timeout", promptTokens: null, completionTokens: null, @@ -1115,10 +1221,10 @@ describe("requestReview", () => { keyRejected: false, }), ) - expect(stub.sendCalls.map((sendCall) => sendCall.chatRequest.model)).toEqual([ - "openai/gpt-5-mini", - "anthropic/claude-haiku-4.5", - "anthropic/claude-haiku-4.5", + expect(stub.modelCalls.map((modelCall) => modelCall.request.model)).toEqual([ + "z-ai/glm-5.3-flash", + "deepseek/deepseek-v4.1-flash", + "deepseek/deepseek-v4.1-flash", ]) } finally { vi.useRealTimers() @@ -1129,14 +1235,12 @@ describe("requestReview", () => { vi.useFakeTimers() try { const retryDelayMs = 50 - const stub = makeSdkStub({ - sendResponses: [ - { pending: () => new Promise(() => undefined) }, - { value: acceptedChatResult }, - ], + const stub = makeClientStubs({ + modelResults: [{ pending: () => new Promise(() => undefined) }, { value: acceptedResult }], }) const logger = createTestLogger() const client = createOpenRouterClient({ + agent: stub.agent, sdk: stub.sdk, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, @@ -1162,13 +1266,14 @@ describe("requestReview", () => { it("reports the review deadline as the cost lookup's failure when that deadline cuts the lookup short", async () => { vi.useFakeTimers() try { - const stub = makeSdkStub({ - sendResponses: [{ value: makeNoCostChatResult() }], + const stub = makeClientStubs({ + modelResults: [{ value: makeNoCostResult() }], generationResponses: [{ pending: () => new Promise(() => undefined) }], }) const logger = createTestLogger() const deadline = performance.now() + 100 const client = createOpenRouterClient({ + agent: stub.agent, sdk: stub.sdk, requestTimeoutMs: 45_000, remainingReviewMs: () => deadline - performance.now(), @@ -1184,7 +1289,7 @@ describe("requestReview", () => { message: "request deadline elapsed", data: { operation: "generation cost lookup", - generationId: "gen-no-cost", + generationId: ACCEPTED_GENERATION_ID, timeoutMs: 100, }, }, @@ -1193,7 +1298,7 @@ describe("requestReview", () => { message: "generation cost lookup failed", data: { operation: "generation cost lookup", - generationId: "gen-no-cost", + generationId: ACCEPTED_GENERATION_ID, error: "review deadline exceeded", }, }, @@ -1206,12 +1311,13 @@ describe("requestReview", () => { it("degrades to a null cost with both warnings when the generation lookup exceeds the deadline", async () => { vi.useFakeTimers() try { - const stub = makeSdkStub({ - sendResponses: [{ value: makeNoCostChatResult() }], + const stub = makeClientStubs({ + modelResults: [{ value: makeNoCostResult() }], generationResponses: [{ pending: () => new Promise(() => undefined) }], }) const logger = createTestLogger() const client = createOpenRouterClient({ + agent: stub.agent, sdk: stub.sdk, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, @@ -1229,7 +1335,7 @@ describe("requestReview", () => { message: "request deadline elapsed", data: { operation: "generation cost lookup", - generationId: "gen-no-cost", + generationId: ACCEPTED_GENERATION_ID, timeoutMs: 45_000, }, }, @@ -1240,7 +1346,7 @@ describe("requestReview", () => { message: "generation cost lookup failed", data: { operation: "generation cost lookup", - generationId: "gen-no-cost", + generationId: ACCEPTED_GENERATION_ID, error: "no response within 45s", }, }, @@ -1250,9 +1356,9 @@ describe("requestReview", () => { } }) - it("clears the chat attempt timeout timer after a successful response", async () => { - const stub = makeSdkStub({ - sendResponses: [{ value: acceptedChatResult }], + it("clears the review attempt timeout timer after a successful response", async () => { + const stub = makeClientStubs({ + modelResults: [{ value: acceptedResult }], }) const { client, logger } = makeClient(stub) @@ -1267,11 +1373,12 @@ describe("requestReview", () => { it("does not sleep after the final attempt when retries are exhausted", async () => { const retryDelayMs = 50 - const stub = makeSdkStub({ - sendResponses: [{ error: makeStatusError(429) }, { error: makeStatusError(429) }], + const stub = makeClientStubs({ + modelResults: [{ error: makeStatusError(429) }, { error: makeStatusError(429) }], }) const logger = createTestLogger() const client = createOpenRouterClient({ + agent: stub.agent, sdk: stub.sdk, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, @@ -1285,10 +1392,10 @@ describe("requestReview", () => { ).rejects.toThrow( new ReviewRequestError({ message: - "review request failed after 2 attempt(s): openai/gpt-5-mini: api_error (HTTP 429: HTTP 429); openai/gpt-5-mini: api_error (HTTP 429: HTTP 429)", + "review request failed after 2 attempt(s): z-ai/glm-5.3-flash: api_error (HTTP 429: HTTP 429); z-ai/glm-5.3-flash: api_error (HTTP 429: HTTP 429)", attempts: [ { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "api_error", promptTokens: null, completionTokens: null, @@ -1296,7 +1403,7 @@ describe("requestReview", () => { errorSummary: "HTTP 429: HTTP 429", }, { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "api_error", promptTokens: null, completionTokens: null, @@ -1313,22 +1420,15 @@ describe("requestReview", () => { // after the final attempt. Only one sleep (between attempts 1 and 2). const retrySleepCalls = setTimeoutSpy.mock.calls.filter((call) => call[1] === retryDelayMs) expect(retrySleepCalls).toHaveLength(1) - expect(stub.sendCalls).toHaveLength(2) + expect(stub.modelCalls).toHaveLength(2) } finally { setTimeoutSpy.mockRestore() } }) it("truncates an error message longer than 200 characters in the attempt summary", async () => { - const stub = makeSdkStub({ - sendResponses: [ - { - error: Object.assign(new Error("x".repeat(250)), { - statusCode: 429, - }), - }, - { value: acceptedChatResult }, - ], + const stub = makeClientStubs({ + modelResults: [{ error: makeStatusError(429, "x".repeat(250)) }, { value: acceptedResult }], }) const { client, logger } = makeClient(stub) @@ -1338,20 +1438,20 @@ describe("requestReview", () => { }) it("advances to the fallback model without a retry on a 404", async () => { - const stub = makeSdkStub({ - sendResponses: [{ error: makeStatusError(404) }, { value: acceptedChatResult }], + const stub = makeClientStubs({ + modelResults: [{ error: makeStatusError(404) }, { value: acceptedResult }], }) const { client, logger } = makeClient(stub) const result = await client.requestReview(requestParams, logger) - expect(stub.sendCalls.map((sendCall) => sendCall.chatRequest.model)).toEqual([ - "openai/gpt-5-mini", - "anthropic/claude-haiku-4.5", + expect(stub.modelCalls.map((modelCall) => modelCall.request.model)).toEqual([ + "z-ai/glm-5.3-flash", + "deepseek/deepseek-v4.1-flash", ]) expect(result.attempts).toEqual([ { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "api_error", promptTokens: null, completionTokens: null, @@ -1359,21 +1459,21 @@ describe("requestReview", () => { errorSummary: "HTTP 404: HTTP 404", }, { - model: "anthropic/claude-haiku-4.5", + model: "deepseek/deepseek-v4.1-flash", outcome: "accepted", - promptTokens: 12000, - completionTokens: 800, - costUsd: 0.0421, + promptTokens: 54, + completionTokens: 102, + costUsd: 0.000021036, errorSummary: null, }, ]) }) it("retries the same model with an output ceiling that fits after a context-overflow 400", async () => { - const stub = makeSdkStub({ - sendResponses: [ + const stub = makeClientStubs({ + modelResults: [ { error: makeContextOverflowError({ contextLength: 262_144, inputTokens: 135_762 }) }, - { value: acceptedChatResult }, + { value: acceptedResult }, ], }) const { client, logger } = makeClient(stub) @@ -1382,15 +1482,15 @@ describe("requestReview", () => { // 262,144 window − 135,762 input − 8,192 margin expect(sentCeilings(stub)).toEqual([ - { model: "openai/gpt-5-mini", maxCompletionTokens: 128_000 }, - { model: "openai/gpt-5-mini", maxCompletionTokens: 118_190 }, + { model: "z-ai/glm-5.3-flash", maxCompletionTokens: 128_000 }, + { model: "z-ai/glm-5.3-flash", maxCompletionTokens: 118_190 }, ]) expect(result).toEqual({ review: acceptedReview, - modelUsed: "openai/gpt-5-mini", + modelUsed: "z-ai/glm-5.3-flash", attempts: [ { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "api_error", promptTokens: null, completionTokens: null, @@ -1399,11 +1499,11 @@ describe("requestReview", () => { "HTTP 400: This endpoint's maximum context length is 262144 tokens. However, you requested about 263762 tokens (135762 of text input, 128000 in the output). Please reduce the length of either one, or use the con…", }, { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "accepted", - promptTokens: 12000, - completionTokens: 800, - costUsd: 0.0421, + promptTokens: 54, + completionTokens: 102, + costUsd: 0.000021036, errorSummary: null, }, ], @@ -1412,17 +1512,17 @@ describe("requestReview", () => { { level: "info", message: CEILING_RETRY_LOG, - data: { model: "openai/gpt-5-mini", maxCompletionTokens: 118_190 }, + data: { model: "z-ai/glm-5.3-flash", maxCompletionTokens: 118_190 }, }, ]) }) it("starts the fallback model at the full output ceiling after the primary lowered it", async () => { - const stub = makeSdkStub({ - sendResponses: [ + const stub = makeClientStubs({ + modelResults: [ { error: makeContextOverflowError({ contextLength: 262_144, inputTokens: 135_762 }) }, { error: makeStatusError(500) }, - { value: acceptedChatResult }, + { value: acceptedResult }, ], }) const { client, logger } = makeClient(stub) @@ -1430,17 +1530,17 @@ describe("requestReview", () => { await client.requestReview(requestParams, logger) expect(sentCeilings(stub)).toEqual([ - { model: "openai/gpt-5-mini", maxCompletionTokens: 128_000 }, - { model: "openai/gpt-5-mini", maxCompletionTokens: 118_190 }, - { model: "anthropic/claude-haiku-4.5", maxCompletionTokens: 128_000 }, + { model: "z-ai/glm-5.3-flash", maxCompletionTokens: 128_000 }, + { model: "z-ai/glm-5.3-flash", maxCompletionTokens: 118_190 }, + { model: "deepseek/deepseek-v4.1-flash", maxCompletionTokens: 128_000 }, ]) }) it("advances to the fallback without a retry when a context-overflow 400 leaves no room for output", async () => { - const stub = makeSdkStub({ - sendResponses: [ + const stub = makeClientStubs({ + modelResults: [ { error: makeContextOverflowError({ contextLength: 262_144, inputTokens: 258_000 }) }, - { value: acceptedChatResult }, + { value: acceptedResult }, ], }) const { client, logger } = makeClient(stub) @@ -1448,17 +1548,17 @@ describe("requestReview", () => { await client.requestReview(requestParams, logger) expect(sentCeilings(stub)).toEqual([ - { model: "openai/gpt-5-mini", maxCompletionTokens: 128_000 }, - { model: "anthropic/claude-haiku-4.5", maxCompletionTokens: 128_000 }, + { model: "z-ai/glm-5.3-flash", maxCompletionTokens: 128_000 }, + { model: "deepseek/deepseek-v4.1-flash", maxCompletionTokens: 128_000 }, ]) }) it("advances to the fallback without a retry when the fitted output ceiling is too small for a review", async () => { // 262,144 window − 240,000 input − 8,192 margin = 13,952, under the 32,768 floor - const stub = makeSdkStub({ - sendResponses: [ + const stub = makeClientStubs({ + modelResults: [ { error: makeContextOverflowError({ contextLength: 262_144, inputTokens: 240_000 }) }, - { value: acceptedChatResult }, + { value: acceptedResult }, ], }) const { client, logger } = makeClient(stub) @@ -1466,17 +1566,17 @@ describe("requestReview", () => { await client.requestReview(requestParams, logger) expect(sentCeilings(stub)).toEqual([ - { model: "openai/gpt-5-mini", maxCompletionTokens: 128_000 }, - { model: "anthropic/claude-haiku-4.5", maxCompletionTokens: 128_000 }, + { model: "z-ai/glm-5.3-flash", maxCompletionTokens: 128_000 }, + { model: "deepseek/deepseek-v4.1-flash", maxCompletionTokens: 128_000 }, ]) }) it("retries at a fitted output ceiling exactly at the 32,768-token floor", async () => { // 262,144 window − 221,184 input − 8,192 margin = 32,768, the floor itself - const stub = makeSdkStub({ - sendResponses: [ + const stub = makeClientStubs({ + modelResults: [ { error: makeContextOverflowError({ contextLength: 262_144, inputTokens: 221_184 }) }, - { value: acceptedChatResult }, + { value: acceptedResult }, ], }) const { client, logger } = makeClient(stub) @@ -1484,18 +1584,18 @@ describe("requestReview", () => { await client.requestReview(requestParams, logger) expect(sentCeilings(stub)).toEqual([ - { model: "openai/gpt-5-mini", maxCompletionTokens: 128_000 }, - { model: "openai/gpt-5-mini", maxCompletionTokens: 32_768 }, + { model: "z-ai/glm-5.3-flash", maxCompletionTokens: 128_000 }, + { model: "z-ai/glm-5.3-flash", maxCompletionTokens: 32_768 }, ]) }) it("advances to the fallback without a retry when the rejected ceiling already fit the endpoint's window", async () => { // 262,144 window − 125,952 input − 8,192 margin = 128,000, the rejected // ceiling itself, so lowering it cannot fix the 400 - const stub = makeSdkStub({ - sendResponses: [ + const stub = makeClientStubs({ + modelResults: [ { error: makeContextOverflowError({ contextLength: 262_144, inputTokens: 125_952 }) }, - { value: acceptedChatResult }, + { value: acceptedResult }, ], }) const { client, logger } = makeClient(stub) @@ -1503,18 +1603,18 @@ describe("requestReview", () => { await client.requestReview(requestParams, logger) expect(sentCeilings(stub)).toEqual([ - { model: "openai/gpt-5-mini", maxCompletionTokens: 128_000 }, - { model: "anthropic/claude-haiku-4.5", maxCompletionTokens: 128_000 }, + { model: "z-ai/glm-5.3-flash", maxCompletionTokens: 128_000 }, + { model: "deepseek/deepseek-v4.1-flash", maxCompletionTokens: 128_000 }, ]) expect(logsWithMessage(logger, CEILING_RETRY_LOG)).toEqual([]) }) it("does not lower the ceiling when the context-overflow 400 was the model's last attempt", async () => { - const stub = makeSdkStub({ - sendResponses: [ + const stub = makeClientStubs({ + modelResults: [ { error: makeStatusError(500) }, { error: makeContextOverflowError({ contextLength: 262_144, inputTokens: 135_762 }) }, - { value: acceptedChatResult }, + { value: acceptedResult }, ], }) const { client, logger } = makeClient(stub) @@ -1522,59 +1622,110 @@ describe("requestReview", () => { await client.requestReview(requestParams, logger) expect(sentCeilings(stub)).toEqual([ - { model: "openai/gpt-5-mini", maxCompletionTokens: 128_000 }, - { model: "openai/gpt-5-mini", maxCompletionTokens: 128_000 }, - { model: "anthropic/claude-haiku-4.5", maxCompletionTokens: 128_000 }, + { model: "z-ai/glm-5.3-flash", maxCompletionTokens: 128_000 }, + { model: "z-ai/glm-5.3-flash", maxCompletionTokens: 128_000 }, + { model: "deepseek/deepseek-v4.1-flash", maxCompletionTokens: 128_000 }, ]) expect(logsWithMessage(logger, CEILING_RETRY_LOG)).toEqual([]) }) it("keeps the full ceiling when a non-400 error carries context-overflow wording", async () => { - const overflowWordedGatewayError = Object.assign( - makeContextOverflowError({ contextLength: 262_144, inputTokens: 135_762 }), - { statusCode: 502 }, + const overflowWordedGatewayError = makeStatusError( + 502, + makeContextOverflowError({ contextLength: 262_144, inputTokens: 135_762 }).message, ) - const stub = makeSdkStub({ - sendResponses: [{ error: overflowWordedGatewayError }, { value: acceptedChatResult }], + const stub = makeClientStubs({ + modelResults: [{ error: overflowWordedGatewayError }, { value: acceptedResult }], }) const { client, logger } = makeClient(stub) await client.requestReview(requestParams, logger) expect(sentCeilings(stub)).toEqual([ - { model: "openai/gpt-5-mini", maxCompletionTokens: 128_000 }, - { model: "openai/gpt-5-mini", maxCompletionTokens: 128_000 }, + { model: "z-ai/glm-5.3-flash", maxCompletionTokens: 128_000 }, + { model: "z-ai/glm-5.3-flash", maxCompletionTokens: 128_000 }, ]) }) it("advances to the fallback without a retry on a 400 that is not a context overflow", async () => { - const stub = makeSdkStub({ - sendResponses: [{ error: makeStatusError(400) }, { value: acceptedChatResult }], + const stub = makeClientStubs({ + modelResults: [{ error: makeStatusError(400) }, { value: acceptedResult }], }) const { client, logger } = makeClient(stub) await client.requestReview(requestParams, logger) expect(sentCeilings(stub)).toEqual([ - { model: "openai/gpt-5-mini", maxCompletionTokens: 128_000 }, - { model: "anthropic/claude-haiku-4.5", maxCompletionTokens: 128_000 }, + { model: "z-ai/glm-5.3-flash", maxCompletionTokens: 128_000 }, + { model: "deepseek/deepseek-v4.1-flash", maxCompletionTokens: 128_000 }, + ]) + }) + + it("advances to the fallback on the captured overflow body, whose 8K window leaves no room for a retry", async () => { + const stub = makeClientStubs({ + modelResults: [{ error: makeBadRequestError(overflowErrorBody) }, { value: acceptedResult }], + }) + const { client, logger } = makeClient(stub) + + const result = await client.requestReview(requestParams, logger) + + expect(sentCeilings(stub)).toEqual([ + { model: "z-ai/glm-5.3-flash", maxCompletionTokens: 128_000 }, + { model: "deepseek/deepseek-v4.1-flash", maxCompletionTokens: 128_000 }, ]) + expect(result.attempts[0]).toEqual({ + model: "z-ai/glm-5.3-flash", + outcome: "api_error", + promptTokens: null, + completionTokens: null, + costUsd: null, + errorSummary: + "HTTP 400: This endpoint's maximum context length is 8192 tokens. However, you requested about 11593 tokens (4593 of text input, 7000 in the output). Please reduce the length of either one, or use the context-co…", + }) + }) + + it("aborts the ladder after exactly one call on the captured bad-key body", async () => { + const badKeyError = new UnauthorizedResponseError( + errorBodySchema.parse(badKeyErrorBody), + makeHttpMeta(401, JSON.stringify(badKeyErrorBody)), + ) + const stub = makeClientStubs({ modelResults: [{ error: badKeyError }] }) + const { client, logger } = makeClient(stub) + + await expect(client.requestReview(requestParams, logger)).rejects.toThrow( + new ReviewRequestError({ + message: + "OpenRouter auth/credit error — aborting without fallback: z-ai/glm-5.3-flash: api_error (HTTP 401: User not found.)", + attempts: [ + { + model: "z-ai/glm-5.3-flash", + outcome: "api_error", + promptTokens: null, + completionTokens: null, + costUsd: null, + errorSummary: "HTTP 401: User not found.", + }, + ], + keyRejected: true, + }), + ) + expect(stub.modelCalls).toHaveLength(1) }) it.each([401, 402, 403])( "aborts the ladder after exactly one call on a %d", async (statusCode) => { - const stub = makeSdkStub({ - sendResponses: [{ error: makeStatusError(statusCode) }], + const stub = makeClientStubs({ + modelResults: [{ error: makeStatusError(statusCode) }], }) const { client, logger } = makeClient(stub) await expect(client.requestReview(requestParams, logger)).rejects.toThrow( new ReviewRequestError({ - message: `OpenRouter auth/credit error — aborting without fallback: openai/gpt-5-mini: api_error (HTTP ${statusCode}: HTTP ${statusCode})`, + message: `OpenRouter auth/credit error — aborting without fallback: z-ai/glm-5.3-flash: api_error (HTTP ${statusCode}: HTTP ${statusCode})`, attempts: [ { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "api_error", promptTokens: null, completionTokens: null, @@ -1585,13 +1736,13 @@ describe("requestReview", () => { keyRejected: true, }), ) - expect(stub.sendCalls).toHaveLength(1) + expect(stub.modelCalls).toHaveLength(1) }, ) it("throws a per-attempt summary after exhausting a fallback-less ladder", async () => { - const stub = makeSdkStub({ - sendResponses: [{ error: makeStatusError(500) }, { error: makeStatusError(503) }], + const stub = makeClientStubs({ + modelResults: [{ error: makeStatusError(500) }, { error: makeStatusError(503) }], }) const { client, logger } = makeClient(stub) @@ -1600,10 +1751,10 @@ describe("requestReview", () => { ).rejects.toThrow( new ReviewRequestError({ message: - "review request failed after 2 attempt(s): openai/gpt-5-mini: api_error (HTTP 500: HTTP 500); openai/gpt-5-mini: api_error (HTTP 503: HTTP 503)", + "review request failed after 2 attempt(s): z-ai/glm-5.3-flash: api_error (HTTP 500: HTTP 500); z-ai/glm-5.3-flash: api_error (HTTP 503: HTTP 503)", attempts: [ { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "api_error", promptTokens: null, completionTokens: null, @@ -1611,7 +1762,7 @@ describe("requestReview", () => { errorSummary: "HTTP 500: HTTP 500", }, { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "api_error", promptTokens: null, completionTokens: null, @@ -1622,12 +1773,12 @@ describe("requestReview", () => { keyRejected: false, }), ) - expect(stub.sendCalls).toHaveLength(2) + expect(stub.modelCalls).toHaveLength(2) }) it("carries every billed attempt on the error when the ladder is exhausted", async () => { - const stub = makeSdkStub({ - sendResponses: [{ error: makeStatusError(500) }, { error: makeStatusError(503) }], + const stub = makeClientStubs({ + modelResults: [{ error: makeStatusError(500) }, { error: makeStatusError(503) }], }) const { client, logger } = makeClient(stub) @@ -1641,7 +1792,7 @@ describe("requestReview", () => { expect(failure.keyRejected).toBe(false) expect(failure.attempts).toEqual([ { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "api_error", promptTokens: null, completionTokens: null, @@ -1649,7 +1800,7 @@ describe("requestReview", () => { errorSummary: "HTTP 500: HTTP 500", }, { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "api_error", promptTokens: null, completionTokens: null, @@ -1660,8 +1811,8 @@ describe("requestReview", () => { }) it("marks the error as key-rejected on an auth/credit failure", async () => { - const stub = makeSdkStub({ - sendResponses: [{ error: makeStatusError(402) }], + const stub = makeClientStubs({ + modelResults: [{ error: makeStatusError(402) }], }) const { client, logger } = makeClient(stub) @@ -1673,7 +1824,7 @@ describe("requestReview", () => { expect(failure.keyRejected).toBe(true) expect(failure.attempts).toEqual([ { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "api_error", promptTokens: null, completionTokens: null, @@ -1684,12 +1835,12 @@ describe("requestReview", () => { }) it("makes at most four calls across both ladder models", async () => { - const stub = makeSdkStub({ - sendResponses: [ - { value: invalidJsonChatResult }, - { value: invalidJsonChatResult }, - { value: invalidJsonChatResult }, - { value: invalidJsonChatResult }, + const stub = makeClientStubs({ + modelResults: [ + { value: invalidJsonResult }, + { value: invalidJsonResult }, + { value: invalidJsonResult }, + { value: invalidJsonResult }, ], }) const { client, logger } = makeClient(stub) @@ -1697,68 +1848,71 @@ describe("requestReview", () => { await expect(client.requestReview(requestParams, logger)).rejects.toThrow( new ReviewRequestError({ message: - "review request failed after 4 attempt(s): openai/gpt-5-mini: invalid_json (response content is not valid JSON); openai/gpt-5-mini: invalid_json (response content is not valid JSON); anthropic/claude-haiku-4.5: invalid_json (response content is not valid JSON); anthropic/claude-haiku-4.5: invalid_json (response content is not valid JSON)", + "review request failed after 4 attempt(s): z-ai/glm-5.3-flash: invalid_json (response content is not valid JSON); z-ai/glm-5.3-flash: invalid_json (response content is not valid JSON); deepseek/deepseek-v4.1-flash: invalid_json (response content is not valid JSON); deepseek/deepseek-v4.1-flash: invalid_json (response content is not valid JSON)", attempts: [ { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "invalid_json", - promptTokens: 11800, - completionTokens: 42, - costUsd: null, + promptTokens: 54, + completionTokens: 102, + costUsd: 0.000021036, errorSummary: "response content is not valid JSON", }, { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "invalid_json", - promptTokens: 11800, - completionTokens: 42, - costUsd: null, + promptTokens: 54, + completionTokens: 102, + costUsd: 0.000021036, errorSummary: "response content is not valid JSON", }, { - model: "anthropic/claude-haiku-4.5", + model: "deepseek/deepseek-v4.1-flash", outcome: "invalid_json", - promptTokens: 11800, - completionTokens: 42, - costUsd: null, + promptTokens: 54, + completionTokens: 102, + costUsd: 0.000021036, errorSummary: "response content is not valid JSON", }, { - model: "anthropic/claude-haiku-4.5", + model: "deepseek/deepseek-v4.1-flash", outcome: "invalid_json", - promptTokens: 11800, - completionTokens: 42, - costUsd: null, + promptTokens: 54, + completionTokens: 102, + costUsd: 0.000021036, errorSummary: "response content is not valid JSON", }, ], keyRejected: false, }), ) - expect(stub.sendCalls).toHaveLength(4) + expect(stub.modelCalls).toHaveLength(4) }) - it("treats an unrecognized chat response shape as a retryable api_error", async () => { - const stub = makeSdkStub({ - sendResponses: [{ value: { unexpected: true } }, { value: acceptedChatResult }], + it("treats an unrecognized review response shape as a retryable api_error", async () => { + const stub = makeClientStubs({ + modelResults: [ + { value: { response: { unexpected: true }, text: acceptedResult.text } }, + { value: acceptedResult }, + ], }) const { client, logger } = makeClient(stub) const result = await client.requestReview(requestParams, logger) expect(result.attempts[0]).toEqual({ - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "api_error", promptTokens: null, completionTokens: null, costUsd: null, - errorSummary: "unexpected chat response shape", + errorSummary: "unexpected review response shape", }) }) it("looks up the generation cost when the accepted attempt has no usage cost", async () => { - const stub = makeSdkStub({ - sendResponses: [{ value: makeNoCostChatResult() }], + const stub = makeClientStubs({ + modelResults: [{ value: makeNoCostResult() }], generationResponses: [{ value: { data: { totalCost: 0.0399 } } }], }) const { client, logger } = makeClient(stub) @@ -1767,7 +1921,7 @@ describe("requestReview", () => { expect(stub.generationCalls).toEqual([ { - id: "gen-no-cost", + id: ACCEPTED_GENERATION_ID, options: { retries: { strategy: "none" }, signal: expect.any(AbortSignal), @@ -1778,8 +1932,8 @@ describe("requestReview", () => { }) it("degrades to a null cost with a warning when the generation lookup fails", async () => { - const stub = makeSdkStub({ - sendResponses: [{ value: makeNoCostChatResult() }], + const stub = makeClientStubs({ + modelResults: [{ value: makeNoCostResult() }], generationResponses: [{ error: makeStatusError(500) }], }) const { client, logger } = makeClient(stub) @@ -1793,7 +1947,7 @@ describe("requestReview", () => { message: "generation cost lookup failed", data: { operation: "generation cost lookup", - generationId: "gen-no-cost", + generationId: ACCEPTED_GENERATION_ID, error: "HTTP 500", }, }, @@ -1801,15 +1955,15 @@ describe("requestReview", () => { }) it("degrades to a null cost with a warning when the generation lookup throws synchronously", async () => { - const sdkWithThrowingLookup: OpenRouterLike = { - chat: { send: async () => makeNoCostChatResult() }, + const sdkWithThrowingLookup: GenerationsLike = { generations: { getGeneration: () => { throw new Error("lookup threw before returning a promise") }, }, } - const { client, logger } = makeClient({ sdk: sdkWithThrowingLookup }) + const { agent } = makeClientStubs({ modelResults: [{ value: makeNoCostResult() }] }) + const { client, logger } = makeClient({ agent, sdk: sdkWithThrowingLookup }) const result = await client.requestReview(requestParams, logger) @@ -1820,7 +1974,7 @@ describe("requestReview", () => { message: "generation cost lookup failed", data: { operation: "generation cost lookup", - generationId: "gen-no-cost", + generationId: ACCEPTED_GENERATION_ID, error: "lookup threw before returning a promise", }, }, @@ -1828,8 +1982,8 @@ describe("requestReview", () => { }) it("degrades to a null cost with a warning when the generation response has an unexpected shape", async () => { - const stub = makeSdkStub({ - sendResponses: [{ value: makeNoCostChatResult() }], + const stub = makeClientStubs({ + modelResults: [{ value: makeNoCostResult() }], generationResponses: [{ value: { data: {} } }], }) const { client, logger } = makeClient(stub) @@ -1841,19 +1995,14 @@ describe("requestReview", () => { { level: "warn", message: "unexpected generation response shape", - data: { operation: "generation cost lookup", generationId: "gen-no-cost" }, + data: { operation: "generation cost lookup", generationId: ACCEPTED_GENERATION_ID }, }, ]) }) - it("records null token counts when the response omits the usage block", async () => { - const noUsageResult = { - id: "gen-no-usage", - model: "openai/gpt-5-mini", - choices: [{ message: { content: JSON.stringify(acceptedReview) } }], - } - const stub = makeSdkStub({ - sendResponses: [{ value: noUsageResult }], + it("records null token counts when the response carries no usage block", async () => { + const stub = makeClientStubs({ + modelResults: [{ value: makeNoUsageResult() }], generationResponses: [{ value: { data: { totalCost: 0.0399 } } }], }) const { client, logger } = makeClient(stub) @@ -1864,7 +2013,7 @@ describe("requestReview", () => { // accepted attempt through the generation-cost fallback expect(result.attempts).toEqual([ { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "accepted", promptTokens: null, completionTokens: null, diff --git a/src/openrouter/client.ts b/src/openrouter/client.ts index 0b951857..815618de 100644 --- a/src/openrouter/client.ts +++ b/src/openrouter/client.ts @@ -31,37 +31,39 @@ export type StructuredReviewResult = { attempts: ModelAttempt[] } -/** Structural subset of the SDK's ChatRequest — exactly what we send. */ -export type ChatRequestSubset = { +/** Structural subset of the Agent SDK's call-model input — exactly what we + * send, as one Responses API request with no tools. */ +export type ReviewRequest = { model: string - messages: { role: "system" | "user"; content: string }[] - maxCompletionTokens: number - responseFormat: { - type: "json_schema" - jsonSchema: { + instructions: string + input: string + maxOutputTokens: number + text: { + format: { + type: "json_schema" name: string strict: boolean schema: Record } } - stream: false } /** - * Structural subset of the OpenRouter SDK — test stubs are plain objects. - * Method syntax keeps the real SDK assignable under strictFunctionTypes; - * responses are `unknown` on purpose: Zod parses across the boundary. + * Structural subset of the Agent SDK client — test stubs are plain objects. + * Method syntax keeps the real client assignable under strictFunctionTypes. + * `getResponse` resolves `unknown` on purpose: Zod parses across the + * boundary. `getText` is the SDK's own extraction of the message text. */ -export type OpenRouterLike = { - chat: { - send( - request: { chatRequest: ChatRequestSubset }, - options?: { - retries?: { strategy: "backoff" | "none" } - signal?: AbortSignal - }, - ): Promise - } +export type AgentLike = { + callModel( + request: ReviewRequest & { signal: AbortSignal }, + options?: { retries?: { strategy: "backoff" | "none" } }, + ): { getResponse(): Promise; getText(): Promise } +} + +/** Structural subset of the OpenRouter SDK's REST client — the generation + * cost lookup is the one endpoint the Agent SDK does not cover. */ +export type GenerationsLike = { generations: { getGeneration( request: { id: string }, @@ -88,17 +90,18 @@ export type OpenRouterClient = { ) => Promise } -const chatResultSchema = z.object({ +/** The fields read from the Agent SDK's materialized Responses result. The + * SDK types `usage` as both nullable and optional, so the block is nullish. */ +const responsesResultSchema = z.object({ id: z.string(), model: z.string(), - choices: z.array(z.object({ message: z.object({ content: z.unknown() }) })), usage: z .object({ - promptTokens: z.number(), - completionTokens: z.number(), - cost: z.number().nullable().optional(), + inputTokens: z.number(), + outputTokens: z.number(), + cost: z.number().nullish(), }) - .optional(), + .nullish(), }) const generationResponseSchema = z.object({ @@ -126,7 +129,7 @@ const MAX_ATTEMPTS_PER_MODEL = 2 * rate-limit burst has a recovery window. */ const RETRY_DELAY_MS = 1_000 -/** The output ceiling, sent as the request's `maxCompletionTokens`, for every +/** The output ceiling, sent as the request's `maxOutputTokens`, for every * attempt except a context-overflow retry. Without one, some providers apply * a default output limit that cuts a large review off mid-JSON. */ const MAX_COMPLETION_TOKENS = 128_000 @@ -302,7 +305,7 @@ const parseJsonOrNull = (text: string): { parsed: unknown } | null => { } } -const buildChatRequest = ({ +const buildReviewRequest = ({ systemPrompt, userPrompt, model, @@ -312,22 +315,19 @@ const buildChatRequest = ({ userPrompt: string model: string maxCompletionTokens: number -}): ChatRequestSubset => ({ +}): ReviewRequest => ({ model, - messages: [ - { role: "system", content: systemPrompt }, - { role: "user", content: userPrompt }, - ], - maxCompletionTokens, - responseFormat: { - type: "json_schema", - jsonSchema: { + instructions: systemPrompt, + input: userPrompt, + maxOutputTokens: maxCompletionTokens, + text: { + format: { + type: "json_schema", name: "review_response", strict: true, schema: reviewResponseJsonSchema, }, }, - stream: false, }) type SingleAttempt = @@ -393,12 +393,14 @@ const summarizeAttempts = (attempts: ModelAttempt[]): string => { /** The client holds no logger. Each requestReview call passes its own, so * concurrent callers can tag their requests' lines with their own props. */ export const createOpenRouterClient = ({ + agent, sdk, requestTimeoutMs, remainingReviewMs, retryDelayMs = RETRY_DELAY_MS, }: { - sdk: OpenRouterLike + agent: AgentLike + sdk: GenerationsLike /** Per-attempt deadline. When it elapses the attempt records outcome * `timeout` and the ladder moves on — to the next model when one * exists, otherwise a same-model retry while attempts remain — whether @@ -424,26 +426,36 @@ export const createOpenRouterClient = ({ } } - /** Sends one chat request under the per-attempt deadline. Its only log lines - * are withDeadline's; requestReview logs each attempt's outcome. */ + /** Sends one review request under the per-attempt deadline. Its only log + * lines are withDeadline's; requestReview logs each attempt's outcome. */ const attemptOnce = async ( - chatRequest: ChatRequestSubset, + reviewRequest: ReviewRequest, logger: Logger, ): Promise => { - const { model } = chatRequest + const { model } = reviewRequest // No remaining-time guard here. requestReview calls ensureReviewTimeRemaining // before every attempt, so some review time is always left const callTimeout = getCallTimeout(remainingReviewMs()) + // The Agent SDK streams every request and materializes the result on the + // first read. A "Stream ended without completion event" rejection on every + // model means its bundled SDK copy rejected a changed wire field or + // ignored a stream error event; only the raw stream tells those apart const sendResult = await withDeadline( { - start: (signal) => { - return sdk.chat.send({ chatRequest }, { retries: { strategy: "none" }, signal }) + start: async (signal) => { + const modelResult = agent.callModel( + { ...reviewRequest, signal }, + { retries: { strategy: "none" } }, + ) + const response = await modelResult.getResponse() + const text = await modelResult.getText() + return { response, text } }, timeoutMs: callTimeout.timeoutMs, }, - logger.child({ operation: "chat request", model }), + logger.child({ operation: "review request", model }), ) // Retryable even when the review deadline caused the timeout, because @@ -480,7 +492,7 @@ export const createOpenRouterClient = ({ } const fittedCeiling = fittedMaxCompletionTokens({ error: sendResult.error, - rejectedMaxCompletionTokens: chatRequest.maxCompletionTokens, + rejectedMaxCompletionTokens: reviewRequest.maxOutputTokens, }) if (fittedCeiling) { @@ -497,7 +509,7 @@ export const createOpenRouterClient = ({ return { kind: "failed", attempt, retryable, keyRejected } } - const parsedResult = chatResultSchema.safeParse(sendResult.value) + const parsedResult = responsesResultSchema.safeParse(sendResult.value.response) // A malformed response body points at a provider or gateway glitch rather // than the request, so a second call can come back well-formed @@ -510,20 +522,20 @@ export const createOpenRouterClient = ({ promptTokens: null, completionTokens: null, costUsd: null, - errorSummary: "unexpected chat response shape", + errorSummary: "unexpected review response shape", }, retryable: true, keyRejected: false, } } - const chatResult = parsedResult.data + const responsesResult = parsedResult.data // Failed-validation attempts still carry usage — they were billed too const usage = { - promptTokens: chatResult.usage?.promptTokens ?? null, - completionTokens: chatResult.usage?.completionTokens ?? null, - costUsd: chatResult.usage?.cost ?? null, + promptTokens: responsesResult.usage?.inputTokens ?? null, + completionTokens: responsesResult.usage?.outputTokens ?? null, + costUsd: responsesResult.usage?.cost ?? null, } const failedValidation = (outcome: AttemptOutcome, errorSummary: string): SingleAttempt => ({ kind: "failed", @@ -532,9 +544,11 @@ export const createOpenRouterClient = ({ keyRejected: false, }) - const content = chatResult.choices[0]?.message.content + // The SDK extracts "" when the output holds no message item, such as a + // reasoning-only response + const content = sendResult.value.text - if (typeof content !== "string" || content === "") { + if (content === "") { return failedValidation("empty_content", "response had no text content") } @@ -559,8 +573,8 @@ export const createOpenRouterClient = ({ return { kind: "accepted", review: parsedReview.data, - routedModel: chatResult.model, - generationId: chatResult.id, + routedModel: responsesResult.model, + generationId: responsesResult.id, attempt: { model, outcome: "accepted", ...usage, errorSummary: null }, } } @@ -645,7 +659,7 @@ export const createOpenRouterClient = ({ while (modelAttemptNumber <= MAX_ATTEMPTS_PER_MODEL) { ensureReviewTimeRemaining() const attemptResult = await attemptOnce( - buildChatRequest({ systemPrompt, userPrompt, model: ladderModel, maxCompletionTokens }), + buildReviewRequest({ systemPrompt, userPrompt, model: ladderModel, maxCompletionTokens }), logger, ) From ff7cff2ca76960929b71f9f8009ef3587b913dfd Mon Sep 17 00:00:00 2001 From: aliasunder <32620895+aliasunder@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:49:31 -0500 Subject: [PATCH 03/20] docs: name the Agent SDK in the structure tree and remove the Responses probe Co-Authored-By: Claude Fable 5.1 --- .github/scripts/probe-responses.ts | 179 -------------------------- .github/workflows/probe_responses.yml | 40 ------ AGENTS.md | 5 +- 3 files changed, 3 insertions(+), 221 deletions(-) delete mode 100644 .github/scripts/probe-responses.ts delete mode 100644 .github/workflows/probe_responses.yml diff --git a/.github/scripts/probe-responses.ts b/.github/scripts/probe-responses.ts deleted file mode 100644 index 716c2d61..00000000 --- a/.github/scripts/probe-responses.ts +++ /dev/null @@ -1,179 +0,0 @@ -// Temporary probe for the Agent SDK migration. It captures the raw Responses -// API payloads (accepted, context overflow, bad key) that become the client's -// fixtures, printing each HTTP body between markers from the SDK's response -// hooks. It runs once from the probe workflow on the migration branch under -// bun (which resolves the src import), then both files are removed. -import { OpenRouter } from "@openrouter/agent" -import envVar from "env-var" -import { reviewResponseJsonSchema } from "../../src/review/finding.js" - -const apiKey = envVar.from(process.env).get("OPENROUTER_KEY").required().asString() - -/** Every primary and fallback slug a consumer runs (repo variables read 2026-11-11). */ -const consumerSlugs = [ - "deepseek/deepseek-v4.1-flash:online", - "z-ai/glm-5.3-flash:online", - "deepseek/deepseek-v4.1-flash-1017:online", - "deepseek/deepseek-v4-flash-1017:online", - "deepseek/deepseek-v4-flash-0731:online", - "minimax/minimax-m3:online", - "deepseek/deepseek-v4-flash-1017", - "z-ai/glm-5.3-flash", -] - -/** The ceiling the client sends on every attempt except a fitted retry. */ -const PRODUCTION_MAX_OUTPUT_TOKENS = 128_000 - -/** The overflow target has an 8,192-token window and endpoints that cap output - * at 7,372 tokens, so a 2K prompt plus a 7,000 ceiling overflows the window - * while staying under the output cap (public endpoints API, 2026-11-11). */ -const OVERFLOW_MODEL = "sao10k/l3-lunaris-8b" -const OVERFLOW_MAX_OUTPUT_TOKENS = 7_000 - -const REQUEST_TIMEOUT_MS = 120_000 - -/** Prints one captured payload between markers the session greps for. */ -const printCapture = (label: string, payload: string): void => { - console.log(`<<>>`) - console.log(payload) - console.log(`<<>>`) -} - -/** Reads a cloned HTTP body without disturbing the SDK's own read. */ -const captureBody = async (label: string, response: Response): Promise => { - const text = await response.clone().text() - printCapture(`${label} status=${response.status}`, text) -} - -const makeClient = (key: string, label: string): OpenRouter => { - return new OpenRouter({ - apiKey: key, - hooks: [ - { - afterSuccess: async (_context: unknown, response: Response): Promise => { - await captureBody(`${label} raw`, response) - return response - }, - afterError: async ( - _context: unknown, - response: Response | null, - error: unknown, - ): Promise<{ response: Response | null; error: unknown }> => { - if (response) await captureBody(`${label} raw-error`, response) - return { response, error } - }, - }, - ], - }) -} - -const describeError = (error: unknown): string => { - if (!(error instanceof Error)) return JSON.stringify({ thrown: String(error) }) - const statusCode = "statusCode" in error ? error.statusCode : undefined - const body = - "body" in error && typeof error.body === "string" ? error.body.slice(0, 4000) : undefined - return JSON.stringify({ - name: error.name, - constructor: error.constructor.name, - statusCode, - message: error.message, - body, - }) -} - -const reviewInstructions = - "You are a code reviewer. Reply with JSON that matches the schema: an analysis string and a findings array." -const reviewInput = - 'Review this one-line change and report no findings: `export const greeting = "hello"`.' - -const runAccepted = async (model: string): Promise => { - const client = makeClient(apiKey, `accepted ${model}`) - try { - const result = client.callModel( - { - model, - instructions: reviewInstructions, - input: reviewInput, - maxOutputTokens: PRODUCTION_MAX_OUTPUT_TOKENS, - text: { - format: { - type: "json_schema", - name: "review_response", - strict: true, - schema: reviewResponseJsonSchema, - }, - }, - }, - { retries: { strategy: "none" }, timeoutMs: REQUEST_TIMEOUT_MS }, - ) - const response = await result.getResponse() - printCapture(`accepted ${model} parsed`, JSON.stringify(response)) - printCapture(`accepted ${model} text`, await result.getText()) - } catch (error) { - printCapture(`accepted ${model} threw`, describeError(error)) - } -} - -const runOverflow = async (): Promise => { - const client = makeClient(apiKey, "overflow") - const filler = "The quick brown fox jumps over the lazy dog. ".repeat(400) - try { - const result = client.callModel( - { - model: OVERFLOW_MODEL, - instructions: reviewInstructions, - input: `${filler}\n${reviewInput}`, - maxOutputTokens: OVERFLOW_MAX_OUTPUT_TOKENS, - // Endpoints with an 8K window compress the prompt by default, which - // would hide the overflow; the probe needs the rejection itself - plugins: [{ id: "context-compression", enabled: false }], - text: { - format: { - type: "json_schema", - name: "review_response", - strict: true, - schema: reviewResponseJsonSchema, - }, - }, - }, - { retries: { strategy: "none" }, timeoutMs: REQUEST_TIMEOUT_MS }, - ) - const response = await result.getResponse() - printCapture("overflow returned", JSON.stringify(response)) - } catch (error) { - printCapture("overflow threw", describeError(error)) - } -} - -const runBadKey = async (): Promise => { - const client = makeClient("sk-or-v1-invalid-probe-key", "bad-key") - try { - const result = client.callModel( - { - model: "deepseek/deepseek-v4.1-flash", - instructions: reviewInstructions, - input: reviewInput, - maxOutputTokens: PRODUCTION_MAX_OUTPUT_TOKENS, - text: { - format: { - type: "json_schema", - name: "review_response", - strict: true, - schema: reviewResponseJsonSchema, - }, - }, - }, - { retries: { strategy: "none" }, timeoutMs: REQUEST_TIMEOUT_MS }, - ) - const response = await result.getResponse() - printCapture("bad-key returned", JSON.stringify(response)) - } catch (error) { - printCapture("bad-key threw", describeError(error)) - } -} - -for (const slug of consumerSlugs) { - await runAccepted(slug) -} -await runOverflow() -await runBadKey() diff --git a/.github/workflows/probe_responses.yml b/.github/workflows/probe_responses.yml deleted file mode 100644 index 5fa10ed8..00000000 --- a/.github/workflows/probe_responses.yml +++ /dev/null @@ -1,40 +0,0 @@ -# Temporary workflow for the Agent SDK migration. It captures raw Responses -# API payloads for the client's fixtures and is removed before the PR merges. -name: Probe Responses - -on: - pull_request: - types: [opened, synchronize, reopened] - -permissions: - contents: read - -jobs: - probe: - runs-on: ubuntu-latest - # The probe sends real requests with the repo's key, so a single run is - # enough to capture every payload - timeout-minutes: 10 - # Only the owner's same-repo PRs run secrets-bearing jobs - if: >- - github.event.pull_request.user.login == github.repository_owner && - github.event.pull_request.head.repo.full_name == github.repository - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - with: - node-version-file: .nvmrc - cache: npm - - - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 - - - name: Install dependencies - run: npm ci - - - name: Capture Responses payloads - env: - OPENROUTER_KEY: ${{ secrets.OPENROUTER_KEY }} - run: bun .github/scripts/probe-responses.ts diff --git a/AGENTS.md b/AGENTS.md index 7ba23a4f..1112360c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,7 +28,7 @@ src/ config.ts # action inputs → validated ActionConfig logger.ts # structured JSON logger — levels, child contexts, lazy props github/ # GitHub I/O: event payload → PrContext, octokit wrappers (diff fetch, review posting) - openrouter/ # OpenRouter I/O: @openrouter/sdk wrapper, per-attempt and shared review deadlines, retry ladder (up to two attempts per model, then the fallback_model input; on HTTP errors, timeouts, invalid structured output), cost summary + openrouter/ # OpenRouter I/O: review requests through @openrouter/agent (callModel, one Responses API request per attempt), generation cost lookup through @openrouter/sdk, per-attempt and shared review deadlines, retry ladder (up to two attempts per model, then the fallback_model input; on HTTP errors, timeouts, invalid structured output), cost summary diff/ # pure transforms over parse-diff output: git-quoted path decoding, annotation, commentable lines, diff-level exclusion (patterns, gitattributes linguist rules, wildcard cap: at most 2 "*" per path segment so matching cannot backtrack for minutes) context/ # workspace I/O: conventions file, root .gitattributes, changed files, import-trace scan, doc-mention scan, priority docs review/ # pure review logic (the model call is a function the caller passes in): finding schema, phases + stage dispatch (defined under Review instruction authoring), prompt, non-finding filter and unknown-file filter (README sections of those names), cross-phase merge, path normalization, selection, comment mapping, markdown code spans, title similarity, context notes, summary @@ -47,7 +47,8 @@ backwards dependency and a bug — enforced by a the outermost layer, so only tests import from a `__tests__/` folder. Use the official SDKs — don't hand-roll what `@actions/core`, -`@actions/github` (octokit), `@openrouter/sdk`, or `parse-diff` already do. +`@actions/github` (octokit), `@openrouter/agent`, `@openrouter/sdk`, or +`parse-diff` already do. Before writing any parsing or validation helper, check the SDK's utility surface first: `@actions/core` ships `getBooleanInput` (strict YAML 1.2 booleans), which replaced a hand-rolled boolean-string Zod transform here. From 3f4a713358bea4d053857df91dac6cc7f5b54589 Mon Sep 17 00:00:00 2001 From: aliasunder <32620895+aliasunder@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:58:00 -0500 Subject: [PATCH 04/20] fix(review): list @openrouter/agent among the official SDKs in CONTRIBUTING.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AGENTS.md names the Agent SDK in its official-SDK list; the contributor guide's copy of that list now matches. Ship-Check: pr-review · claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- CONTRIBUTING.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 22c5a289..724586db 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -43,7 +43,8 @@ points: constructs SDK clients and touches `process.env`, except that `logger.ts` reads `LOG_LEVEL` - Use the official SDKs (`@actions/core`, `@actions/github`, - `@openrouter/sdk`, `parse-diff`) — don't hand-roll what they already do + `@openrouter/agent`, `@openrouter/sdk`, `parse-diff`) — don't hand-roll + what they already do - Relative imports carry explicit `.js` extensions (ESM runtime requirement) - Tests are behavioral specs: one focused `it()` per behavior, exact assertions, `const` per test From a011934c37ba8fae9f0de9a775f9bacd4683d617 Mon Sep 17 00:00:00 2001 From: aliasunder <32620895+aliasunder@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:06:40 -0500 Subject: [PATCH 05/20] style: name the output ceiling maxOutputTokens and clarify the Agent SDK client MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rename the output-ceiling identifiers to the Responses field's name (MAX_OUTPUT_TOKENS, MIN_FITTED_MAX_OUTPUT_TOKENS, fittedMaxOutputTokens, the maxOutputTokens local and its fitted-retry log field) - Rename createOpenRouterClient's sdk param to costLookupClient - Name @openrouter/agent and its bundled @openrouter/sdk copy where the client relies on them; state where callModel takes its abort signal, why getText sends no second request, and that a Responses id is the generation id the cost lookup takes - Map requestReview's loop exits; give the 128_000 ceiling its reason - main.ts: say why unhandled rejections only warn and why cleanups are a Set - Docs: drop API detail from the AGENTS.md structure line; give each OpenRouter SDK its role in CONTRIBUTING.md Ship-Check: code-quality · claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- CONTRIBUTING.md | 5 +- src/__tests__/orchestrate.test.ts | 6 +- src/main.ts | 5 +- src/openrouter/__tests__/client.test.ts | 92 ++++++++++---------- src/openrouter/client.ts | 108 +++++++++++++++--------- 6 files changed, 125 insertions(+), 93 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 1112360c..c4e9a93f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,7 +28,7 @@ src/ config.ts # action inputs → validated ActionConfig logger.ts # structured JSON logger — levels, child contexts, lazy props github/ # GitHub I/O: event payload → PrContext, octokit wrappers (diff fetch, review posting) - openrouter/ # OpenRouter I/O: review requests through @openrouter/agent (callModel, one Responses API request per attempt), generation cost lookup through @openrouter/sdk, per-attempt and shared review deadlines, retry ladder (up to two attempts per model, then the fallback_model input; on HTTP errors, timeouts, invalid structured output), cost summary + openrouter/ # OpenRouter I/O: review requests through @openrouter/agent, generation cost lookup through @openrouter/sdk, per-attempt and shared review deadlines, retry ladder (up to two attempts per model, then the fallback_model input; on HTTP errors, timeouts, invalid structured output), cost summary diff/ # pure transforms over parse-diff output: git-quoted path decoding, annotation, commentable lines, diff-level exclusion (patterns, gitattributes linguist rules, wildcard cap: at most 2 "*" per path segment so matching cannot backtrack for minutes) context/ # workspace I/O: conventions file, root .gitattributes, changed files, import-trace scan, doc-mention scan, priority docs review/ # pure review logic (the model call is a function the caller passes in): finding schema, phases + stage dispatch (defined under Review instruction authoring), prompt, non-finding filter and unknown-file filter (README sections of those names), cross-phase merge, path normalization, selection, comment mapping, markdown code spans, title similarity, context notes, summary diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 724586db..fc120b3c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -43,8 +43,9 @@ points: constructs SDK clients and touches `process.env`, except that `logger.ts` reads `LOG_LEVEL` - Use the official SDKs (`@actions/core`, `@actions/github`, - `@openrouter/agent`, `@openrouter/sdk`, `parse-diff`) — don't hand-roll - what they already do + `@openrouter/agent` for review requests, `@openrouter/sdk` for the + generation cost lookup, `parse-diff`) — don't hand-roll what they + already do - Relative imports carry explicit `.js` extensions (ESM runtime requirement) - Tests are behavioral specs: one focused `it()` per behavior, exact assertions, `const` per test diff --git a/src/__tests__/orchestrate.test.ts b/src/__tests__/orchestrate.test.ts index e0b8eea2..77688a09 100644 --- a/src/__tests__/orchestrate.test.ts +++ b/src/__tests__/orchestrate.test.ts @@ -4725,7 +4725,7 @@ describe("staged phases", () => { }) const client = createOpenRouterClient({ agent: { callModel }, - sdk: { generations: { getGeneration } }, + costLookupClient: { generations: { getGeneration } }, requestTimeoutMs: 900_000, remainingReviewMs, }) @@ -4816,7 +4816,7 @@ describe("staged phases", () => { .mockImplementation(() => makePendingModelResultStub(lateResponse.promise)) const client = createOpenRouterClient({ agent: { callModel }, - sdk: { generations: { getGeneration: vi.fn() } }, + costLookupClient: { generations: { getGeneration: vi.fn() } }, requestTimeoutMs: 900_000, remainingReviewMs, }) @@ -4924,7 +4924,7 @@ describe("staged phases", () => { }) const client = createOpenRouterClient({ agent: { callModel }, - sdk: { generations: { getGeneration: vi.fn() } }, + costLookupClient: { generations: { getGeneration: vi.fn() } }, requestTimeoutMs: 900_000, remainingReviewMs: () => Infinity, retryDelayMs: 0, diff --git a/src/main.ts b/src/main.ts index 6030de66..4f1ce065 100644 --- a/src/main.ts +++ b/src/main.ts @@ -13,6 +13,8 @@ import { createPromptedGenerateFindings, orchestrate } from "./orchestrate.js" const actionStartedAt = performance.now() const logger = createLogger("umm-actually") +// The action's own unawaited promises cannot reject, so a rejection here comes +// from a promise inside an SDK. Logging it keeps Node from exiting mid-review process.on("unhandledRejection", (error) => { logger.warn("unhandled promise rejection (likely SDK internal)", { error: describeError(error), @@ -21,6 +23,7 @@ process.on("unhandledRejection", (error) => { // Cleanups to run when the job is cancelled. It holds at most one entry, the // open check run's completion call, registered only while that run is open. +// A Set lets the unregister callback delete exactly the cleanup it added. // Mutable on purpose — signal handlers can only reach shared state const cancellationCleanups = new Set<() => Promise>() @@ -138,7 +141,7 @@ try { // passes its own phase-tagged logger to requestReview openrouterClient: createOpenRouterClient({ agent: new OpenRouterAgent({ apiKey: config.openrouterApiKey }), - sdk: new OpenRouter({ apiKey: config.openrouterApiKey }), + costLookupClient: new OpenRouter({ apiKey: config.openrouterApiKey }), requestTimeoutMs: config.requestTimeoutSeconds * 1000, remainingReviewMs, }), diff --git a/src/openrouter/__tests__/client.test.ts b/src/openrouter/__tests__/client.test.ts index de687167..cba11a3b 100644 --- a/src/openrouter/__tests__/client.test.ts +++ b/src/openrouter/__tests__/client.test.ts @@ -168,7 +168,7 @@ const makeClientStubs = ({ }, } - const sdk: GenerationsLike = { + const costLookupClient: GenerationsLike = { generations: { getGeneration: async (request, options) => { generationCalls.push({ ...request, options }) @@ -184,14 +184,14 @@ const makeClientStubs = ({ }, } - return { agent, sdk, modelCalls, generationCalls } + return { agent, costLookupClient, modelCalls, generationCalls } } -const makeClient = (stub: { agent: AgentLike; sdk: GenerationsLike }) => { +const makeClient = (stub: { agent: AgentLike; costLookupClient: GenerationsLike }) => { const logger = createTestLogger() const client = createOpenRouterClient({ agent: stub.agent, - sdk: stub.sdk, + costLookupClient: stub.costLookupClient, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, retryDelayMs: 0, @@ -243,10 +243,10 @@ const makeContextOverflowError = ({ /** The model and output ceiling of every review request sent, in order. */ const sentCeilings = (stub: { modelCalls: { request: ReviewRequest }[] -}): { model: string; maxCompletionTokens: number }[] => { +}): { model: string; maxOutputTokens: number }[] => { return stub.modelCalls.map(({ request }) => ({ model: request.model, - maxCompletionTokens: request.maxOutputTokens, + maxOutputTokens: request.maxOutputTokens, })) } @@ -301,7 +301,7 @@ describe("requestReview", () => { const logger = createTestLogger() const client = createOpenRouterClient({ agent: stub.agent, - sdk: stub.sdk, + costLookupClient: stub.costLookupClient, requestTimeoutMs: 45_000, remainingReviewMs: () => 0, }) @@ -339,7 +339,7 @@ describe("requestReview", () => { const logger = createTestLogger() const client = createOpenRouterClient({ agent: stub.agent, - sdk: stub.sdk, + costLookupClient: stub.costLookupClient, requestTimeoutMs: 45_000, remainingReviewMs: () => deadline - performance.now(), }) @@ -412,7 +412,7 @@ describe("requestReview", () => { const logger = createTestLogger() const client = createOpenRouterClient({ agent: stub.agent, - sdk: stub.sdk, + costLookupClient: stub.costLookupClient, requestTimeoutMs: 45_000, remainingReviewMs: () => deadline - performance.now(), }) @@ -459,7 +459,7 @@ describe("requestReview", () => { const logger = createTestLogger() const client = createOpenRouterClient({ agent: stub.agent, - sdk: stub.sdk, + costLookupClient: stub.costLookupClient, requestTimeoutMs: 45_000, remainingReviewMs, }) @@ -504,7 +504,7 @@ describe("requestReview", () => { const logger = createTestLogger() const client = createOpenRouterClient({ agent: stub.agent, - sdk: stub.sdk, + costLookupClient: stub.costLookupClient, requestTimeoutMs: 45_000, remainingReviewMs: () => deadline - performance.now(), }) @@ -810,7 +810,7 @@ describe("requestReview", () => { const logger = createTestLogger() const client = createOpenRouterClient({ agent: stub.agent, - sdk: stub.sdk, + costLookupClient: stub.costLookupClient, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, retryDelayMs: 0, @@ -888,7 +888,7 @@ describe("requestReview", () => { const logger = createTestLogger() const client = createOpenRouterClient({ agent: stub.agent, - sdk: stub.sdk, + costLookupClient: stub.costLookupClient, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, retryDelayMs: 0, @@ -974,7 +974,7 @@ describe("requestReview", () => { const logger = createTestLogger() const client = createOpenRouterClient({ agent: stub.agent, - sdk: stub.sdk, + costLookupClient: stub.costLookupClient, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, retryDelayMs: 0, @@ -1022,7 +1022,7 @@ describe("requestReview", () => { const logger = createTestLogger() const client = createOpenRouterClient({ agent: stub.agent, - sdk: stub.sdk, + costLookupClient: stub.costLookupClient, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, retryDelayMs: 0, @@ -1078,7 +1078,7 @@ describe("requestReview", () => { const logger = createTestLogger() const client = createOpenRouterClient({ agent: stub.agent, - sdk: stub.sdk, + costLookupClient: stub.costLookupClient, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, retryDelayMs: 0, @@ -1121,7 +1121,7 @@ describe("requestReview", () => { const logger = createTestLogger() const client = createOpenRouterClient({ agent: stub.agent, - sdk: stub.sdk, + costLookupClient: stub.costLookupClient, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, retryDelayMs: 0, @@ -1177,7 +1177,7 @@ describe("requestReview", () => { const logger = createTestLogger() const client = createOpenRouterClient({ agent: stub.agent, - sdk: stub.sdk, + costLookupClient: stub.costLookupClient, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, retryDelayMs: 0, @@ -1241,7 +1241,7 @@ describe("requestReview", () => { const logger = createTestLogger() const client = createOpenRouterClient({ agent: stub.agent, - sdk: stub.sdk, + costLookupClient: stub.costLookupClient, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, retryDelayMs, @@ -1274,7 +1274,7 @@ describe("requestReview", () => { const deadline = performance.now() + 100 const client = createOpenRouterClient({ agent: stub.agent, - sdk: stub.sdk, + costLookupClient: stub.costLookupClient, requestTimeoutMs: 45_000, remainingReviewMs: () => deadline - performance.now(), }) @@ -1318,7 +1318,7 @@ describe("requestReview", () => { const logger = createTestLogger() const client = createOpenRouterClient({ agent: stub.agent, - sdk: stub.sdk, + costLookupClient: stub.costLookupClient, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, retryDelayMs: 0, @@ -1379,7 +1379,7 @@ describe("requestReview", () => { const logger = createTestLogger() const client = createOpenRouterClient({ agent: stub.agent, - sdk: stub.sdk, + costLookupClient: stub.costLookupClient, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, retryDelayMs, @@ -1482,8 +1482,8 @@ describe("requestReview", () => { // 262,144 window − 135,762 input − 8,192 margin expect(sentCeilings(stub)).toEqual([ - { model: "z-ai/glm-5.3-flash", maxCompletionTokens: 128_000 }, - { model: "z-ai/glm-5.3-flash", maxCompletionTokens: 118_190 }, + { model: "z-ai/glm-5.3-flash", maxOutputTokens: 128_000 }, + { model: "z-ai/glm-5.3-flash", maxOutputTokens: 118_190 }, ]) expect(result).toEqual({ review: acceptedReview, @@ -1512,7 +1512,7 @@ describe("requestReview", () => { { level: "info", message: CEILING_RETRY_LOG, - data: { model: "z-ai/glm-5.3-flash", maxCompletionTokens: 118_190 }, + data: { model: "z-ai/glm-5.3-flash", maxOutputTokens: 118_190 }, }, ]) }) @@ -1530,9 +1530,9 @@ describe("requestReview", () => { await client.requestReview(requestParams, logger) expect(sentCeilings(stub)).toEqual([ - { model: "z-ai/glm-5.3-flash", maxCompletionTokens: 128_000 }, - { model: "z-ai/glm-5.3-flash", maxCompletionTokens: 118_190 }, - { model: "deepseek/deepseek-v4.1-flash", maxCompletionTokens: 128_000 }, + { model: "z-ai/glm-5.3-flash", maxOutputTokens: 128_000 }, + { model: "z-ai/glm-5.3-flash", maxOutputTokens: 118_190 }, + { model: "deepseek/deepseek-v4.1-flash", maxOutputTokens: 128_000 }, ]) }) @@ -1548,8 +1548,8 @@ describe("requestReview", () => { await client.requestReview(requestParams, logger) expect(sentCeilings(stub)).toEqual([ - { model: "z-ai/glm-5.3-flash", maxCompletionTokens: 128_000 }, - { model: "deepseek/deepseek-v4.1-flash", maxCompletionTokens: 128_000 }, + { model: "z-ai/glm-5.3-flash", maxOutputTokens: 128_000 }, + { model: "deepseek/deepseek-v4.1-flash", maxOutputTokens: 128_000 }, ]) }) @@ -1566,8 +1566,8 @@ describe("requestReview", () => { await client.requestReview(requestParams, logger) expect(sentCeilings(stub)).toEqual([ - { model: "z-ai/glm-5.3-flash", maxCompletionTokens: 128_000 }, - { model: "deepseek/deepseek-v4.1-flash", maxCompletionTokens: 128_000 }, + { model: "z-ai/glm-5.3-flash", maxOutputTokens: 128_000 }, + { model: "deepseek/deepseek-v4.1-flash", maxOutputTokens: 128_000 }, ]) }) @@ -1584,8 +1584,8 @@ describe("requestReview", () => { await client.requestReview(requestParams, logger) expect(sentCeilings(stub)).toEqual([ - { model: "z-ai/glm-5.3-flash", maxCompletionTokens: 128_000 }, - { model: "z-ai/glm-5.3-flash", maxCompletionTokens: 32_768 }, + { model: "z-ai/glm-5.3-flash", maxOutputTokens: 128_000 }, + { model: "z-ai/glm-5.3-flash", maxOutputTokens: 32_768 }, ]) }) @@ -1603,8 +1603,8 @@ describe("requestReview", () => { await client.requestReview(requestParams, logger) expect(sentCeilings(stub)).toEqual([ - { model: "z-ai/glm-5.3-flash", maxCompletionTokens: 128_000 }, - { model: "deepseek/deepseek-v4.1-flash", maxCompletionTokens: 128_000 }, + { model: "z-ai/glm-5.3-flash", maxOutputTokens: 128_000 }, + { model: "deepseek/deepseek-v4.1-flash", maxOutputTokens: 128_000 }, ]) expect(logsWithMessage(logger, CEILING_RETRY_LOG)).toEqual([]) }) @@ -1622,9 +1622,9 @@ describe("requestReview", () => { await client.requestReview(requestParams, logger) expect(sentCeilings(stub)).toEqual([ - { model: "z-ai/glm-5.3-flash", maxCompletionTokens: 128_000 }, - { model: "z-ai/glm-5.3-flash", maxCompletionTokens: 128_000 }, - { model: "deepseek/deepseek-v4.1-flash", maxCompletionTokens: 128_000 }, + { model: "z-ai/glm-5.3-flash", maxOutputTokens: 128_000 }, + { model: "z-ai/glm-5.3-flash", maxOutputTokens: 128_000 }, + { model: "deepseek/deepseek-v4.1-flash", maxOutputTokens: 128_000 }, ]) expect(logsWithMessage(logger, CEILING_RETRY_LOG)).toEqual([]) }) @@ -1642,8 +1642,8 @@ describe("requestReview", () => { await client.requestReview(requestParams, logger) expect(sentCeilings(stub)).toEqual([ - { model: "z-ai/glm-5.3-flash", maxCompletionTokens: 128_000 }, - { model: "z-ai/glm-5.3-flash", maxCompletionTokens: 128_000 }, + { model: "z-ai/glm-5.3-flash", maxOutputTokens: 128_000 }, + { model: "z-ai/glm-5.3-flash", maxOutputTokens: 128_000 }, ]) }) @@ -1656,8 +1656,8 @@ describe("requestReview", () => { await client.requestReview(requestParams, logger) expect(sentCeilings(stub)).toEqual([ - { model: "z-ai/glm-5.3-flash", maxCompletionTokens: 128_000 }, - { model: "deepseek/deepseek-v4.1-flash", maxCompletionTokens: 128_000 }, + { model: "z-ai/glm-5.3-flash", maxOutputTokens: 128_000 }, + { model: "deepseek/deepseek-v4.1-flash", maxOutputTokens: 128_000 }, ]) }) @@ -1670,8 +1670,8 @@ describe("requestReview", () => { const result = await client.requestReview(requestParams, logger) expect(sentCeilings(stub)).toEqual([ - { model: "z-ai/glm-5.3-flash", maxCompletionTokens: 128_000 }, - { model: "deepseek/deepseek-v4.1-flash", maxCompletionTokens: 128_000 }, + { model: "z-ai/glm-5.3-flash", maxOutputTokens: 128_000 }, + { model: "deepseek/deepseek-v4.1-flash", maxOutputTokens: 128_000 }, ]) expect(result.attempts[0]).toEqual({ model: "z-ai/glm-5.3-flash", @@ -1963,7 +1963,7 @@ describe("requestReview", () => { }, } const { agent } = makeClientStubs({ modelResults: [{ value: makeNoCostResult() }] }) - const { client, logger } = makeClient({ agent, sdk: sdkWithThrowingLookup }) + const { client, logger } = makeClient({ agent, costLookupClient: sdkWithThrowingLookup }) const result = await client.requestReview(requestParams, logger) diff --git a/src/openrouter/client.ts b/src/openrouter/client.ts index 815618de..88893787 100644 --- a/src/openrouter/client.ts +++ b/src/openrouter/client.ts @@ -11,9 +11,10 @@ export type AttemptOutcome = "accepted" | "api_error" | "timeout" | "empty_content" | "invalid_json" | "schema_mismatch" export type ModelAttempt = { - /** The model this attempt requested. The ladder is the primary model, then - * the fallback when one is set. OpenRouter's routed model is reported only - * for the accepted attempt, as StructuredReviewResult.modelUsed. */ + /** The model this attempt requested from the model ladder, which tries the + * primary model and then the fallback when one is set. OpenRouter's routed + * model is reported only for the accepted attempt, as + * StructuredReviewResult.modelUsed. */ model: string outcome: AttemptOutcome promptTokens: number | null @@ -31,8 +32,9 @@ export type StructuredReviewResult = { attempts: ModelAttempt[] } -/** Structural subset of the Agent SDK's call-model input — exactly what we - * send, as one Responses API request with no tools. */ +/** Structural subset of the `callModel` input of the Agent SDK + * (`@openrouter/agent`) — exactly what we send, as one request with no tools + * to OpenRouter's Responses API (`/responses`). */ export type ReviewRequest = { model: string instructions: string @@ -49,10 +51,13 @@ export type ReviewRequest = { } /** - * Structural subset of the Agent SDK client — test stubs are plain objects. - * Method syntax keeps the real client assignable under strictFunctionTypes. - * `getResponse` resolves `unknown` on purpose: Zod parses across the - * boundary. `getText` is the SDK's own extraction of the message text. + * Structural subset of the Agent SDK client. + * - Test stubs are plain objects; method syntax keeps the real client + * assignable under strictFunctionTypes. + * - `getResponse` resolves `unknown` on purpose: Zod parses across the boundary. + * - `getText` is the SDK's own extraction of the message text. + * - The abort signal goes in the request, where the SDK takes the signal that + * cancels the whole call and aborts its in-flight stream. */ export type AgentLike = { callModel( @@ -90,8 +95,9 @@ export type OpenRouterClient = { ) => Promise } -/** The fields read from the Agent SDK's materialized Responses result. The - * SDK types `usage` as both nullable and optional, so the block is nullish. */ +/** The fields read from the Responses result that the Agent SDK assembles + * from the streamed events. The SDK types `usage` as both nullable and + * optional, so the block is nullish. */ const responsesResultSchema = z.object({ id: z.string(), model: z.string(), @@ -131,8 +137,10 @@ const RETRY_DELAY_MS = 1_000 /** The output ceiling, sent as the request's `maxOutputTokens`, for every * attempt except a context-overflow retry. Without one, some providers apply - * a default output limit that cuts a large review off mid-JSON. */ -const MAX_COMPLETION_TOKENS = 128_000 + * a default output limit that cuts a large review off mid-JSON. 128_000 is + * several times what a review usually spends, so in practice only a runaway + * response reaches it. */ +const MAX_OUTPUT_TOKENS = 128_000 /** OpenRouter's 400 when the prompt plus the output ceiling exceeds the routed * endpoint's context window, e.g. "This endpoint's maximum context length is @@ -150,10 +158,12 @@ const CONTEXT_FIT_MARGIN_TOKENS = 8_192 * of thousands of output tokens, reasoning included, so a smaller ceiling * would cut many off mid-JSON and still bill the full prompt. Below it, the * 400 fails like any other structural 4xx. */ -const MIN_FITTED_MAX_COMPLETION_TOKENS = 32_768 +const MIN_FITTED_MAX_OUTPUT_TOKENS = 32_768 -/** OpenRouter SDK errors carry a numeric `statusCode` — duck-typed so stubs - * and future SDK versions need no instanceof on SDK internals. */ +/** HTTP errors from both SDKs carry the response's numeric `statusCode`. + * `@openrouter/agent` throws them from its own bundled copy of + * `@openrouter/sdk`, whose error classes differ from the repo's copy, so the + * check is duck-typed rather than an instanceof. */ const errorStatusCode = (error: unknown): number | undefined => { if (typeof error !== "object" || error === null) return undefined if (!("statusCode" in error) || typeof error.statusCode !== "number") { @@ -165,12 +175,12 @@ const errorStatusCode = (error: unknown): number | undefined => { /** The largest output ceiling that fits the endpoint a context-overflow error * names, or null when the error is something else or no usable smaller * ceiling fits. */ -const fittedMaxCompletionTokens = ({ +const fittedMaxOutputTokens = ({ error, - rejectedMaxCompletionTokens, + rejectedMaxOutputTokens, }: { error: unknown - rejectedMaxCompletionTokens: number + rejectedMaxOutputTokens: number }): number | null => { if (errorStatusCode(error) !== 400 || !(error instanceof Error)) return null @@ -183,11 +193,11 @@ const fittedMaxCompletionTokens = ({ const fittedCeiling = Number(contextLength) - Number(inputTokens) - CONTEXT_FIT_MARGIN_TOKENS // The endpoint leaves too little room for a review to finish its JSON - if (fittedCeiling < MIN_FITTED_MAX_COMPLETION_TOKENS) return null + if (fittedCeiling < MIN_FITTED_MAX_OUTPUT_TOKENS) return null // The estimated input plus the rejected ceiling already fit the window, so // that ceiling did not cause this 400 and lowering it cannot fix it - if (fittedCeiling >= rejectedMaxCompletionTokens) return null + if (fittedCeiling >= rejectedMaxOutputTokens) return null return fittedCeiling } @@ -309,17 +319,17 @@ const buildReviewRequest = ({ systemPrompt, userPrompt, model, - maxCompletionTokens, + maxOutputTokens, }: { systemPrompt: string userPrompt: string model: string - maxCompletionTokens: number + maxOutputTokens: number }): ReviewRequest => ({ model, instructions: systemPrompt, input: userPrompt, - maxOutputTokens: maxCompletionTokens, + maxOutputTokens, text: { format: { type: "json_schema", @@ -348,7 +358,7 @@ type SingleAttempt = | { kind: "context_overflow" attempt: ModelAttempt - fittedMaxCompletionTokens: number + fittedMaxOutputTokens: number } /** Thrown when the model ladder ends without an accepted review; retains every @@ -356,7 +366,7 @@ type SingleAttempt = export class ReviewRequestError extends Error { readonly attempts: ModelAttempt[] /** An auth or credit status (401/402/403) stopped the ladder. The key fails - * for every model, so the stage runner skips the stages after this one. */ + * for every model, so `runStages` skips the review stages after this one. */ readonly keyRejected: boolean /** The whole review's time budget ran out, as reported by the * `remainingReviewMs` that createOpenRouterClient takes. */ @@ -394,13 +404,13 @@ const summarizeAttempts = (attempts: ModelAttempt[]): string => { * concurrent callers can tag their requests' lines with their own props. */ export const createOpenRouterClient = ({ agent, - sdk, + costLookupClient, requestTimeoutMs, remainingReviewMs, retryDelayMs = RETRY_DELAY_MS, }: { agent: AgentLike - sdk: GenerationsLike + costLookupClient: GenerationsLike /** Per-attempt deadline. When it elapses the attempt records outcome * `timeout` and the ladder moves on — to the next model when one * exists, otherwise a same-model retry while attempts remain — whether @@ -438,10 +448,14 @@ export const createOpenRouterClient = ({ // before every attempt, so some review time is always left const callTimeout = getCallTimeout(remainingReviewMs()) - // The Agent SDK streams every request and materializes the result on the - // first read. A "Stream ended without completion event" rejection on every - // model means its bundled SDK copy rejected a changed wire field or - // ignored a stream error event; only the raw stream tells those apart + // The Agent SDK starts the request's stream on the first read and + // assembles the Responses result when the stream completes. + // - A stream that closes without a completion event rejects with "Stream + // ended without completion event", a status-less error that retries + // like a network error. + // - When every model fails that way, the `@openrouter/sdk` copy bundled + // inside `@openrouter/agent` rejected a changed wire field or ignored a + // stream error event; only the raw stream tells those apart. const sendResult = await withDeadline( { start: async (signal) => { @@ -450,6 +464,9 @@ export const createOpenRouterClient = ({ { retries: { strategy: "none" } }, ) const response = await modelResult.getResponse() + + // getText reads the response getResponse already awaited, so it + // sends no second request const text = await modelResult.getText() return { response, text } }, @@ -490,13 +507,13 @@ export const createOpenRouterClient = ({ ? `HTTP ${statusCode}: ${summarizeError(sendResult.error)}` : summarizeError(sendResult.error), } - const fittedCeiling = fittedMaxCompletionTokens({ + const fittedCeiling = fittedMaxOutputTokens({ error: sendResult.error, - rejectedMaxCompletionTokens: reviewRequest.maxOutputTokens, + rejectedMaxOutputTokens: reviewRequest.maxOutputTokens, }) if (fittedCeiling) { - return { kind: "context_overflow", attempt, fittedMaxCompletionTokens: fittedCeiling } + return { kind: "context_overflow", attempt, fittedMaxOutputTokens: fittedCeiling } } const keyRejected = statusCode ? KEY_REJECTED_STATUSES.has(statusCode) : false @@ -574,6 +591,9 @@ export const createOpenRouterClient = ({ kind: "accepted", review: parsedReview.data, routedModel: responsesResult.model, + + // A Responses result's `id` is its generation id (`gen-…`), the id the + // generation cost lookup takes generationId: responsesResult.id, attempt: { model, outcome: "accepted", ...usage, errorSummary: null }, } @@ -600,7 +620,7 @@ export const createOpenRouterClient = ({ const lookup = await withDeadline( { start: (signal) => { - return sdk.generations.getGeneration( + return costLookupClient.generations.getGeneration( { id: generationId }, { retries: { strategy: "none" }, signal }, ) @@ -648,18 +668,26 @@ export const createOpenRouterClient = ({ }) } + // Each model's attempts end in one of four ways: + // - An accepted review returns. + // - A rejected key or a spent review deadline throws, ending the ladder. + // - A timeout with a next model left, or a structural failure, breaks to + // the next model. + // - A failed last attempt ends the while loop and moves to the next model. + // When the last model's attempts end without a review, the throw after the + // loops reports every attempt for (const [ladderIndex, ladderModel] of modelLadder.entries()) { const nextLadderModel = modelLadder[ladderIndex + 1] // Both are reassigned across this model's attempts. Each model starts at // the full ceiling because its own endpoints decide what fits, so a // context overflow lowers the ceiling only for the same model's retry - let maxCompletionTokens = MAX_COMPLETION_TOKENS + let maxOutputTokens = MAX_OUTPUT_TOKENS let modelAttemptNumber = 1 while (modelAttemptNumber <= MAX_ATTEMPTS_PER_MODEL) { ensureReviewTimeRemaining() const attemptResult = await attemptOnce( - buildReviewRequest({ systemPrompt, userPrompt, model: ladderModel, maxCompletionTokens }), + buildReviewRequest({ systemPrompt, userPrompt, model: ladderModel, maxOutputTokens }), logger, ) @@ -727,10 +755,10 @@ export const createOpenRouterClient = ({ // fitted ceiling, so the loop ends and any next model starts at the // full ceiling if (retryRemains && attemptResult.kind === "context_overflow") { - maxCompletionTokens = attemptResult.fittedMaxCompletionTokens + maxOutputTokens = attemptResult.fittedMaxOutputTokens logger.info("retrying with an output ceiling that fits the endpoint's context window", { model: ladderModel, - maxCompletionTokens, + maxOutputTokens, }) } From f521e87931b7bec6deea473587c02947a253acff Mon Sep 17 00:00:00 2001 From: aliasunder <32620895+aliasunder@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:11:33 -0500 Subject: [PATCH 06/20] test(openrouter): cover the routed model and tighten the Agent SDK client tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Assert modelUsed comes from the response's model with a requested model that differs from it; every other test requested the model the fixture reports, so returning the requested model passed them all. - Drop the per-fixture transform cases: the module-scope transform throws at collection first, so those cases could never be the failing test. - Assert the SDK-thrown TimeoutError test's calls in one positional toEqual over model, signal and retry options. - Rename sdkWithThrowingLookup to throwingCostLookupClient. Ship-Check: test-audit · claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- src/openrouter/__tests__/client.test.ts | 65 +++++++++++++++++++------ 1 file changed, 49 insertions(+), 16 deletions(-) diff --git a/src/openrouter/__tests__/client.test.ts b/src/openrouter/__tests__/client.test.ts index cba11a3b..fc8d8999 100644 --- a/src/openrouter/__tests__/client.test.ts +++ b/src/openrouter/__tests__/client.test.ts @@ -270,14 +270,6 @@ const requestParams = { } describe("captured Responses fixtures", () => { - it.each([ - { label: "accepted", payload: acceptedWirePayload }, - { label: "invalid JSON", payload: invalidJsonWirePayload }, - { label: "schema mismatch", payload: schemaMismatchWirePayload }, - ])("the $label wire payload passes the SDK's result transform", ({ payload }) => { - expect(openResponsesResultFromJSON(JSON.stringify(payload)).ok).toBe(true) - }) - it("the overflow and bad-key bodies build the SDK's error classes with their status and message", () => { const overflowError = makeBadRequestError(overflowErrorBody) const badKeyError = new UnauthorizedResponseError( @@ -592,6 +584,41 @@ describe("requestReview", () => { }) }) + it("reports the response's routed model as modelUsed when it differs from the requested model", async () => { + const stub = makeClientStubs({ modelResults: [{ value: acceptedResult }] }) + const { client, logger } = makeClient(stub) + + const result = await client.requestReview( + { ...requestParams, model: "openrouter/auto", fallbackModel: null }, + logger, + ) + + expect(result).toEqual({ + review: acceptedReview, + modelUsed: "z-ai/glm-5.3-flash", + attempts: [ + { + model: "openrouter/auto", + outcome: "accepted", + ...ACCEPTED_USAGE, + errorSummary: null, + }, + ], + }) + expect(logsWithMessage(logger, "review response accepted")).toEqual([ + { + level: "info", + message: "review response accepted", + data: { + model: "openrouter/auto", + routedModel: "z-ai/glm-5.3-flash", + generationId: ACCEPTED_GENERATION_ID, + totalAttemptCount: 1, + }, + }, + ]) + }) + it("retries the same model once on invalid JSON, recording both attempts", async () => { const stub = makeClientStubs({ modelResults: [{ value: invalidJsonResult }, { value: acceptedResult }], @@ -745,12 +772,18 @@ describe("requestReview", () => { const result = await client.requestReview(requestParams, logger) - expect(stub.modelCalls.map((modelCall) => modelCall.request.model)).toEqual([ - "z-ai/glm-5.3-flash", - "z-ai/glm-5.3-flash", - ]) - expect(stub.modelCalls[0]?.options).toEqual({ retries: { strategy: "none" } }) - expect(stub.modelCalls[0]?.request.signal).toBeInstanceOf(AbortSignal) + const actionLadderCall = { + model: "z-ai/glm-5.3-flash", + signal: expect.any(AbortSignal), + options: { retries: { strategy: "none" } }, + } + expect( + stub.modelCalls.map(({ request, options }) => ({ + model: request.model, + signal: request.signal, + options, + })), + ).toEqual([actionLadderCall, actionLadderCall]) expect(result.attempts[0]).toEqual({ model: "z-ai/glm-5.3-flash", outcome: "api_error", @@ -1955,7 +1988,7 @@ describe("requestReview", () => { }) it("degrades to a null cost with a warning when the generation lookup throws synchronously", async () => { - const sdkWithThrowingLookup: GenerationsLike = { + const throwingCostLookupClient: GenerationsLike = { generations: { getGeneration: () => { throw new Error("lookup threw before returning a promise") @@ -1963,7 +1996,7 @@ describe("requestReview", () => { }, } const { agent } = makeClientStubs({ modelResults: [{ value: makeNoCostResult() }] }) - const { client, logger } = makeClient({ agent, costLookupClient: sdkWithThrowingLookup }) + const { client, logger } = makeClient({ agent, costLookupClient: throwingCostLookupClient }) const result = await client.requestReview(requestParams, logger) From 7636dcdfcbc06a7d6e3dd9f16d153c28e4926f9d Mon Sep 17 00:00:00 2001 From: aliasunder <32620895+aliasunder@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:12:22 -0500 Subject: [PATCH 07/20] test(openrouter): assert whole attempts in cost tests and the cleared timer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - The cost lookup tests assert the full accepted attempt instead of picking costUsd off the first one. - The timer test asserts no timer is left pending under fake timers; a clearTimeout spy passed on any clearTimeout call anywhere. Ship-Check: test-audit · claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- src/openrouter/__tests__/client.test.ts | 67 ++++++++++++++++++++----- 1 file changed, 54 insertions(+), 13 deletions(-) diff --git a/src/openrouter/__tests__/client.test.ts b/src/openrouter/__tests__/client.test.ts index fc8d8999..6d0c6fad 100644 --- a/src/openrouter/__tests__/client.test.ts +++ b/src/openrouter/__tests__/client.test.ts @@ -1361,7 +1361,15 @@ describe("requestReview", () => { await vi.advanceTimersByTimeAsync(45_000) const result = await reviewPromise - expect(result.attempts[0]?.costUsd).toBeNull() + expect(result.attempts).toEqual([ + { + model: requestParams.model, + outcome: "accepted", + ...ACCEPTED_USAGE, + costUsd: null, + errorSummary: null, + }, + ]) expect(logsWithMessage(logger, "request deadline elapsed")).toEqual([ { level: "warn", @@ -1390,17 +1398,18 @@ describe("requestReview", () => { }) it("clears the review attempt timeout timer after a successful response", async () => { - const stub = makeClientStubs({ - modelResults: [{ value: acceptedResult }], - }) - const { client, logger } = makeClient(stub) - - const clearTimeoutSpy = vi.spyOn(globalThis, "clearTimeout") + vi.useFakeTimers() try { + const stub = makeClientStubs({ + modelResults: [{ value: acceptedResult }], + }) + const { client, logger } = makeClient(stub) + await client.requestReview(requestParams, logger) - expect(clearTimeoutSpy).toHaveBeenCalled() + + expect(vi.getTimerCount()).toBe(0) } finally { - clearTimeoutSpy.mockRestore() + vi.useRealTimers() } }) @@ -1961,7 +1970,15 @@ describe("requestReview", () => { }, }, ]) - expect(result.attempts[0]?.costUsd).toBe(0.0399) + expect(result.attempts).toEqual([ + { + model: requestParams.model, + outcome: "accepted", + ...ACCEPTED_USAGE, + costUsd: 0.0399, + errorSummary: null, + }, + ]) }) it("degrades to a null cost with a warning when the generation lookup fails", async () => { @@ -1973,7 +1990,15 @@ describe("requestReview", () => { const result = await client.requestReview(requestParams, logger) - expect(result.attempts[0]?.costUsd).toBeNull() + expect(result.attempts).toEqual([ + { + model: requestParams.model, + outcome: "accepted", + ...ACCEPTED_USAGE, + costUsd: null, + errorSummary: null, + }, + ]) expect(logsWithMessage(logger, "generation cost lookup failed")).toEqual([ { level: "warn", @@ -2000,7 +2025,15 @@ describe("requestReview", () => { const result = await client.requestReview(requestParams, logger) - expect(result.attempts[0]?.costUsd).toBeNull() + expect(result.attempts).toEqual([ + { + model: requestParams.model, + outcome: "accepted", + ...ACCEPTED_USAGE, + costUsd: null, + errorSummary: null, + }, + ]) expect(logsWithMessage(logger, "generation cost lookup failed")).toEqual([ { level: "warn", @@ -2023,7 +2056,15 @@ describe("requestReview", () => { const result = await client.requestReview(requestParams, logger) - expect(result.attempts[0]?.costUsd).toBeNull() + expect(result.attempts).toEqual([ + { + model: requestParams.model, + outcome: "accepted", + ...ACCEPTED_USAGE, + costUsd: null, + errorSummary: null, + }, + ]) expect(logsWithMessage(logger, "unexpected generation response shape")).toEqual([ { level: "warn", From 8c178c51ee34d769504a5923812f5c286d6bfa79 Mon Sep 17 00:00:00 2001 From: aliasunder <32620895+aliasunder@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:22:37 -0500 Subject: [PATCH 08/20] fix(openrouter): describe where the Agent SDK gets the Responses result MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SDK takes the result from the stream's completion event rather than assembling it from the streamed events, and the generation cost lookup is the one endpoint this client calls that the Agent SDK does not cover. Ship-Check: bug-check · claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- src/openrouter/client.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/openrouter/client.ts b/src/openrouter/client.ts index 88893787..24935ef6 100644 --- a/src/openrouter/client.ts +++ b/src/openrouter/client.ts @@ -67,7 +67,8 @@ export type AgentLike = { } /** Structural subset of the OpenRouter SDK's REST client — the generation - * cost lookup is the one endpoint the Agent SDK does not cover. */ + * cost lookup is the one endpoint this client calls that the Agent SDK does + * not cover. */ export type GenerationsLike = { generations: { getGeneration( @@ -95,8 +96,8 @@ export type OpenRouterClient = { ) => Promise } -/** The fields read from the Responses result that the Agent SDK assembles - * from the streamed events. The SDK types `usage` as both nullable and +/** The fields read from the Responses result that the Agent SDK takes from + * the stream's completion event. The SDK types `usage` as both nullable and * optional, so the block is nullish. */ const responsesResultSchema = z.object({ id: z.string(), @@ -448,8 +449,8 @@ export const createOpenRouterClient = ({ // before every attempt, so some review time is always left const callTimeout = getCallTimeout(remainingReviewMs()) - // The Agent SDK starts the request's stream on the first read and - // assembles the Responses result when the stream completes. + // The Agent SDK starts the request's stream on the first read and takes + // the Responses result from the stream's completion event. // - A stream that closes without a completion event rejects with "Stream // ended without completion event", a status-less error that retries // like a network error. From c4bd39e8cafae4133e8f68dffec506ff884b1d80 Mon Sep 17 00:00:00 2001 From: aliasunder <32620895+aliasunder@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:46:27 -0500 Subject: [PATCH 09/20] refactor(openrouter): read the Responses result through the SDK's own validation The Agent SDK validates the completion event before getResponse resolves, so the client reads id, model and usage through a structural type instead of parsing them a second time. The test stub for an unrecognized shape, which production cannot produce, goes with the branch it exercised. Co-Authored-By: Claude Fable 5.1 --- src/__tests__/orchestrate.test.ts | 13 +++--- src/openrouter/__tests__/client.test.ts | 26 ++--------- src/openrouter/client.ts | 58 +++++++++---------------- 3 files changed, 29 insertions(+), 68 deletions(-) diff --git a/src/__tests__/orchestrate.test.ts b/src/__tests__/orchestrate.test.ts index 77688a09..96fdebe7 100644 --- a/src/__tests__/orchestrate.test.ts +++ b/src/__tests__/orchestrate.test.ts @@ -10,6 +10,7 @@ import { createOpenRouterClient, type ModelAttempt, type OpenRouterClient, + type ResponsesResult, type ReviewRequest, type StructuredReviewResult, } from "../openrouter/client.js" @@ -63,24 +64,22 @@ const fixtureReviewResponse: ReviewResponse = JSON.parse( /** The pair a `callModel` stub hands back once its request settles: the * materialized response the client parses and the text the SDK extracts. */ -type ModelCallResult = { response: unknown; text: string } +type ModelCallResult = { response: ResponsesResult; text: string } /** A `callModel` result that settles with the given promise. Both reads await * the one promise, the way the SDK's reads share one materialized stream. */ const makePendingModelResultStub = ( settled: Promise, -): { getResponse(): Promise; getText(): Promise } => ({ +): { getResponse(): Promise; getText(): Promise } => ({ getResponse: async () => (await settled).response, getText: async () => (await settled).text, }) /** A `callModel` result that resolves at once with an accepted-shaped response * carrying the fixture review as its text. */ -const makeModelResultStub = (response: { - id: string - model: string - usage: { inputTokens: number; outputTokens: number; cost?: number } -}): { getResponse(): Promise; getText(): Promise } => { +const makeModelResultStub = ( + response: ResponsesResult, +): { getResponse(): Promise; getText(): Promise } => { return makePendingModelResultStub( Promise.resolve({ response, text: JSON.stringify(fixtureReviewResponse) }), ) diff --git a/src/openrouter/__tests__/client.test.ts b/src/openrouter/__tests__/client.test.ts index 6d0c6fad..fc0c84ab 100644 --- a/src/openrouter/__tests__/client.test.ts +++ b/src/openrouter/__tests__/client.test.ts @@ -14,6 +14,7 @@ import { ReviewRequestError, type AgentLike, type GenerationsLike, + type ResponsesResult, type ReviewRequest, } from "../client.js" @@ -73,7 +74,7 @@ const extractMessageText = (payload: unknown): string => { * through the SDK's own transform. The repo's @openrouter/sdk copy runs it * here as a drift guard on hand-edited fixtures; the Agent SDK's bundled * copy parses every field the client reads identically. */ -const transformWirePayload = (payload: unknown): unknown => { +const transformWirePayload = (payload: unknown): ResponsesResult => { const transformed = openResponsesResultFromJSON(JSON.stringify(payload)) if (!transformed.ok) throw transformed.error @@ -82,7 +83,7 @@ const transformWirePayload = (payload: unknown): unknown => { /** What one `callModel` stub call resolves: the transformed response and the * SDK-extracted text, as the client reads them. */ -type ModelCallResult = { response: unknown; text: string } +type ModelCallResult = { response: ResponsesResult; text: string } const toModelCallResult = (payload: unknown): ModelCallResult => ({ response: transformWirePayload(payload), @@ -1931,27 +1932,6 @@ describe("requestReview", () => { expect(stub.modelCalls).toHaveLength(4) }) - it("treats an unrecognized review response shape as a retryable api_error", async () => { - const stub = makeClientStubs({ - modelResults: [ - { value: { response: { unexpected: true }, text: acceptedResult.text } }, - { value: acceptedResult }, - ], - }) - const { client, logger } = makeClient(stub) - - const result = await client.requestReview(requestParams, logger) - - expect(result.attempts[0]).toEqual({ - model: "z-ai/glm-5.3-flash", - outcome: "api_error", - promptTokens: null, - completionTokens: null, - costUsd: null, - errorSummary: "unexpected review response shape", - }) - }) - it("looks up the generation cost when the accepted attempt has no usage cost", async () => { const stub = makeClientStubs({ modelResults: [{ value: makeNoCostResult() }], diff --git a/src/openrouter/client.ts b/src/openrouter/client.ts index 24935ef6..fbed1fb9 100644 --- a/src/openrouter/client.ts +++ b/src/openrouter/client.ts @@ -50,11 +50,28 @@ export type ReviewRequest = { } } +/** The fields read from the Responses result the Agent SDK takes from the + * stream's completion event. The SDK's own schema validates the result + * before it resolves, so the client reads it through this structural type + * rather than parsing it again; the SDK types `usage` as both nullable and + * optional. */ +export type ResponsesResult = { + id: string + model: string + usage?: + | { + inputTokens: number + outputTokens: number + cost?: number | null | undefined + } + | null + | undefined +} + /** * Structural subset of the Agent SDK client. * - Test stubs are plain objects; method syntax keeps the real client * assignable under strictFunctionTypes. - * - `getResponse` resolves `unknown` on purpose: Zod parses across the boundary. * - `getText` is the SDK's own extraction of the message text. * - The abort signal goes in the request, where the SDK takes the signal that * cancels the whole call and aborts its in-flight stream. @@ -63,7 +80,7 @@ export type AgentLike = { callModel( request: ReviewRequest & { signal: AbortSignal }, options?: { retries?: { strategy: "backoff" | "none" } }, - ): { getResponse(): Promise; getText(): Promise } + ): { getResponse(): Promise; getText(): Promise } } /** Structural subset of the OpenRouter SDK's REST client — the generation @@ -96,21 +113,6 @@ export type OpenRouterClient = { ) => Promise } -/** The fields read from the Responses result that the Agent SDK takes from - * the stream's completion event. The SDK types `usage` as both nullable and - * optional, so the block is nullish. */ -const responsesResultSchema = z.object({ - id: z.string(), - model: z.string(), - usage: z - .object({ - inputTokens: z.number(), - outputTokens: z.number(), - cost: z.number().nullish(), - }) - .nullish(), -}) - const generationResponseSchema = z.object({ data: z.object({ totalCost: z.number() }), }) @@ -527,27 +529,7 @@ export const createOpenRouterClient = ({ return { kind: "failed", attempt, retryable, keyRejected } } - const parsedResult = responsesResultSchema.safeParse(sendResult.value.response) - - // A malformed response body points at a provider or gateway glitch rather - // than the request, so a second call can come back well-formed - if (!parsedResult.success) { - return { - kind: "failed", - attempt: { - model, - outcome: "api_error", - promptTokens: null, - completionTokens: null, - costUsd: null, - errorSummary: "unexpected review response shape", - }, - retryable: true, - keyRejected: false, - } - } - - const responsesResult = parsedResult.data + const responsesResult = sendResult.value.response // Failed-validation attempts still carry usage — they were billed too const usage = { From 01401b3a1f57774866e1e5403cb744688347db15 Mon Sep 17 00:00:00 2001 From: aliasunder <32620895+aliasunder@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:49:56 -0500 Subject: [PATCH 10/20] test(orchestrate): say the client reads the stubbed Responses result MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The client no longer parses the result; the stub comment now matches. Ship-Check: test-audit · claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- src/__tests__/orchestrate.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/__tests__/orchestrate.test.ts b/src/__tests__/orchestrate.test.ts index 96fdebe7..e7160003 100644 --- a/src/__tests__/orchestrate.test.ts +++ b/src/__tests__/orchestrate.test.ts @@ -63,7 +63,7 @@ const fixtureReviewResponse: ReviewResponse = JSON.parse( ) /** The pair a `callModel` stub hands back once its request settles: the - * materialized response the client parses and the text the SDK extracts. */ + * materialized response the client reads and the text the SDK extracts. */ type ModelCallResult = { response: ResponsesResult; text: string } /** A `callModel` result that settles with the given promise. Both reads await From 419fda97f9d25b1e945f86f517d8254869554321 Mon Sep 17 00:00:00 2001 From: aliasunder <32620895+aliasunder@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:53:15 -0500 Subject: [PATCH 11/20] test(openrouter): name what the client stubs stand in for The text extractor and the status errors in the client tests mirror the Agent SDK's bundled copy rather than calling it, so the comments say which contract each test pins, and the empty-content case is named for the mapping it asserts. Co-Authored-By: Claude Fable 5.1 --- src/openrouter/__tests__/client.test.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/openrouter/__tests__/client.test.ts b/src/openrouter/__tests__/client.test.ts index fc0c84ab..8d3c483d 100644 --- a/src/openrouter/__tests__/client.test.ts +++ b/src/openrouter/__tests__/client.test.ts @@ -61,7 +61,9 @@ const parseAcceptedPayload = (): z.infer => { } /** The text the Agent SDK extracts from a payload: the first message item's - * output text, or "" when the output holds no message item. */ + * output text, or "" when the output holds no message item. This mirrors the + * SDK's convention rather than calling it, so the tests pin the client's + * text-to-outcome mapping, not the SDK's extraction. */ const extractMessageText = (payload: unknown): string => { const message = wirePayloadSchema.parse(payload).output.find((item) => item.type === "message") const outputText = message?.content?.find((part) => part.type === "output_text") @@ -212,7 +214,11 @@ const makeHttpMeta = ( body, }) -/** An SDK error of the given status, as the SDK's base class carries it. */ +/** An SDK error of the given status, as the SDK's base class carries it. The + * repo's @openrouter/sdk copy stands in for the copy bundled inside + * @openrouter/agent, which throws the production errors; both set + * `statusCode` from the HTTP response. An error without `statusCode` takes + * the status-less path the network-error test pins. */ const makeStatusError = (statusCode: number, message = `HTTP ${statusCode}`): OpenRouterError => { return new OpenRouterError(message, makeHttpMeta(statusCode, "")) } @@ -652,7 +658,7 @@ describe("requestReview", () => { ]) }) - it("retries the same model once when the output holds no message item", async () => { + it("records empty_content and retries the same model once when the SDK extracts no text", async () => { const stub = makeClientStubs({ modelResults: [{ value: makeReasoningOnlyResult() }, { value: acceptedResult }], }) From 26f3f9338389dc2867e6abe9af6a2a436caa3e98 Mon Sep 17 00:00:00 2001 From: aliasunder <32620895+aliasunder@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:56:20 -0500 Subject: [PATCH 12/20] fix(openrouter): retry the status-200 error for a rejected non-streamed body MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Agent SDK accepts a non-streamed JSON body when the API answers without a stream, and throws a status-200 error when that body fails its result schema. The retry rule treated every status outside 5xx, 408 and 429 as structural, so that error moved straight to the fallback model, while the same malformed result over a stream fails status-less and retries. Only a 4xx other than 408 or 429 is structural now. Ship-Check: bug-check · claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- src/openrouter/__tests__/client.test.ts | 25 +++++++++++++++++++++++++ src/openrouter/client.ts | 16 ++++++++++------ 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/src/openrouter/__tests__/client.test.ts b/src/openrouter/__tests__/client.test.ts index 8d3c483d..95a0a6fd 100644 --- a/src/openrouter/__tests__/client.test.ts +++ b/src/openrouter/__tests__/client.test.ts @@ -747,6 +747,31 @@ describe("requestReview", () => { }) }) + it("retries the same model once when the SDK rejects a non-streamed 200 body", async () => { + const stub = makeClientStubs({ + modelResults: [ + { error: makeStatusError(200, "Response validation failed") }, + { value: acceptedResult }, + ], + }) + const { client, logger } = makeClient(stub) + + const result = await client.requestReview(requestParams, logger) + + expect(stub.modelCalls.map((modelCall) => modelCall.request.model)).toEqual([ + "z-ai/glm-5.3-flash", + "z-ai/glm-5.3-flash", + ]) + expect(result.attempts[0]).toEqual({ + model: "z-ai/glm-5.3-flash", + outcome: "api_error", + promptTokens: null, + completionTokens: null, + costUsd: null, + errorSummary: "HTTP 200: Response validation failed", + }) + }) + it("retries a status-less network error and records its bare message", async () => { const stub = makeClientStubs({ modelResults: [{ error: new Error("socket hang up") }, { value: acceptedResult }], diff --git a/src/openrouter/client.ts b/src/openrouter/client.ts index fbed1fb9..6ea63dfb 100644 --- a/src/openrouter/client.ts +++ b/src/openrouter/client.ts @@ -521,12 +521,16 @@ export const createOpenRouterClient = ({ const keyRejected = statusCode ? KEY_REJECTED_STATUSES.has(statusCode) : false - // An unknown status, a 5xx, a 408, or a 429 is transient and retries; - // any other 4xx, such as a 400 bad request or a 401/402/403 auth or - // credit status, is structural and does not - const retryable = !statusCode || statusCode >= 500 || RETRYABLE_4XX_STATUSES.has(statusCode) - - return { kind: "failed", attempt, retryable, keyRejected } + // A 4xx other than a 408 or a 429, such as a 400 bad request or a + // 401/402/403 auth or credit status, is structural and does not retry. + // Every other failure retries. That includes the status-200 error the SDK + // throws when a non-streamed body fails its result schema, because a + // second call can come back well-formed + const isStructuralClientError = statusCode + ? statusCode >= 400 && statusCode < 500 && !RETRYABLE_4XX_STATUSES.has(statusCode) + : false + + return { kind: "failed", attempt, retryable: !isStructuralClientError, keyRejected } } const responsesResult = sendResult.value.response From ed52c13994b00053c8107b07f39f944ef5f71043 Mon Sep 17 00:00:00 2001 From: aliasunder <32620895+aliasunder@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:58:04 -0500 Subject: [PATCH 13/20] feat(openrouter): log a successful generation cost lookup An accepted response without usage.cost sends the client to the generation endpoint, and nothing recorded how often that happens. The info line gives the run logs the rate that decides whether the fallback stays. Co-Authored-By: Claude Fable 5.1 --- src/openrouter/__tests__/client.test.ts | 11 +++++++++++ src/openrouter/client.ts | 7 ++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/openrouter/__tests__/client.test.ts b/src/openrouter/__tests__/client.test.ts index 95a0a6fd..d0db119c 100644 --- a/src/openrouter/__tests__/client.test.ts +++ b/src/openrouter/__tests__/client.test.ts @@ -1990,6 +1990,17 @@ describe("requestReview", () => { errorSummary: null, }, ]) + expect(logsWithMessage(logger, "generation cost looked up")).toEqual([ + { + level: "info", + message: "generation cost looked up", + data: { + operation: "generation cost lookup", + generationId: ACCEPTED_GENERATION_ID, + costUsd: 0.0399, + }, + }, + ]) }) it("degrades to a null cost with a warning when the generation lookup fails", async () => { diff --git a/src/openrouter/client.ts b/src/openrouter/client.ts index 6ea63dfb..39290715 100644 --- a/src/openrouter/client.ts +++ b/src/openrouter/client.ts @@ -636,7 +636,12 @@ export const createOpenRouterClient = ({ } // parsed.data = Zod safeParse result; .data = API envelope's data property - return parsed.data.data.totalCost + const costUsd = parsed.data.data.totalCost + + // Logged at info because how often an accepted response arrives without + // its cost is what decides whether this fallback stays + lookupLogger.info("generation cost looked up", { costUsd }) + return costUsd } const requestReview: OpenRouterClient["requestReview"] = async ( From c3cea6b17a6e3988b218ade55fca756e0e661bd1 Mon Sep 17 00:00:00 2001 From: aliasunder <32620895+aliasunder@users.noreply.github.com> Date: Wed, 11 Nov 2026 20:02:41 -0500 Subject: [PATCH 14/20] test(openrouter): pin the 408 retry and assert every retry test's full attempt list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 200-body retry test now throws the SDK's own ResponseValidationError. The 408 status shares the 429 retry test, and the index-picked attempt assertions in the retry and truncation tests now assert the whole list. Ship-Check: test-audit · claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- src/openrouter/__tests__/client.test.ts | 163 +++++++++++++++++------- 1 file changed, 115 insertions(+), 48 deletions(-) diff --git a/src/openrouter/__tests__/client.test.ts b/src/openrouter/__tests__/client.test.ts index d0db119c..6a032dc4 100644 --- a/src/openrouter/__tests__/client.test.ts +++ b/src/openrouter/__tests__/client.test.ts @@ -3,6 +3,7 @@ import { openResponsesResultFromJSON } from "@openrouter/sdk/models" import { BadRequestResponseError, OpenRouterError, + ResponseValidationError, UnauthorizedResponseError, } from "@openrouter/sdk/models/errors" import { describe, expect, it, vi } from "vitest" @@ -223,6 +224,20 @@ const makeStatusError = (statusCode: number, message = `HTTP ${statusCode}`): Op return new OpenRouterError(message, makeHttpMeta(statusCode, "")) } +/** The error the SDK throws when a non-streamed 200 body fails its result + * schema, built the way the SDK's response matcher builds it. */ +const makeResponseValidationError = (): ResponseValidationError => { + const body = JSON.stringify({ object: "response" }) + const message = "Response validation failed" + + return new ResponseValidationError(message, { + ...makeHttpMeta(200, body), + cause: new Error("output: Invalid input: expected array, received undefined"), + rawValue: JSON.parse(body), + rawMessage: message, + }) +} + /** The `{ error: { code, message } }` body OpenRouter sends on a rejected request. */ const errorBodySchema = z.object({ error: z.object({ code: z.number(), message: z.string() }) }) @@ -725,9 +740,9 @@ describe("requestReview", () => { ]) }) - it("retries the same model once on a 429", async () => { + it.each([408, 429])("retries the same model once on a %i", async (statusCode) => { const stub = makeClientStubs({ - modelResults: [{ error: makeStatusError(429) }, { value: acceptedResult }], + modelResults: [{ error: makeStatusError(statusCode) }, { value: acceptedResult }], }) const { client, logger } = makeClient(stub) @@ -737,22 +752,27 @@ describe("requestReview", () => { "z-ai/glm-5.3-flash", "z-ai/glm-5.3-flash", ]) - expect(result.attempts[0]).toEqual({ - model: "z-ai/glm-5.3-flash", - outcome: "api_error", - promptTokens: null, - completionTokens: null, - costUsd: null, - errorSummary: "HTTP 429: HTTP 429", - }) + expect(result.attempts).toEqual([ + { + model: "z-ai/glm-5.3-flash", + outcome: "api_error", + promptTokens: null, + completionTokens: null, + costUsd: null, + errorSummary: `HTTP ${statusCode}: HTTP ${statusCode}`, + }, + { + model: "z-ai/glm-5.3-flash", + outcome: "accepted", + ...ACCEPTED_USAGE, + errorSummary: null, + }, + ]) }) it("retries the same model once when the SDK rejects a non-streamed 200 body", async () => { const stub = makeClientStubs({ - modelResults: [ - { error: makeStatusError(200, "Response validation failed") }, - { value: acceptedResult }, - ], + modelResults: [{ error: makeResponseValidationError() }, { value: acceptedResult }], }) const { client, logger } = makeClient(stub) @@ -762,14 +782,22 @@ describe("requestReview", () => { "z-ai/glm-5.3-flash", "z-ai/glm-5.3-flash", ]) - expect(result.attempts[0]).toEqual({ - model: "z-ai/glm-5.3-flash", - outcome: "api_error", - promptTokens: null, - completionTokens: null, - costUsd: null, - errorSummary: "HTTP 200: Response validation failed", - }) + expect(result.attempts).toEqual([ + { + model: "z-ai/glm-5.3-flash", + outcome: "api_error", + promptTokens: null, + completionTokens: null, + costUsd: null, + errorSummary: "HTTP 200: Response validation failed", + }, + { + model: "z-ai/glm-5.3-flash", + outcome: "accepted", + ...ACCEPTED_USAGE, + errorSummary: null, + }, + ]) }) it("retries a status-less network error and records its bare message", async () => { @@ -784,14 +812,22 @@ describe("requestReview", () => { "z-ai/glm-5.3-flash", "z-ai/glm-5.3-flash", ]) - expect(result.attempts[0]).toEqual({ - model: "z-ai/glm-5.3-flash", - outcome: "api_error", - promptTokens: null, - completionTokens: null, - costUsd: null, - errorSummary: "socket hang up", - }) + expect(result.attempts).toEqual([ + { + model: "z-ai/glm-5.3-flash", + outcome: "api_error", + promptTokens: null, + completionTokens: null, + costUsd: null, + errorSummary: "socket hang up", + }, + { + model: "z-ai/glm-5.3-flash", + outcome: "accepted", + ...ACCEPTED_USAGE, + errorSummary: null, + }, + ]) }) it("retries an SDK-thrown TimeoutError as an api_error through the action's ladder, not the SDK's", async () => { @@ -816,14 +852,22 @@ describe("requestReview", () => { options, })), ).toEqual([actionLadderCall, actionLadderCall]) - expect(result.attempts[0]).toEqual({ - model: "z-ai/glm-5.3-flash", - outcome: "api_error", - promptTokens: null, - completionTokens: null, - costUsd: null, - errorSummary: "Request timed out", - }) + expect(result.attempts).toEqual([ + { + model: "z-ai/glm-5.3-flash", + outcome: "api_error", + promptTokens: null, + completionTokens: null, + costUsd: null, + errorSummary: "Request timed out", + }, + { + model: "z-ai/glm-5.3-flash", + outcome: "accepted", + ...ACCEPTED_USAGE, + errorSummary: null, + }, + ]) expect(logsWithMessage(logger, "review attempt failed")).toEqual([ { level: "warn", @@ -1508,7 +1552,22 @@ describe("requestReview", () => { const result = await client.requestReview(requestParams, logger) - expect(result.attempts[0]?.errorSummary).toBe(`HTTP 429: ${"x".repeat(200)}…`) + expect(result.attempts).toEqual([ + { + model: "z-ai/glm-5.3-flash", + outcome: "api_error", + promptTokens: null, + completionTokens: null, + costUsd: null, + errorSummary: `HTTP 429: ${"x".repeat(200)}…`, + }, + { + model: "z-ai/glm-5.3-flash", + outcome: "accepted", + ...ACCEPTED_USAGE, + errorSummary: null, + }, + ]) }) it("advances to the fallback model without a retry on a 404", async () => { @@ -1747,15 +1806,23 @@ describe("requestReview", () => { { model: "z-ai/glm-5.3-flash", maxOutputTokens: 128_000 }, { model: "deepseek/deepseek-v4.1-flash", maxOutputTokens: 128_000 }, ]) - expect(result.attempts[0]).toEqual({ - model: "z-ai/glm-5.3-flash", - outcome: "api_error", - promptTokens: null, - completionTokens: null, - costUsd: null, - errorSummary: - "HTTP 400: This endpoint's maximum context length is 8192 tokens. However, you requested about 11593 tokens (4593 of text input, 7000 in the output). Please reduce the length of either one, or use the context-co…", - }) + expect(result.attempts).toEqual([ + { + model: "z-ai/glm-5.3-flash", + outcome: "api_error", + promptTokens: null, + completionTokens: null, + costUsd: null, + errorSummary: + "HTTP 400: This endpoint's maximum context length is 8192 tokens. However, you requested about 11593 tokens (4593 of text input, 7000 in the output). Please reduce the length of either one, or use the context-co…", + }, + { + model: "deepseek/deepseek-v4.1-flash", + outcome: "accepted", + ...ACCEPTED_USAGE, + errorSummary: null, + }, + ]) }) it("aborts the ladder after exactly one call on the captured bad-key body", async () => { From 429000a5110d7d947193c2e2c1e6b939b26fbbb2 Mon Sep 17 00:00:00 2001 From: aliasunder <32620895+aliasunder@users.noreply.github.com> Date: Wed, 11 Nov 2026 20:03:05 -0500 Subject: [PATCH 15/20] docs(openrouter): state the retry rule's one structural class in its constant's doc Co-Authored-By: Claude Fable 5.1 --- src/openrouter/client.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/openrouter/client.ts b/src/openrouter/client.ts index 39290715..2b0a2d57 100644 --- a/src/openrouter/client.ts +++ b/src/openrouter/client.ts @@ -122,9 +122,9 @@ const generationResponseSchema = z.object({ const KEY_REJECTED_STATUSES = new Set([401, 402, 403]) /** The transient 4xx statuses — HTTP request timeout (408) and rate limit - * (429). attemptOnce also marks 5xx and status-less network errors - * retryable, and a context-overflow 400 retries with a smaller output - * ceiling; any other 4xx is structural and skips the retry. */ + * (429). Any other 4xx is structural and skips the retry, except a + * context-overflow 400, which retries with a smaller output ceiling; + * attemptOnce retries every failure outside the 4xx range. */ const RETRYABLE_4XX_STATUSES = new Set([408, 429]) /** Cap on same-model attempts for failures that settle quickly (HTTP errors, From 1f74eea81b1f1791b1b6bfe8d471271766c7f078 Mon Sep 17 00:00:00 2001 From: aliasunder <32620895+aliasunder@users.noreply.github.com> Date: Wed, 11 Nov 2026 20:08:35 -0500 Subject: [PATCH 16/20] test(openrouter): build the overflow error from the captured body's wording The fitted-retry tests now run on the live overflow message with only its numbers replaced, so a reworded rejection fails the suite instead of silently disabling the retry. Co-Authored-By: Claude Fable 5.1 --- src/openrouter/__tests__/client.test.ts | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/openrouter/__tests__/client.test.ts b/src/openrouter/__tests__/client.test.ts index 6a032dc4..7a4a8dd7 100644 --- a/src/openrouter/__tests__/client.test.ts +++ b/src/openrouter/__tests__/client.test.ts @@ -249,8 +249,15 @@ const makeBadRequestError = (body: unknown): BadRequestResponseError => { ) } +/** The numbers in the captured overflow body's message: the window, the + * requested total, the input estimate and the output ceiling. */ +const OVERFLOW_NUMBERS_PATTERN = + /maximum context length is \d+ tokens\. However, you requested about \d+ tokens \(\d+ of text input, \d+ in the output\)/ + /** OpenRouter's 400 when the prompt plus the requested output ceiling exceeds - * the routed endpoint's context window, worded as the live API words it. */ + * the routed endpoint's context window: the captured body's wording with + * only its numbers replaced, so every fitted-retry test runs on the live + * wording the detector must match. */ const makeContextOverflowError = ({ contextLength, inputTokens, @@ -258,7 +265,15 @@ const makeContextOverflowError = ({ contextLength: number inputTokens: number }): BadRequestResponseError => { - const message = `This endpoint's maximum context length is ${contextLength} tokens. However, you requested about ${inputTokens + 128_000} tokens (${inputTokens} of text input, 128000 in the output). Please reduce the length of either one, or use the context-compression plugin to compress your prompt automatically.` + const capturedMessage = errorBodySchema.parse(overflowErrorBody).error.message + + if (!OVERFLOW_NUMBERS_PATTERN.test(capturedMessage)) { + throw new Error("the captured overflow body no longer carries the expected numbers") + } + const message = capturedMessage.replace( + OVERFLOW_NUMBERS_PATTERN, + `maximum context length is ${contextLength} tokens. However, you requested about ${inputTokens + 128_000} tokens (${inputTokens} of text input, 128000 in the output)`, + ) return makeBadRequestError({ error: { code: 400, message } }) } From ab65d28e2d12891019422b61d8080c341dfa4c77 Mon Sep 17 00:00:00 2001 From: aliasunder <32620895+aliasunder@users.noreply.github.com> Date: Wed, 11 Nov 2026 20:50:10 -0500 Subject: [PATCH 17/20] refactor(openrouter): drop the generation cost lookup OpenRouter includes the cost in every response's usage block, and no run has shown it missing, so the accepted attempt records the cost it carries and a missing one stays null. The lookup, its second client and the runtime @openrouter/sdk dependency go with it; the package stays a dev dependency for the test fixtures' transform and error classes. Co-Authored-By: Claude Fable 5.1 --- AGENTS.md | 5 +- CONTRIBUTING.md | 4 +- README.md | 2 +- action.yml | 2 +- package-lock.json | 3 +- package.json | 2 +- src/__tests__/orchestrate.test.ts | 91 ------ src/main.ts | 2 - src/openrouter/__tests__/client.test.ts | 391 +----------------------- src/openrouter/client.ts | 91 +----- src/orchestrate.ts | 3 +- 11 files changed, 24 insertions(+), 572 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index c4e9a93f..9b941dc3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,7 +28,7 @@ src/ config.ts # action inputs → validated ActionConfig logger.ts # structured JSON logger — levels, child contexts, lazy props github/ # GitHub I/O: event payload → PrContext, octokit wrappers (diff fetch, review posting) - openrouter/ # OpenRouter I/O: review requests through @openrouter/agent, generation cost lookup through @openrouter/sdk, per-attempt and shared review deadlines, retry ladder (up to two attempts per model, then the fallback_model input; on HTTP errors, timeouts, invalid structured output), cost summary + openrouter/ # OpenRouter I/O: review requests through @openrouter/agent, per-attempt and shared review deadlines, retry ladder (up to two attempts per model, then the fallback_model input; on HTTP errors, timeouts, invalid structured output), cost summary diff/ # pure transforms over parse-diff output: git-quoted path decoding, annotation, commentable lines, diff-level exclusion (patterns, gitattributes linguist rules, wildcard cap: at most 2 "*" per path segment so matching cannot backtrack for minutes) context/ # workspace I/O: conventions file, root .gitattributes, changed files, import-trace scan, doc-mention scan, priority docs review/ # pure review logic (the model call is a function the caller passes in): finding schema, phases + stage dispatch (defined under Review instruction authoring), prompt, non-finding filter and unknown-file filter (README sections of those names), cross-phase merge, path normalization, selection, comment mapping, markdown code spans, title similarity, context notes, summary @@ -47,8 +47,7 @@ backwards dependency and a bug — enforced by a the outermost layer, so only tests import from a `__tests__/` folder. Use the official SDKs — don't hand-roll what `@actions/core`, -`@actions/github` (octokit), `@openrouter/agent`, `@openrouter/sdk`, or -`parse-diff` already do. +`@actions/github` (octokit), `@openrouter/agent`, or `parse-diff` already do. Before writing any parsing or validation helper, check the SDK's utility surface first: `@actions/core` ships `getBooleanInput` (strict YAML 1.2 booleans), which replaced a hand-rolled boolean-string Zod transform here. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fc120b3c..9ded4182 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -43,9 +43,7 @@ points: constructs SDK clients and touches `process.env`, except that `logger.ts` reads `LOG_LEVEL` - Use the official SDKs (`@actions/core`, `@actions/github`, - `@openrouter/agent` for review requests, `@openrouter/sdk` for the - generation cost lookup, `parse-diff`) — don't hand-roll what they - already do + `@openrouter/agent`, `parse-diff`) — don't hand-roll what they already do - Relative imports carry explicit `.js` extensions (ESM runtime requirement) - Tests are behavioral specs: one focused `it()` per behavior, exact assertions, `const` per test diff --git a/README.md b/README.md index f7759181..2ce662a6 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ An empty value, such as an unset repo variable, selects the input's default, so | `model` | `anthropic/claude-sonnet-4-6` | OpenRouter model slug exactly as listed on openrouter.ai/models | | `fallback_model` | `""` | Model tried after the primary model's attempts fail (up to two per model). A timed-out primary attempt moves to this model immediately, without retrying the primary. Auth and credit errors (HTTP 401/402/403) stop the review without trying it. Empty = no fallback | | `request_timeout_seconds` | `900` | Deadline for one model request, in seconds, capped by the time left in `review_timeout_seconds`. When it elapses, the attempt is recorded as `timeout` and the review moves to `fallback_model` if one is set; a timeout on the last model (the primary, when no fallback is set) may be retried once. The HTTP call is aborted best-effort: a request the provider keeps serving is still billed, and its cost-summary row shows `n/a`. Empty = 900 | -| `review_timeout_seconds` | `1500` | Shared model-work budget from action start, in seconds, including context preparation, all phases, retries, fallbacks and cost lookups. Completed phases post when it expires, with incomplete coverage identified; the run fails if none completed. Leave room below the workflow job timeout for setup and GitHub publication. External cancellation stops the run at once; unlike the deadline, it does not post completed phases. Empty = 1500 | +| `review_timeout_seconds` | `1500` | Shared model-work budget from action start, in seconds, including context preparation, all phases, retries and fallbacks. Completed phases post when it expires, with incomplete coverage identified; the run fails if none completed. Leave room below the workflow job timeout for setup and GitHub publication. External cancellation stops the run at once; unlike the deadline, it does not post completed phases. Empty = 1500 | | `max_findings` | `""` _(uncapped)_ | Cap on posted findings, highest severity first. Empty = all validated findings post. | | `severity_threshold` | `low` | Minimum severity to post: `low` \| `medium` \| `high` \| `critical` | | `conventions_file` | `AGENTS.md` | Repo-relative path to the conventions file included in the prompt (truncated at `conventions_budget_tokens`). When the PR also changes the file, its full text is sent at most once, never twice | diff --git a/action.yml b/action.yml index 9313c789..442fdcec 100644 --- a/action.yml +++ b/action.yml @@ -25,7 +25,7 @@ inputs: required: false default: "900" review_timeout_seconds: - description: Shared model-work budget from action start, in seconds, including context preparation, all phases, retries, fallbacks and cost lookups. Completed phases post when it expires, with incomplete coverage identified; the run fails if none completed. Leave room below the workflow job timeout for setup and GitHub publication. External cancellation stops the run at once; unlike the deadline, it does not post completed phases. Empty = 1500 + description: Shared model-work budget from action start, in seconds, including context preparation, all phases, retries and fallbacks. Completed phases post when it expires, with incomplete coverage identified; the run fails if none completed. Leave room below the workflow job timeout for setup and GitHub publication. External cancellation stops the run at once; unlike the deadline, it does not post completed phases. Empty = 1500 required: false default: "1500" max_findings: diff --git a/package-lock.json b/package-lock.json index 8b3f3634..0a6d70a9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,6 @@ "@actions/core": "3.0.1", "@actions/github": "9.1.1", "@openrouter/agent": "1.1.0", - "@openrouter/sdk": "1.5.33", "env-var": "7.5.0", "luxon": "3.7.2", "parse-diff": "0.12.0", @@ -20,6 +19,7 @@ }, "devDependencies": { "@eslint/js": "10.0.1", + "@openrouter/sdk": "1.5.33", "@stylistic/eslint-plugin": "5.10.0", "@types/luxon": "3.7.6", "@types/node": "26.6.6", @@ -535,6 +535,7 @@ "version": "1.5.33", "resolved": "https://registry.npmjs.org/@openrouter/sdk/-/sdk-1.5.33.tgz", "integrity": "sha512-hbg4q/tZHOnggBk7Sjkel6COExpbe1+2p4Mt2YyvAloDXtNNaFkMab0SgVfoBdTgzweBlrJJ4jUHwHt5rFiadA==", + "dev": true, "hasInstallScript": true, "license": "Apache-2.0", "dependencies": { diff --git a/package.json b/package.json index c24edf9a..f6baf772 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,6 @@ "@actions/core": "3.0.1", "@actions/github": "9.1.1", "@openrouter/agent": "1.1.0", - "@openrouter/sdk": "1.5.33", "env-var": "7.5.0", "luxon": "3.7.2", "parse-diff": "0.12.0", @@ -41,6 +40,7 @@ }, "devDependencies": { "@eslint/js": "10.0.1", + "@openrouter/sdk": "1.5.33", "@stylistic/eslint-plugin": "5.10.0", "@types/luxon": "3.7.6", "@types/node": "26.6.6", diff --git a/src/__tests__/orchestrate.test.ts b/src/__tests__/orchestrate.test.ts index e7160003..56a25ed1 100644 --- a/src/__tests__/orchestrate.test.ts +++ b/src/__tests__/orchestrate.test.ts @@ -4708,95 +4708,6 @@ describe("staged phases", () => { ]) }) - it("publishes full coverage without a deadline warning when only accepted-response cost lookup expires", async () => { - vi.useFakeTimers() - try { - const deadline = Date.now() + 1000 - const remainingReviewMs = () => deadline - Date.now() - const costResponse = Promise.withResolvers() - const getGeneration = vi.fn(() => costResponse.promise) - const callModel = vi.fn(() => { - return makeModelResultStub({ - id: "accepted-before-deadline", - model: "test/model", - usage: { inputTokens: 10, outputTokens: 20 }, - }) - }) - const client = createOpenRouterClient({ - agent: { callModel }, - costLookupClient: { generations: { getGeneration } }, - requestTimeoutMs: 900_000, - remainingReviewMs, - }) - const stubs = makeOrchestrateDeps({ - remainingReviewMs, - generateFindings: createPromptedGenerateFindings( - { - openrouterClient: client, - model: "test/model", - fallbackModel: "fallback/model", - }, - createTestLogger(), - ), - }) - const pending = orchestrate(stubs.deps, createTestLogger()) - await vi.advanceTimersByTimeAsync(1000) - const result = await pending - const expectedCost = renderCostSummary({ - attempts: [ - { - phase: "combined", - model: "test/model", - outcome: "accepted", - promptTokens: 10, - completionTokens: 20, - costUsd: null, - errorSummary: null, - }, - ], - modelUsed: "test/model", - }) - expect(remainingReviewMs()).toBe(0) - expect(callModel).toHaveBeenCalledTimes(1) - expect(getGeneration).toHaveBeenCalledExactlyOnceWith( - { id: "accepted-before-deadline" }, - { retries: { strategy: "none" }, signal: expect.any(AbortSignal) }, - ) - expect(result).toEqual({ - findingsCount: expectedSelection.selected.length, - reviewUrl: "https://github.com/test/review/1", - modelUsed: "test/model", - skippedReason: "", - phases: [{ phase: "combined", status: "completed" }], - reviewSummaryMarkdown: expectedReviewSummary(), - costSummaryMarkdown: expectedCost, - conventionsNote: null, - }) - expect(stubs.upsertSummaryCommentCalls).toEqual([ - expectedStatus({ - isFirstRun: true, - postedCount: expectedSelection.selected.length, - totalCount: expectedSelection.selected.length, - }), - ]) - expect(stubs.postFindingsReviewCalls).toEqual([ - expectedFindingsReview(expectedSelection.selected), - ]) - expect(stubs.updateCheckRunCalls).toEqual([ - { - checkRunId: 555, - conclusion: "success", - output: { - title: `${expectedSelection.selected.length} findings`, - summary: `Reviewed with \`test/model\` — ${expectedSelection.selected.length} findings posted.\n\n${expectedCost}`, - }, - }, - ]) - } finally { - vi.useRealTimers() - } - }) - it("publishes completed parallel phases once when a provider ignores the review deadline abort", async () => { vi.useFakeTimers() try { @@ -4815,7 +4726,6 @@ describe("staged phases", () => { .mockImplementation(() => makePendingModelResultStub(lateResponse.promise)) const client = createOpenRouterClient({ agent: { callModel }, - costLookupClient: { generations: { getGeneration: vi.fn() } }, requestTimeoutMs: 900_000, remainingReviewMs, }) @@ -4923,7 +4833,6 @@ describe("staged phases", () => { }) const client = createOpenRouterClient({ agent: { callModel }, - costLookupClient: { generations: { getGeneration: vi.fn() } }, requestTimeoutMs: 900_000, remainingReviewMs: () => Infinity, retryDelayMs: 0, diff --git a/src/main.ts b/src/main.ts index 4f1ce065..2b39c6ff 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,7 +1,6 @@ import * as core from "@actions/core" import { context, getOctokit } from "@actions/github" import { OpenRouter as OpenRouterAgent } from "@openrouter/agent" -import { OpenRouter } from "@openrouter/sdk" import envVar from "env-var" import { parseConfig, type RawInputs } from "./config.js" import { createContextReader } from "./context/workspace.js" @@ -141,7 +140,6 @@ try { // passes its own phase-tagged logger to requestReview openrouterClient: createOpenRouterClient({ agent: new OpenRouterAgent({ apiKey: config.openrouterApiKey }), - costLookupClient: new OpenRouter({ apiKey: config.openrouterApiKey }), requestTimeoutMs: config.requestTimeoutSeconds * 1000, remainingReviewMs, }), diff --git a/src/openrouter/__tests__/client.test.ts b/src/openrouter/__tests__/client.test.ts index 7a4a8dd7..95fc36ea 100644 --- a/src/openrouter/__tests__/client.test.ts +++ b/src/openrouter/__tests__/client.test.ts @@ -14,7 +14,6 @@ import { createOpenRouterClient, ReviewRequestError, type AgentLike, - type GenerationsLike, type ResponsesResult, type ReviewRequest, } from "../client.js" @@ -97,8 +96,7 @@ const acceptedResult = toModelCallResult(acceptedWirePayload) const invalidJsonResult = toModelCallResult(invalidJsonWirePayload) const schemaMismatchResult = toModelCallResult(schemaMismatchWirePayload) -/** The accepted payload with its usage cost removed — the case that routes - * the client to the generation cost lookup. */ +/** The accepted payload with its usage cost removed. */ const makeNoCostResult = (): ModelCallResult => { const { usage, ...payload } = parseAcceptedPayload() const usageWithoutCost = Object.fromEntries( @@ -126,26 +124,11 @@ type StubModelResult = | { error: unknown } | { pending: (request: { signal: AbortSignal }) => Promise } -type StubOptions = { signal?: AbortSignal } | undefined - -type StubGenerationResponse = - { value: unknown } | { error: unknown } | { pending: (options: StubOptions) => Promise } - -const makeClientStubs = ({ - modelResults, - generationResponses = [], -}: { - modelResults: StubModelResult[] - generationResponses?: StubGenerationResponse[] -}) => { +const makeClientStubs = ({ modelResults }: { modelResults: StubModelResult[] }) => { const modelCalls: { request: ReviewRequest & { signal: AbortSignal } options: { retries?: { strategy: "backoff" | "none" } } | undefined }[] = [] - const generationCalls: { - id: string - options: StubOptions - }[] = [] const takeNextModelResult = (request: { signal: AbortSignal }): Promise => { const next = modelResults.at(modelCalls.length - 1) @@ -172,30 +155,13 @@ const makeClientStubs = ({ }, } - const costLookupClient: GenerationsLike = { - generations: { - getGeneration: async (request, options) => { - generationCalls.push({ ...request, options }) - const next = generationResponses.at(generationCalls.length - 1) - - if (!next) { - throw new Error(`stub: unexpected getGeneration call #${generationCalls.length}`) - } - if ("error" in next) throw next.error - if ("pending" in next) return next.pending(options) - return next.value - }, - }, - } - - return { agent, costLookupClient, modelCalls, generationCalls } + return { agent, modelCalls } } -const makeClient = (stub: { agent: AgentLike; costLookupClient: GenerationsLike }) => { +const makeClient = (stub: { agent: AgentLike }) => { const logger = createTestLogger() const client = createOpenRouterClient({ agent: stub.agent, - costLookupClient: stub.costLookupClient, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, retryDelayMs: 0, @@ -330,7 +296,6 @@ describe("requestReview", () => { const logger = createTestLogger() const client = createOpenRouterClient({ agent: stub.agent, - costLookupClient: stub.costLookupClient, requestTimeoutMs: 45_000, remainingReviewMs: () => 0, }) @@ -349,7 +314,6 @@ describe("requestReview", () => { deadlineExceeded: true, }) expect(stub.modelCalls).toEqual([]) - expect(stub.generationCalls).toEqual([]) }) it.each(["primary", "fallback"])( @@ -368,7 +332,6 @@ describe("requestReview", () => { const logger = createTestLogger() const client = createOpenRouterClient({ agent: stub.agent, - costLookupClient: stub.costLookupClient, requestTimeoutMs: 45_000, remainingReviewMs: () => deadline - performance.now(), }) @@ -425,7 +388,6 @@ describe("requestReview", () => { late.resolve(acceptedResult) await vi.advanceTimersByTimeAsync(1000) expect(error.attempts).toEqual(expectedAttempts) - expect(stub.generationCalls).toEqual([]) expect(vi.getTimerCount()).toBe(0) } finally { vi.useRealTimers() @@ -441,7 +403,6 @@ describe("requestReview", () => { const logger = createTestLogger() const client = createOpenRouterClient({ agent: stub.agent, - costLookupClient: stub.costLookupClient, requestTimeoutMs: 45_000, remainingReviewMs: () => deadline - performance.now(), }) @@ -473,104 +434,6 @@ describe("requestReview", () => { } }) - it("skips cost lookup when an accepted response exhausts the remaining review time", async () => { - const remainingReviewMs = vi.fn(() => 100) - const stub = makeClientStubs({ - modelResults: [ - { - pending: async () => { - remainingReviewMs.mockReturnValue(0) - return makeNoCostResult() - }, - }, - ], - }) - const logger = createTestLogger() - const client = createOpenRouterClient({ - agent: stub.agent, - costLookupClient: stub.costLookupClient, - requestTimeoutMs: 45_000, - remainingReviewMs, - }) - expect(await client.requestReview(requestParams, logger)).toEqual({ - review: acceptedReview, - modelUsed: requestParams.model, - attempts: [ - { - model: requestParams.model, - outcome: "accepted", - promptTokens: 54, - completionTokens: 102, - costUsd: null, - errorSummary: null, - }, - ], - }) - expect(stub.modelCalls.map(({ request }) => request.model)).toEqual([requestParams.model]) - expect(stub.generationCalls).toEqual([]) - expect(logsWithMessage(logger, "generation cost lookup skipped")).toEqual([ - { - level: "debug", - message: "generation cost lookup skipped", - data: { - operation: "generation cost lookup", - generationId: ACCEPTED_GENERATION_ID, - reason: "review deadline exceeded", - }, - }, - ]) - }) - - it("retains an accepted review when cost lookup reaches the review deadline and ignores its late response", async () => { - vi.useFakeTimers() - try { - const late = Promise.withResolvers() - const stub = makeClientStubs({ - modelResults: [{ value: makeNoCostResult() }], - generationResponses: [{ pending: () => late.promise }], - }) - const deadline = performance.now() + 100 - const logger = createTestLogger() - const client = createOpenRouterClient({ - agent: stub.agent, - costLookupClient: stub.costLookupClient, - requestTimeoutMs: 45_000, - remainingReviewMs: () => deadline - performance.now(), - }) - const request = client.requestReview(requestParams, logger) - await vi.advanceTimersByTimeAsync(100) - const result = await request - const expected = { - review: acceptedReview, - modelUsed: requestParams.model, - attempts: [ - { - model: requestParams.model, - outcome: "accepted", - promptTokens: 54, - completionTokens: 102, - costUsd: null, - errorSummary: null, - }, - ], - } - expect(result).toEqual(expected) - expect( - stub.generationCalls.map(({ id, options }) => ({ - id, - aborted: options?.signal?.aborted, - })), - ).toEqual([{ id: ACCEPTED_GENERATION_ID, aborted: true }]) - late.resolve({ data: { totalCost: 42 } }) - await vi.advanceTimersByTimeAsync(1000) - expect(result).toEqual(expected) - expect(stub.modelCalls.map(({ request }) => request.model)).toEqual([requestParams.model]) - expect(vi.getTimerCount()).toBe(0) - } finally { - vi.useRealTimers() - } - }) - it("sends one Responses request with the prompts, the full output ceiling, the strict review schema and an abort signal", async () => { const stub = makeClientStubs({ modelResults: [{ value: acceptedResult }] }) const { client, logger } = makeClient(stub) @@ -900,7 +763,6 @@ describe("requestReview", () => { it("passes a live (non-aborted) AbortSignal to the SDK call", async () => { const stub = makeClientStubs({ modelResults: [{ value: acceptedResult }], - generationResponses: [{ value: { data: { totalCost: 0.01 } } }], }) const { client, logger } = makeClient(stub) @@ -934,7 +796,6 @@ describe("requestReview", () => { const logger = createTestLogger() const client = createOpenRouterClient({ agent: stub.agent, - costLookupClient: stub.costLookupClient, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, retryDelayMs: 0, @@ -1012,7 +873,6 @@ describe("requestReview", () => { const logger = createTestLogger() const client = createOpenRouterClient({ agent: stub.agent, - costLookupClient: stub.costLookupClient, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, retryDelayMs: 0, @@ -1098,7 +958,6 @@ describe("requestReview", () => { const logger = createTestLogger() const client = createOpenRouterClient({ agent: stub.agent, - costLookupClient: stub.costLookupClient, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, retryDelayMs: 0, @@ -1146,7 +1005,6 @@ describe("requestReview", () => { const logger = createTestLogger() const client = createOpenRouterClient({ agent: stub.agent, - costLookupClient: stub.costLookupClient, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, retryDelayMs: 0, @@ -1202,7 +1060,6 @@ describe("requestReview", () => { const logger = createTestLogger() const client = createOpenRouterClient({ agent: stub.agent, - costLookupClient: stub.costLookupClient, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, retryDelayMs: 0, @@ -1245,7 +1102,6 @@ describe("requestReview", () => { const logger = createTestLogger() const client = createOpenRouterClient({ agent: stub.agent, - costLookupClient: stub.costLookupClient, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, retryDelayMs: 0, @@ -1301,7 +1157,6 @@ describe("requestReview", () => { const logger = createTestLogger() const client = createOpenRouterClient({ agent: stub.agent, - costLookupClient: stub.costLookupClient, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, retryDelayMs: 0, @@ -1365,7 +1220,6 @@ describe("requestReview", () => { const logger = createTestLogger() const client = createOpenRouterClient({ agent: stub.agent, - costLookupClient: stub.costLookupClient, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, retryDelayMs, @@ -1387,107 +1241,6 @@ describe("requestReview", () => { } }) - it("reports the review deadline as the cost lookup's failure when that deadline cuts the lookup short", async () => { - vi.useFakeTimers() - try { - const stub = makeClientStubs({ - modelResults: [{ value: makeNoCostResult() }], - generationResponses: [{ pending: () => new Promise(() => undefined) }], - }) - const logger = createTestLogger() - const deadline = performance.now() + 100 - const client = createOpenRouterClient({ - agent: stub.agent, - costLookupClient: stub.costLookupClient, - requestTimeoutMs: 45_000, - remainingReviewMs: () => deadline - performance.now(), - }) - - const reviewPromise = client.requestReview(requestParams, logger) - await vi.advanceTimersByTimeAsync(100) - await reviewPromise - - expect(logger.messages.filter((entry) => entry.level === "warn")).toEqual([ - { - level: "warn", - message: "request deadline elapsed", - data: { - operation: "generation cost lookup", - generationId: ACCEPTED_GENERATION_ID, - timeoutMs: 100, - }, - }, - { - level: "warn", - message: "generation cost lookup failed", - data: { - operation: "generation cost lookup", - generationId: ACCEPTED_GENERATION_ID, - error: "review deadline exceeded", - }, - }, - ]) - } finally { - vi.useRealTimers() - } - }) - - it("degrades to a null cost with both warnings when the generation lookup exceeds the deadline", async () => { - vi.useFakeTimers() - try { - const stub = makeClientStubs({ - modelResults: [{ value: makeNoCostResult() }], - generationResponses: [{ pending: () => new Promise(() => undefined) }], - }) - const logger = createTestLogger() - const client = createOpenRouterClient({ - agent: stub.agent, - costLookupClient: stub.costLookupClient, - remainingReviewMs: () => Infinity, - requestTimeoutMs: 45_000, - retryDelayMs: 0, - }) - - const reviewPromise = client.requestReview(requestParams, logger) - await vi.advanceTimersByTimeAsync(45_000) - const result = await reviewPromise - - expect(result.attempts).toEqual([ - { - model: requestParams.model, - outcome: "accepted", - ...ACCEPTED_USAGE, - costUsd: null, - errorSummary: null, - }, - ]) - expect(logsWithMessage(logger, "request deadline elapsed")).toEqual([ - { - level: "warn", - message: "request deadline elapsed", - data: { - operation: "generation cost lookup", - generationId: ACCEPTED_GENERATION_ID, - timeoutMs: 45_000, - }, - }, - ]) - expect(logsWithMessage(logger, "generation cost lookup failed")).toEqual([ - { - level: "warn", - message: "generation cost lookup failed", - data: { - operation: "generation cost lookup", - generationId: ACCEPTED_GENERATION_ID, - error: "no response within 45s", - }, - }, - ]) - } finally { - vi.useRealTimers() - } - }) - it("clears the review attempt timeout timer after a successful response", async () => { vi.useFakeTimers() try { @@ -1512,7 +1265,6 @@ describe("requestReview", () => { const logger = createTestLogger() const client = createOpenRouterClient({ agent: stub.agent, - costLookupClient: stub.costLookupClient, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, retryDelayMs, @@ -2045,50 +1797,9 @@ describe("requestReview", () => { expect(stub.modelCalls).toHaveLength(4) }) - it("looks up the generation cost when the accepted attempt has no usage cost", async () => { - const stub = makeClientStubs({ - modelResults: [{ value: makeNoCostResult() }], - generationResponses: [{ value: { data: { totalCost: 0.0399 } } }], - }) - const { client, logger } = makeClient(stub) - - const result = await client.requestReview(requestParams, logger) - - expect(stub.generationCalls).toEqual([ - { - id: ACCEPTED_GENERATION_ID, - options: { - retries: { strategy: "none" }, - signal: expect.any(AbortSignal), - }, - }, - ]) - expect(result.attempts).toEqual([ - { - model: requestParams.model, - outcome: "accepted", - ...ACCEPTED_USAGE, - costUsd: 0.0399, - errorSummary: null, - }, - ]) - expect(logsWithMessage(logger, "generation cost looked up")).toEqual([ - { - level: "info", - message: "generation cost looked up", - data: { - operation: "generation cost lookup", - generationId: ACCEPTED_GENERATION_ID, - costUsd: 0.0399, - }, - }, - ]) - }) - - it("degrades to a null cost with a warning when the generation lookup fails", async () => { + it("records null token counts when the response carries no usage block", async () => { const stub = makeClientStubs({ - modelResults: [{ value: makeNoCostResult() }], - generationResponses: [{ error: makeStatusError(500) }], + modelResults: [{ value: makeNoUsageResult() }], }) const { client, logger } = makeClient(stub) @@ -2096,108 +1807,30 @@ describe("requestReview", () => { expect(result.attempts).toEqual([ { - model: requestParams.model, - outcome: "accepted", - ...ACCEPTED_USAGE, - costUsd: null, - errorSummary: null, - }, - ]) - expect(logsWithMessage(logger, "generation cost lookup failed")).toEqual([ - { - level: "warn", - message: "generation cost lookup failed", - data: { - operation: "generation cost lookup", - generationId: ACCEPTED_GENERATION_ID, - error: "HTTP 500", - }, - }, - ]) - }) - - it("degrades to a null cost with a warning when the generation lookup throws synchronously", async () => { - const throwingCostLookupClient: GenerationsLike = { - generations: { - getGeneration: () => { - throw new Error("lookup threw before returning a promise") - }, - }, - } - const { agent } = makeClientStubs({ modelResults: [{ value: makeNoCostResult() }] }) - const { client, logger } = makeClient({ agent, costLookupClient: throwingCostLookupClient }) - - const result = await client.requestReview(requestParams, logger) - - expect(result.attempts).toEqual([ - { - model: requestParams.model, + model: "z-ai/glm-5.3-flash", outcome: "accepted", - ...ACCEPTED_USAGE, + promptTokens: null, + completionTokens: null, costUsd: null, errorSummary: null, }, ]) - expect(logsWithMessage(logger, "generation cost lookup failed")).toEqual([ - { - level: "warn", - message: "generation cost lookup failed", - data: { - operation: "generation cost lookup", - generationId: ACCEPTED_GENERATION_ID, - error: "lookup threw before returning a promise", - }, - }, - ]) }) - it("degrades to a null cost with a warning when the generation response has an unexpected shape", async () => { - const stub = makeClientStubs({ - modelResults: [{ value: makeNoCostResult() }], - generationResponses: [{ value: { data: {} } }], - }) + it("records a null cost with its token counts when the usage block carries no cost", async () => { + const stub = makeClientStubs({ modelResults: [{ value: makeNoCostResult() }] }) const { client, logger } = makeClient(stub) const result = await client.requestReview(requestParams, logger) expect(result.attempts).toEqual([ { - model: requestParams.model, + model: "z-ai/glm-5.3-flash", outcome: "accepted", ...ACCEPTED_USAGE, costUsd: null, errorSummary: null, }, ]) - expect(logsWithMessage(logger, "unexpected generation response shape")).toEqual([ - { - level: "warn", - message: "unexpected generation response shape", - data: { operation: "generation cost lookup", generationId: ACCEPTED_GENERATION_ID }, - }, - ]) - }) - - it("records null token counts when the response carries no usage block", async () => { - const stub = makeClientStubs({ - modelResults: [{ value: makeNoUsageResult() }], - generationResponses: [{ value: { data: { totalCost: 0.0399 } } }], - }) - const { client, logger } = makeClient(stub) - - const result = await client.requestReview(requestParams, logger) - - // costUsd from the lookup proves the missing usage block routed the - // accepted attempt through the generation-cost fallback - expect(result.attempts).toEqual([ - { - model: "z-ai/glm-5.3-flash", - outcome: "accepted", - promptTokens: null, - completionTokens: null, - costUsd: 0.0399, - errorSummary: null, - }, - ]) }) }) diff --git a/src/openrouter/client.ts b/src/openrouter/client.ts index 2b0a2d57..8ab074d0 100644 --- a/src/openrouter/client.ts +++ b/src/openrouter/client.ts @@ -1,5 +1,4 @@ import { DateTime } from "luxon" -import { z } from "zod" import type { Logger } from "../logger.js" import { reviewResponseJsonSchema, @@ -83,21 +82,6 @@ export type AgentLike = { ): { getResponse(): Promise; getText(): Promise } } -/** Structural subset of the OpenRouter SDK's REST client — the generation - * cost lookup is the one endpoint this client calls that the Agent SDK does - * not cover. */ -export type GenerationsLike = { - generations: { - getGeneration( - request: { id: string }, - options?: { - retries?: { strategy: "backoff" | "none" } - signal?: AbortSignal - }, - ): Promise - } -} - export type OpenRouterClient = { /** Every line the request logs goes through `logger`, so props the caller * binds to it tag all of them. A timed-out HTTP call can still settle @@ -113,10 +97,6 @@ export type OpenRouterClient = { ) => Promise } -const generationResponseSchema = z.object({ - data: z.object({ totalCost: z.number() }), -}) - /** Auth and credit statuses. The fallback model shares the key, so these end * the ladder without trying it. */ const KEY_REJECTED_STATUSES = new Set([401, 402, 403]) @@ -407,13 +387,11 @@ const summarizeAttempts = (attempts: ModelAttempt[]): string => { * concurrent callers can tag their requests' lines with their own props. */ export const createOpenRouterClient = ({ agent, - costLookupClient, requestTimeoutMs, remainingReviewMs, retryDelayMs = RETRY_DELAY_MS, }: { agent: AgentLike - costLookupClient: GenerationsLike /** Per-attempt deadline. When it elapses the attempt records outcome * `timeout` and the ladder moves on — to the next model when one * exists, otherwise a same-model retry while attempts remain — whether @@ -421,7 +399,7 @@ export const createOpenRouterClient = ({ * best-effort. */ requestTimeoutMs: number /** Milliseconds left in the whole review's time budget; 0 or less once - * it has run out. Caps every attempt, cost lookup, and retry delay. */ + * it has run out. Caps every attempt and retry delay. */ remainingReviewMs: () => number retryDelayMs?: number }): OpenRouterClient => { @@ -579,71 +557,12 @@ export const createOpenRouterClient = ({ review: parsedReview.data, routedModel: responsesResult.model, - // A Responses result's `id` is its generation id (`gen-…`), the id the - // generation cost lookup takes + // A Responses result's `id` is its generation id (`gen-…`) generationId: responsesResult.id, attempt: { model, outcome: "accepted", ...usage, errorSummary: null }, } } - /** A cost lookup failure must never fail a completed review, so every - * failure degrades to a null cost. */ - const lookupGenerationCost = async ( - generationId: string, - logger: Logger, - ): Promise => { - const remainingMs = remainingReviewMs() - - // Every lookup line names the generation, so a failure ties back to it - const lookupLogger = logger.child({ operation: "generation cost lookup", generationId }) - - if (remainingMs <= 0) { - lookupLogger.debug("generation cost lookup skipped", { reason: "review deadline exceeded" }) - return null - } - - const callTimeout = getCallTimeout(remainingMs) - - const lookup = await withDeadline( - { - start: (signal) => { - return costLookupClient.generations.getGeneration( - { id: generationId }, - { retries: { strategy: "none" }, signal }, - ) - }, - timeoutMs: callTimeout.timeoutMs, - }, - lookupLogger, - ) - - if (lookup.status === "timed_out") { - lookupLogger.warn("generation cost lookup failed", { error: callTimeout.timeoutSummary }) - return null - } - if (lookup.status === "rejected") { - lookupLogger.warn("generation cost lookup failed", { - error: summarizeError(lookup.error), - }) - return null - } - - const parsed = generationResponseSchema.safeParse(lookup.value) - - if (!parsed.success) { - lookupLogger.warn("unexpected generation response shape") - return null - } - - // parsed.data = Zod safeParse result; .data = API envelope's data property - const costUsd = parsed.data.data.totalCost - - // Logged at info because how often an accepted response arrives without - // its cost is what decides whether this fallback stays - lookupLogger.info("generation cost looked up", { costUsd }) - return costUsd - } - const requestReview: OpenRouterClient["requestReview"] = async ( { systemPrompt, userPrompt, model, fallbackModel }, logger, @@ -684,11 +603,7 @@ export const createOpenRouterClient = ({ ) if (attemptResult.kind === "accepted") { - // The generation lookup runs only when the response's usage omits the cost - const costUsd = - attemptResult.attempt.costUsd ?? - (await lookupGenerationCost(attemptResult.generationId, logger)) - attempts.push({ ...attemptResult.attempt, costUsd }) + attempts.push(attemptResult.attempt) logger.info("review response accepted", { model: ladderModel, routedModel: attemptResult.routedModel, diff --git a/src/orchestrate.ts b/src/orchestrate.ts index e165fb4b..5fbd574d 100644 --- a/src/orchestrate.ts +++ b/src/orchestrate.ts @@ -1054,8 +1054,7 @@ const runReviewPipeline = async ( const completedPhases = phaseOutcomes.filter((outcome) => outcome.status === "completed") const phases = phaseOutcomes.map(describePhaseOutcome) - // Only a phase that failed on the deadline lost coverage. A completed phase - // whose cost lookup the deadline skipped still delivered its findings. + // Only a phase that failed on the deadline lost coverage const coverageLostToReviewDeadline = phaseOutcomes.some( (outcome) => outcome.status === "failed" && outcome.deadlineExceeded, ) From d68f14f3e6fe33a54049d6b8a4f75fe9593085ae Mon Sep 17 00:00:00 2001 From: aliasunder <32620895+aliasunder@users.noreply.github.com> Date: Wed, 11 Nov 2026 20:53:30 -0500 Subject: [PATCH 18/20] test: restore deadline coverage the cost lookup tests carried and name the null-cost case MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - client: an accepted response that spends the review deadline is still returned - orchestrate: a deadline spent after every phase completed publishes full coverage with no deadline warning - rename the no-usage-block test so its name covers the null cost it asserts Ship-Check: test-audit · claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- src/__tests__/orchestrate.test.ts | 50 +++++++++++++++++++++++++ src/openrouter/__tests__/client.test.ts | 39 ++++++++++++++++++- 2 files changed, 88 insertions(+), 1 deletion(-) diff --git a/src/__tests__/orchestrate.test.ts b/src/__tests__/orchestrate.test.ts index 56a25ed1..8ee48069 100644 --- a/src/__tests__/orchestrate.test.ts +++ b/src/__tests__/orchestrate.test.ts @@ -4708,6 +4708,56 @@ describe("staged phases", () => { ]) }) + it("publishes full coverage without a deadline warning when the review deadline runs out after every phase completed", async () => { + // The only phase's response spends the rest of this test's controllable deadline + const deadline = { remainingMs: 1 } + const stubs = makeOrchestrateDeps({ + remainingReviewMs: () => deadline.remainingMs, + generateFindings: async (reviewContext) => { + stubs.generateFindingsCalls.push(reviewContext) + deadline.remainingMs = 0 + return { + review: fixtureReviewResponse, + modelUsed: "test/model", + attempts: [fixtureAttempt], + } + }, + }) + + const result = await orchestrate(stubs.deps, createTestLogger()) + + expect(result).toEqual({ + findingsCount: expectedSelection.selected.length, + reviewUrl: "https://github.com/test/review/1", + modelUsed: "test/model", + skippedReason: "", + phases: [{ phase: "combined", status: "completed" }], + reviewSummaryMarkdown: expectedReviewSummary(), + costSummaryMarkdown: expectedCostSummary, + conventionsNote: null, + }) + expect(stubs.upsertSummaryCommentCalls).toEqual([ + expectedStatus({ + isFirstRun: true, + postedCount: expectedSelection.selected.length, + totalCount: expectedSelection.selected.length, + }), + ]) + expect(stubs.postFindingsReviewCalls).toEqual([ + expectedFindingsReview(expectedSelection.selected), + ]) + expect(stubs.updateCheckRunCalls).toEqual([ + { + checkRunId: 555, + conclusion: "success", + output: { + title: `${expectedSelection.selected.length} findings`, + summary: `Reviewed with \`test/model\` — ${expectedSelection.selected.length} findings posted.\n\n${expectedCostSummary}`, + }, + }, + ]) + }) + it("publishes completed parallel phases once when a provider ignores the review deadline abort", async () => { vi.useFakeTimers() try { diff --git a/src/openrouter/__tests__/client.test.ts b/src/openrouter/__tests__/client.test.ts index 95fc36ea..de7b5ea3 100644 --- a/src/openrouter/__tests__/client.test.ts +++ b/src/openrouter/__tests__/client.test.ts @@ -434,6 +434,43 @@ describe("requestReview", () => { } }) + it("returns an accepted review whose response arrives as the review deadline runs out", async () => { + // The response spends the rest of this test's controllable deadline + const deadline = { remainingMs: 100 } + const stub = makeClientStubs({ + modelResults: [ + { + pending: async () => { + deadline.remainingMs = 0 + return acceptedResult + }, + }, + ], + }) + const logger = createTestLogger() + const client = createOpenRouterClient({ + agent: stub.agent, + requestTimeoutMs: 45_000, + remainingReviewMs: () => deadline.remainingMs, + }) + + const result = await client.requestReview(requestParams, logger) + + expect(result).toEqual({ + review: acceptedReview, + modelUsed: requestParams.model, + attempts: [ + { + model: requestParams.model, + outcome: "accepted", + ...ACCEPTED_USAGE, + errorSummary: null, + }, + ], + }) + expect(stub.modelCalls.map(({ request }) => request.model)).toEqual([requestParams.model]) + }) + it("sends one Responses request with the prompts, the full output ceiling, the strict review schema and an abort signal", async () => { const stub = makeClientStubs({ modelResults: [{ value: acceptedResult }] }) const { client, logger } = makeClient(stub) @@ -1797,7 +1834,7 @@ describe("requestReview", () => { expect(stub.modelCalls).toHaveLength(4) }) - it("records null token counts when the response carries no usage block", async () => { + it("records null token counts and a null cost when the response carries no usage block", async () => { const stub = makeClientStubs({ modelResults: [{ value: makeNoUsageResult() }], }) From 634947909d613758e7d8063be75d4654e7129e39 Mon Sep 17 00:00:00 2001 From: aliasunder <32620895+aliasunder@users.noreply.github.com> Date: Wed, 11 Nov 2026 20:57:21 -0500 Subject: [PATCH 19/20] docs: list @openrouter/sdk with its test role in the SDK inventories Co-Authored-By: Claude Fable 5.1 --- AGENTS.md | 4 +++- CONTRIBUTING.md | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 9b941dc3..cd5fd13c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -47,7 +47,9 @@ backwards dependency and a bug — enforced by a the outermost layer, so only tests import from a `__tests__/` folder. Use the official SDKs — don't hand-roll what `@actions/core`, -`@actions/github` (octokit), `@openrouter/agent`, or `parse-diff` already do. +`@actions/github` (octokit), `@openrouter/agent`, `@openrouter/sdk` (a dev +dependency: the tests take Responses result and error shapes from it), or +`parse-diff` already do. Before writing any parsing or validation helper, check the SDK's utility surface first: `@actions/core` ships `getBooleanInput` (strict YAML 1.2 booleans), which replaced a hand-rolled boolean-string Zod transform here. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9ded4182..7af8fca7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -43,7 +43,8 @@ points: constructs SDK clients and touches `process.env`, except that `logger.ts` reads `LOG_LEVEL` - Use the official SDKs (`@actions/core`, `@actions/github`, - `@openrouter/agent`, `parse-diff`) — don't hand-roll what they already do + `@openrouter/agent`, `@openrouter/sdk` for Responses result and error shapes + in tests, `parse-diff`) — don't hand-roll what they already do - Relative imports carry explicit `.js` extensions (ESM runtime requirement) - Tests are behavioral specs: one focused `it()` per behavior, exact assertions, `const` per test From fb558ce0ca437c3133f509ca4c984f9d4c909d69 Mon Sep 17 00:00:00 2001 From: aliasunder <32620895+aliasunder@users.noreply.github.com> Date: Wed, 11 Nov 2026 20:57:43 -0500 Subject: [PATCH 20/20] test: plant a key rejection that never fires, for the verification profile Do not merge. --- src/openrouter/client.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/openrouter/client.ts b/src/openrouter/client.ts index 8ab074d0..bef359a7 100644 --- a/src/openrouter/client.ts +++ b/src/openrouter/client.ts @@ -497,7 +497,7 @@ export const createOpenRouterClient = ({ return { kind: "context_overflow", attempt, fittedMaxOutputTokens: fittedCeiling } } - const keyRejected = statusCode ? KEY_REJECTED_STATUSES.has(statusCode) : false + const keyRejected = false // A 4xx other than a 408 or a 429, such as a 400 bad request or a // 401/402/403 auth or credit status, is structural and does not retry.