diff --git a/AGENTS.md b/AGENTS.md index 7ba23a4f..cd5fd13c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,7 +28,7 @@ src/ config.ts # action inputs → validated ActionConfig logger.ts # structured JSON logger — levels, child contexts, lazy props github/ # GitHub I/O: event payload → PrContext, octokit wrappers (diff fetch, review posting) - openrouter/ # OpenRouter I/O: @openrouter/sdk wrapper, per-attempt and shared review deadlines, retry ladder (up to two attempts per model, then the fallback_model input; on HTTP errors, timeouts, invalid structured output), cost summary + openrouter/ # OpenRouter I/O: review requests through @openrouter/agent, per-attempt and shared review deadlines, retry ladder (up to two attempts per model, then the fallback_model input; on HTTP errors, timeouts, invalid structured output), cost summary diff/ # pure transforms over parse-diff output: git-quoted path decoding, annotation, commentable lines, diff-level exclusion (patterns, gitattributes linguist rules, wildcard cap: at most 2 "*" per path segment so matching cannot backtrack for minutes) context/ # workspace I/O: conventions file, root .gitattributes, changed files, import-trace scan, doc-mention scan, priority docs review/ # pure review logic (the model call is a function the caller passes in): finding schema, phases + stage dispatch (defined under Review instruction authoring), prompt, non-finding filter and unknown-file filter (README sections of those names), cross-phase merge, path normalization, selection, comment mapping, markdown code spans, title similarity, context notes, summary @@ -47,7 +47,9 @@ backwards dependency and a bug — enforced by a the outermost layer, so only tests import from a `__tests__/` folder. Use the official SDKs — don't hand-roll what `@actions/core`, -`@actions/github` (octokit), `@openrouter/sdk`, or `parse-diff` already do. +`@actions/github` (octokit), `@openrouter/agent`, `@openrouter/sdk` (a dev +dependency: the tests take Responses result and error shapes from it), or +`parse-diff` already do. Before writing any parsing or validation helper, check the SDK's utility surface first: `@actions/core` ships `getBooleanInput` (strict YAML 1.2 booleans), which replaced a hand-rolled boolean-string Zod transform here. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 22c5a289..7af8fca7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -43,7 +43,8 @@ points: constructs SDK clients and touches `process.env`, except that `logger.ts` reads `LOG_LEVEL` - Use the official SDKs (`@actions/core`, `@actions/github`, - `@openrouter/sdk`, `parse-diff`) — don't hand-roll what they already do + `@openrouter/agent`, `@openrouter/sdk` for Responses result and error shapes + in tests, `parse-diff`) — don't hand-roll what they already do - Relative imports carry explicit `.js` extensions (ESM runtime requirement) - Tests are behavioral specs: one focused `it()` per behavior, exact assertions, `const` per test diff --git a/README.md b/README.md index f7759181..2ce662a6 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ An empty value, such as an unset repo variable, selects the input's default, so | `model` | `anthropic/claude-sonnet-4-6` | OpenRouter model slug exactly as listed on openrouter.ai/models | | `fallback_model` | `""` | Model tried after the primary model's attempts fail (up to two per model). A timed-out primary attempt moves to this model immediately, without retrying the primary. Auth and credit errors (HTTP 401/402/403) stop the review without trying it. Empty = no fallback | | `request_timeout_seconds` | `900` | Deadline for one model request, in seconds, capped by the time left in `review_timeout_seconds`. When it elapses, the attempt is recorded as `timeout` and the review moves to `fallback_model` if one is set; a timeout on the last model (the primary, when no fallback is set) may be retried once. The HTTP call is aborted best-effort: a request the provider keeps serving is still billed, and its cost-summary row shows `n/a`. Empty = 900 | -| `review_timeout_seconds` | `1500` | Shared model-work budget from action start, in seconds, including context preparation, all phases, retries, fallbacks and cost lookups. Completed phases post when it expires, with incomplete coverage identified; the run fails if none completed. Leave room below the workflow job timeout for setup and GitHub publication. External cancellation stops the run at once; unlike the deadline, it does not post completed phases. Empty = 1500 | +| `review_timeout_seconds` | `1500` | Shared model-work budget from action start, in seconds, including context preparation, all phases, retries and fallbacks. Completed phases post when it expires, with incomplete coverage identified; the run fails if none completed. Leave room below the workflow job timeout for setup and GitHub publication. External cancellation stops the run at once; unlike the deadline, it does not post completed phases. Empty = 1500 | | `max_findings` | `""` _(uncapped)_ | Cap on posted findings, highest severity first. Empty = all validated findings post. | | `severity_threshold` | `low` | Minimum severity to post: `low` \| `medium` \| `high` \| `critical` | | `conventions_file` | `AGENTS.md` | Repo-relative path to the conventions file included in the prompt (truncated at `conventions_budget_tokens`). When the PR also changes the file, its full text is sent at most once, never twice | diff --git a/action.yml b/action.yml index 9313c789..442fdcec 100644 --- a/action.yml +++ b/action.yml @@ -25,7 +25,7 @@ inputs: required: false default: "900" review_timeout_seconds: - description: Shared model-work budget from action start, in seconds, including context preparation, all phases, retries, fallbacks and cost lookups. Completed phases post when it expires, with incomplete coverage identified; the run fails if none completed. Leave room below the workflow job timeout for setup and GitHub publication. External cancellation stops the run at once; unlike the deadline, it does not post completed phases. Empty = 1500 + description: Shared model-work budget from action start, in seconds, including context preparation, all phases, retries and fallbacks. Completed phases post when it expires, with incomplete coverage identified; the run fails if none completed. Leave room below the workflow job timeout for setup and GitHub publication. External cancellation stops the run at once; unlike the deadline, it does not post completed phases. Empty = 1500 required: false default: "1500" max_findings: diff --git a/fixtures/openrouter.chat-result.json b/fixtures/openrouter.chat-result.json deleted file mode 100644 index e8c005e4..00000000 --- a/fixtures/openrouter.chat-result.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "id": "gen-fixture-0001", - "object": "chat.completion", - "model": "openai/gpt-5-mini", - "choices": [ - { - "finishReason": "stop", - "message": { - "role": "assistant", - "content": "{\"analysis\":\"src/greeter.ts: verified the empty-key guard and traced register() callers.\",\"findings\":[]}" - } - } - ], - "usage": { "promptTokens": 12000, "completionTokens": 800, "cost": 0.0421 } -} diff --git a/fixtures/openrouter.invalid-json.json b/fixtures/openrouter.invalid-json.json deleted file mode 100644 index 4f617a61..00000000 --- a/fixtures/openrouter.invalid-json.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "id": "gen-fixture-0002", - "object": "chat.completion", - "model": "openai/gpt-5-mini", - "choices": [ - { - "finishReason": "length", - "message": { - "role": "assistant", - "content": "The model rambled instead of emitting JSON {" - } - } - ], - "usage": { "promptTokens": 11800, "completionTokens": 42, "cost": null } -} diff --git a/fixtures/openrouter.responses-accepted.json b/fixtures/openrouter.responses-accepted.json new file mode 100644 index 00000000..e93ce3a4 --- /dev/null +++ b/fixtures/openrouter.responses-accepted.json @@ -0,0 +1,168 @@ +{ + "id": "gen-1794440085-n7DSIWy7fBrBUcqMEvla", + "object": "response", + "created_at": 1794440085, + "model": "z-ai/glm-5.3-flash", + "status": "completed", + "completed_at": 1794440086, + "output": [ + { + "id": "rs_tmp_viau8k9rg1", + "type": "reasoning", + "status": "completed", + "content": [ + { + "type": "reasoning_text", + "text": "User asks to report no findings. Provide JSON with analysis and empty findings array." + } + ], + "summary": [], + "format": "unknown" + }, + { + "id": "msg_tmp_o67rrerum0m", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": "{\n \"analysis\": \"Reviewed the single-line change `export const greeting = \\\"hello\\\"`. This is a simple, valid module-level constant export. The syntax is correct, the value is a static string with no side effects, and naming follows common conventions. No security, correctness, style, or maintainability concerns were identified. As requested, there are no findings to report.\",\n \"findings\": []\n}", + "annotations": [], + "logprobs": [] + } + ] + } + ], + "error": null, + "incomplete_details": null, + "access_programs": null, + "tools": [], + "tool_choice": "auto", + "parallel_tool_calls": true, + "max_output_tokens": 128000, + "temperature": 1, + "top_p": 1, + "presence_penalty": 0, + "frequency_penalty": 0, + "top_logprobs": 0, + "max_tool_calls": null, + "metadata": {}, + "background": false, + "previous_response_id": null, + "service_tier": "auto", + "truncation": "disabled", + "store": false, + "instructions": "You are a code reviewer. Reply with JSON that matches the schema: an analysis string and a findings array.", + "text": { + "format": { + "type": "json_schema", + "name": "review_response", + "strict": true, + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "analysis": { + "type": "string" + }, + "findings": { + "type": "array", + "items": { + "type": "object", + "properties": { + "file": { + "type": "string", + "minLength": 1 + }, + "line": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "end_line": { + "anyOf": [ + { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + { + "type": "null" + } + ] + }, + "category": { + "type": "string", + "enum": ["correctness", "security", "conventions", "tests", "subtle_bugs", "ci"] + }, + "severity": { + "type": "string", + "enum": ["critical", "high", "medium", "low"] + }, + "confidence": { + "type": "string", + "enum": ["high", "medium", "low"] + }, + "title": { + "type": "string", + "minLength": 1 + }, + "description": { + "type": "string", + "minLength": 1 + }, + "suggestion": { + "type": ["string", "null"] + }, + "failure_scenario": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "file", + "line", + "end_line", + "category", + "severity", + "confidence", + "title", + "description", + "suggestion", + "failure_scenario" + ], + "additionalProperties": false + } + } + }, + "required": ["analysis", "findings"], + "additionalProperties": false + } + } + }, + "reasoning": { + "effort": "max" + }, + "safety_identifier": null, + "prompt_cache_key": null, + "prompt_cache_options": null, + "usage": { + "input_tokens": 54, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 102, + "output_tokens_details": { + "reasoning_tokens": 17 + }, + "total_tokens": 156, + "cost": 0.000021036, + "is_byok": false, + "cost_details": { + "upstream_inference_cost": 0.000021036, + "upstream_inference_input_cost": 0.000002268, + "upstream_inference_output_cost": 0.000018768 + } + } +} diff --git a/fixtures/openrouter.responses-bad-key-body.json b/fixtures/openrouter.responses-bad-key-body.json new file mode 100644 index 00000000..9e0cd438 --- /dev/null +++ b/fixtures/openrouter.responses-bad-key-body.json @@ -0,0 +1,6 @@ +{ + "error": { + "message": "User not found.", + "code": 401 + } +} diff --git a/fixtures/openrouter.responses-invalid-json.json b/fixtures/openrouter.responses-invalid-json.json new file mode 100644 index 00000000..5f8e59ec --- /dev/null +++ b/fixtures/openrouter.responses-invalid-json.json @@ -0,0 +1,168 @@ +{ + "id": "gen-1794440085-n7DSIWy7fBrBUcqMEvla", + "object": "response", + "created_at": 1794440085, + "model": "z-ai/glm-5.3-flash", + "status": "completed", + "completed_at": 1794440086, + "output": [ + { + "id": "rs_tmp_viau8k9rg1", + "type": "reasoning", + "status": "completed", + "content": [ + { + "type": "reasoning_text", + "text": "User asks to report no findings. Provide JSON with analysis and empty findings array." + } + ], + "summary": [], + "format": "unknown" + }, + { + "id": "msg_tmp_o67rrerum0m", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": "not json", + "annotations": [], + "logprobs": [] + } + ] + } + ], + "error": null, + "incomplete_details": null, + "access_programs": null, + "tools": [], + "tool_choice": "auto", + "parallel_tool_calls": true, + "max_output_tokens": 128000, + "temperature": 1, + "top_p": 1, + "presence_penalty": 0, + "frequency_penalty": 0, + "top_logprobs": 0, + "max_tool_calls": null, + "metadata": {}, + "background": false, + "previous_response_id": null, + "service_tier": "auto", + "truncation": "disabled", + "store": false, + "instructions": "You are a code reviewer. Reply with JSON that matches the schema: an analysis string and a findings array.", + "text": { + "format": { + "type": "json_schema", + "name": "review_response", + "strict": true, + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "analysis": { + "type": "string" + }, + "findings": { + "type": "array", + "items": { + "type": "object", + "properties": { + "file": { + "type": "string", + "minLength": 1 + }, + "line": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "end_line": { + "anyOf": [ + { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + { + "type": "null" + } + ] + }, + "category": { + "type": "string", + "enum": ["correctness", "security", "conventions", "tests", "subtle_bugs", "ci"] + }, + "severity": { + "type": "string", + "enum": ["critical", "high", "medium", "low"] + }, + "confidence": { + "type": "string", + "enum": ["high", "medium", "low"] + }, + "title": { + "type": "string", + "minLength": 1 + }, + "description": { + "type": "string", + "minLength": 1 + }, + "suggestion": { + "type": ["string", "null"] + }, + "failure_scenario": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "file", + "line", + "end_line", + "category", + "severity", + "confidence", + "title", + "description", + "suggestion", + "failure_scenario" + ], + "additionalProperties": false + } + } + }, + "required": ["analysis", "findings"], + "additionalProperties": false + } + } + }, + "reasoning": { + "effort": "max" + }, + "safety_identifier": null, + "prompt_cache_key": null, + "prompt_cache_options": null, + "usage": { + "input_tokens": 54, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 102, + "output_tokens_details": { + "reasoning_tokens": 17 + }, + "total_tokens": 156, + "cost": 0.000021036, + "is_byok": false, + "cost_details": { + "upstream_inference_cost": 0.000021036, + "upstream_inference_input_cost": 0.000002268, + "upstream_inference_output_cost": 0.000018768 + } + } +} diff --git a/fixtures/openrouter.responses-overflow-body.json b/fixtures/openrouter.responses-overflow-body.json new file mode 100644 index 00000000..8eb6e77e --- /dev/null +++ b/fixtures/openrouter.responses-overflow-body.json @@ -0,0 +1,9 @@ +{ + "error": { + "message": "This endpoint's maximum context length is 8192 tokens. However, you requested about 11593 tokens (4593 of text input, 7000 in the output). Please reduce the length of either one, or use the context-compression plugin to compress your prompt automatically.", + "code": 400, + "metadata": { + "provider_name": null + } + } +} diff --git a/fixtures/openrouter.responses-schema-mismatch.json b/fixtures/openrouter.responses-schema-mismatch.json new file mode 100644 index 00000000..4ea12472 --- /dev/null +++ b/fixtures/openrouter.responses-schema-mismatch.json @@ -0,0 +1,168 @@ +{ + "id": "gen-1794440085-n7DSIWy7fBrBUcqMEvla", + "object": "response", + "created_at": 1794440085, + "model": "z-ai/glm-5.3-flash", + "status": "completed", + "completed_at": 1794440086, + "output": [ + { + "id": "rs_tmp_viau8k9rg1", + "type": "reasoning", + "status": "completed", + "content": [ + { + "type": "reasoning_text", + "text": "User asks to report no findings. Provide JSON with analysis and empty findings array." + } + ], + "summary": [], + "format": "unknown" + }, + { + "id": "msg_tmp_o67rrerum0m", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": "{\"verdict\":\"lgtm\"}", + "annotations": [], + "logprobs": [] + } + ] + } + ], + "error": null, + "incomplete_details": null, + "access_programs": null, + "tools": [], + "tool_choice": "auto", + "parallel_tool_calls": true, + "max_output_tokens": 128000, + "temperature": 1, + "top_p": 1, + "presence_penalty": 0, + "frequency_penalty": 0, + "top_logprobs": 0, + "max_tool_calls": null, + "metadata": {}, + "background": false, + "previous_response_id": null, + "service_tier": "auto", + "truncation": "disabled", + "store": false, + "instructions": "You are a code reviewer. Reply with JSON that matches the schema: an analysis string and a findings array.", + "text": { + "format": { + "type": "json_schema", + "name": "review_response", + "strict": true, + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "analysis": { + "type": "string" + }, + "findings": { + "type": "array", + "items": { + "type": "object", + "properties": { + "file": { + "type": "string", + "minLength": 1 + }, + "line": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "end_line": { + "anyOf": [ + { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + { + "type": "null" + } + ] + }, + "category": { + "type": "string", + "enum": ["correctness", "security", "conventions", "tests", "subtle_bugs", "ci"] + }, + "severity": { + "type": "string", + "enum": ["critical", "high", "medium", "low"] + }, + "confidence": { + "type": "string", + "enum": ["high", "medium", "low"] + }, + "title": { + "type": "string", + "minLength": 1 + }, + "description": { + "type": "string", + "minLength": 1 + }, + "suggestion": { + "type": ["string", "null"] + }, + "failure_scenario": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "file", + "line", + "end_line", + "category", + "severity", + "confidence", + "title", + "description", + "suggestion", + "failure_scenario" + ], + "additionalProperties": false + } + } + }, + "required": ["analysis", "findings"], + "additionalProperties": false + } + } + }, + "reasoning": { + "effort": "max" + }, + "safety_identifier": null, + "prompt_cache_key": null, + "prompt_cache_options": null, + "usage": { + "input_tokens": 54, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 102, + "output_tokens_details": { + "reasoning_tokens": 17 + }, + "total_tokens": 156, + "cost": 0.000021036, + "is_byok": false, + "cost_details": { + "upstream_inference_cost": 0.000021036, + "upstream_inference_input_cost": 0.000002268, + "upstream_inference_output_cost": 0.000018768 + } + } +} diff --git a/fixtures/openrouter.schema-mismatch.json b/fixtures/openrouter.schema-mismatch.json deleted file mode 100644 index 98591dfc..00000000 --- a/fixtures/openrouter.schema-mismatch.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "id": "gen-fixture-0003", - "object": "chat.completion", - "model": "openai/gpt-5-mini", - "choices": [ - { - "finishReason": "stop", - "message": { - "role": "assistant", - "content": "{\"verdict\":\"lgtm\"}" - } - } - ], - "usage": { "promptTokens": 11900, "completionTokens": 12, "cost": 0.001 } -} diff --git a/package-lock.json b/package-lock.json index 1f379b50..0a6d70a9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,7 @@ "dependencies": { "@actions/core": "3.0.1", "@actions/github": "9.1.1", - "@openrouter/sdk": "1.5.33", + "@openrouter/agent": "1.1.0", "env-var": "7.5.0", "luxon": "3.7.2", "parse-diff": "0.12.0", @@ -19,6 +19,7 @@ }, "devDependencies": { "@eslint/js": "10.0.1", + "@openrouter/sdk": "1.5.33", "@stylistic/eslint-plugin": "5.10.0", "@types/luxon": "3.7.6", "@types/node": "26.6.6", @@ -502,10 +503,39 @@ "@octokit/openapi-types": "^27.0.0" } }, + "node_modules/@openrouter/agent": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@openrouter/agent/-/agent-1.1.0.tgz", + "integrity": "sha512-oIBn4AxeHZIUVYYWf2wPRNximYovlD6nnuU9afKhojAmcs8YKVUVs4NFwQh1dhnEO0iVUg7VmW94eT4Y9u0dfg==", + "license": "Apache-2.0", + "dependencies": { + "@openrouter/sdk": "^0.13.7", + "zod": "^4.0.0" + }, + "peerDependencies": { + "@modelcontextprotocol/client": "^2.0.0" + }, + "peerDependenciesMeta": { + "@modelcontextprotocol/client": { + "optional": true + } + } + }, + "node_modules/@openrouter/agent/node_modules/@openrouter/sdk": { + "version": "0.13.67", + "resolved": "https://registry.npmjs.org/@openrouter/sdk/-/sdk-0.13.67.tgz", + "integrity": "sha512-kUVaFYA9Xehq67ndTk98i+tRLfYH/F6xSf/WuC8dsukBzcgBvYQydoeKw4Xe+peXG/wOVtDxukWt5SqzhbloKw==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "zod": "^3.25.0 || ^4.0.0" + } + }, "node_modules/@openrouter/sdk": { "version": "1.5.33", "resolved": "https://registry.npmjs.org/@openrouter/sdk/-/sdk-1.5.33.tgz", "integrity": "sha512-hbg4q/tZHOnggBk7Sjkel6COExpbe1+2p4Mt2YyvAloDXtNNaFkMab0SgVfoBdTgzweBlrJJ4jUHwHt5rFiadA==", + "dev": true, "hasInstallScript": true, "license": "Apache-2.0", "dependencies": { diff --git a/package.json b/package.json index dbc4701d..f6baf772 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "dependencies": { "@actions/core": "3.0.1", "@actions/github": "9.1.1", - "@openrouter/sdk": "1.5.33", + "@openrouter/agent": "1.1.0", "env-var": "7.5.0", "luxon": "3.7.2", "parse-diff": "0.12.0", @@ -40,6 +40,7 @@ }, "devDependencies": { "@eslint/js": "10.0.1", + "@openrouter/sdk": "1.5.33", "@stylistic/eslint-plugin": "5.10.0", "@types/luxon": "3.7.6", "@types/node": "26.6.6", diff --git a/src/__tests__/orchestrate.test.ts b/src/__tests__/orchestrate.test.ts index cae33f8f..8ee48069 100644 --- a/src/__tests__/orchestrate.test.ts +++ b/src/__tests__/orchestrate.test.ts @@ -8,9 +8,10 @@ import type { ContextReader } from "../context/workspace.js" import { ReviewRequestError, createOpenRouterClient, - type ChatRequestSubset, type ModelAttempt, type OpenRouterClient, + type ResponsesResult, + type ReviewRequest, type StructuredReviewResult, } from "../openrouter/client.js" import { buildUserPrompt, estimateTokens, type PromptFile } from "../review/prompt.js" @@ -61,6 +62,29 @@ const fixtureReviewResponse: ReviewResponse = JSON.parse( readFileSync(new URL("../../fixtures/openrouter.response.json", import.meta.url), "utf8"), ) +/** The pair a `callModel` stub hands back once its request settles: the + * materialized response the client reads and the text the SDK extracts. */ +type ModelCallResult = { response: ResponsesResult; text: string } + +/** A `callModel` result that settles with the given promise. Both reads await + * the one promise, the way the SDK's reads share one materialized stream. */ +const makePendingModelResultStub = ( + settled: Promise, +): { getResponse(): Promise; getText(): Promise } => ({ + getResponse: async () => (await settled).response, + getText: async () => (await settled).text, +}) + +/** A `callModel` result that resolves at once with an accepted-shaped response + * carrying the fixture review as its text. */ +const makeModelResultStub = ( + response: ResponsesResult, +): { getResponse(): Promise; getText(): Promise } => { + return makePendingModelResultStub( + Promise.resolve({ response, text: JSON.stringify(fixtureReviewResponse) }), + ) +} + const fixturePrContext: PrContext = { prNumber: 7, title: "feat: trim names before greeting", @@ -4684,91 +4708,54 @@ describe("staged phases", () => { ]) }) - it("publishes full coverage without a deadline warning when only accepted-response cost lookup expires", async () => { - vi.useFakeTimers() - try { - const deadline = Date.now() + 1000 - const remainingReviewMs = () => deadline - Date.now() - const costResponse = Promise.withResolvers() - const getGeneration = vi.fn(() => costResponse.promise) - const send = vi.fn(async () => ({ - id: "accepted-before-deadline", - model: "test/model", - choices: [{ message: { content: JSON.stringify(fixtureReviewResponse) } }], - usage: { promptTokens: 10, completionTokens: 20 }, - })) - const client = createOpenRouterClient({ - sdk: { chat: { send }, generations: { getGeneration } }, - requestTimeoutMs: 900_000, - remainingReviewMs, - }) - const stubs = makeOrchestrateDeps({ - remainingReviewMs, - generateFindings: createPromptedGenerateFindings( - { - openrouterClient: client, - model: "test/model", - fallbackModel: "fallback/model", - }, - createTestLogger(), - ), - }) - const pending = orchestrate(stubs.deps, createTestLogger()) - await vi.advanceTimersByTimeAsync(1000) - const result = await pending - const expectedCost = renderCostSummary({ - attempts: [ - { - phase: "combined", - model: "test/model", - outcome: "accepted", - promptTokens: 10, - completionTokens: 20, - costUsd: null, - errorSummary: null, - }, - ], - modelUsed: "test/model", - }) - expect(remainingReviewMs()).toBe(0) - expect(send).toHaveBeenCalledTimes(1) - expect(getGeneration).toHaveBeenCalledExactlyOnceWith( - { id: "accepted-before-deadline" }, - { retries: { strategy: "none" }, signal: expect.any(AbortSignal) }, - ) - expect(result).toEqual({ - findingsCount: expectedSelection.selected.length, - reviewUrl: "https://github.com/test/review/1", - modelUsed: "test/model", - skippedReason: "", - phases: [{ phase: "combined", status: "completed" }], - reviewSummaryMarkdown: expectedReviewSummary(), - costSummaryMarkdown: expectedCost, - conventionsNote: null, - }) - expect(stubs.upsertSummaryCommentCalls).toEqual([ - expectedStatus({ - isFirstRun: true, - postedCount: expectedSelection.selected.length, - totalCount: expectedSelection.selected.length, - }), - ]) - expect(stubs.postFindingsReviewCalls).toEqual([ - expectedFindingsReview(expectedSelection.selected), - ]) - expect(stubs.updateCheckRunCalls).toEqual([ - { - checkRunId: 555, - conclusion: "success", - output: { - title: `${expectedSelection.selected.length} findings`, - summary: `Reviewed with \`test/model\` — ${expectedSelection.selected.length} findings posted.\n\n${expectedCost}`, - }, + it("publishes full coverage without a deadline warning when the review deadline runs out after every phase completed", async () => { + // The only phase's response spends the rest of this test's controllable deadline + const deadline = { remainingMs: 1 } + const stubs = makeOrchestrateDeps({ + remainingReviewMs: () => deadline.remainingMs, + generateFindings: async (reviewContext) => { + stubs.generateFindingsCalls.push(reviewContext) + deadline.remainingMs = 0 + return { + review: fixtureReviewResponse, + modelUsed: "test/model", + attempts: [fixtureAttempt], + } + }, + }) + + const result = await orchestrate(stubs.deps, createTestLogger()) + + expect(result).toEqual({ + findingsCount: expectedSelection.selected.length, + reviewUrl: "https://github.com/test/review/1", + modelUsed: "test/model", + skippedReason: "", + phases: [{ phase: "combined", status: "completed" }], + reviewSummaryMarkdown: expectedReviewSummary(), + costSummaryMarkdown: expectedCostSummary, + conventionsNote: null, + }) + expect(stubs.upsertSummaryCommentCalls).toEqual([ + expectedStatus({ + isFirstRun: true, + postedCount: expectedSelection.selected.length, + totalCount: expectedSelection.selected.length, + }), + ]) + expect(stubs.postFindingsReviewCalls).toEqual([ + expectedFindingsReview(expectedSelection.selected), + ]) + expect(stubs.updateCheckRunCalls).toEqual([ + { + checkRunId: 555, + conclusion: "success", + output: { + title: `${expectedSelection.selected.length} findings`, + summary: `Reviewed with \`test/model\` — ${expectedSelection.selected.length} findings posted.\n\n${expectedCostSummary}`, }, - ]) - } finally { - vi.useRealTimers() - } + }, + ]) }) it("publishes completed parallel phases once when a provider ignores the review deadline abort", async () => { @@ -4776,20 +4763,19 @@ describe("staged phases", () => { try { const deadline = Date.now() + 1000 const remainingReviewMs = () => deadline - Date.now() - const lateResponse = Promise.withResolvers() + const lateResponse = Promise.withResolvers() const response = { id: "completed", model: "test/model", - choices: [{ message: { content: JSON.stringify(fixtureReviewResponse) } }], - usage: { promptTokens: 10, completionTokens: 20, cost: 0.01 }, + usage: { inputTokens: 10, outputTokens: 20, cost: 0.01 }, } - const send = vi + const callModel = vi .fn() - .mockResolvedValueOnce(response) - .mockResolvedValueOnce(response) - .mockImplementation(() => lateResponse.promise) + .mockReturnValueOnce(makeModelResultStub(response)) + .mockReturnValueOnce(makeModelResultStub(response)) + .mockImplementation(() => makePendingModelResultStub(lateResponse.promise)) const client = createOpenRouterClient({ - sdk: { chat: { send }, generations: { getGeneration: vi.fn() } }, + agent: { callModel }, requestTimeoutMs: 900_000, remainingReviewMs, }) @@ -4808,7 +4794,7 @@ describe("staged phases", () => { const pending = orchestrate(stubs.deps, createTestLogger()) await vi.advanceTimersByTimeAsync(1000) const result = await pending - expect(send).toHaveBeenCalledTimes(3) + expect(callModel).toHaveBeenCalledTimes(3) expect(result.phases).toEqual([ { phase: "correctness-security", status: "completed" }, { phase: "conventions-tests", status: "completed" }, @@ -4858,7 +4844,7 @@ describe("staged phases", () => { reviews: stubs.postFindingsReviewCalls, result, }) - lateResponse.resolve(response) + lateResponse.resolve({ response, text: JSON.stringify(fixtureReviewResponse) }) await vi.runAllTimersAsync() expect({ checks: stubs.updateCheckRunCalls, @@ -4875,8 +4861,7 @@ describe("staged phases", () => { const response = { id: "completed", model: "test/model", - choices: [{ message: { content: JSON.stringify(fixtureReviewResponse) } }], - usage: { promptTokens: 10, completionTokens: 20, cost: 0.01 }, + usage: { inputTokens: 10, outputTokens: 20, cost: 0.01 }, } // correctness-security is dispatched first, so a logger shared across the @@ -4884,19 +4869,20 @@ describe("staged phases", () => { // The failure is keyed on that phase's prompt text, not on call order const correctnessSecurityPassScope = "this pass covers correctness & security" const failedPrompts = new Set() - const send = vi.fn(async ({ chatRequest }: { chatRequest: ChatRequestSubset }) => { - const systemPrompt = first(chatRequest.messages).content + const callModel = vi.fn(({ instructions }: ReviewRequest) => { const isFirstCorrectnessSecurityRequest = - systemPrompt.includes(correctnessSecurityPassScope) && !failedPrompts.has(systemPrompt) + instructions.includes(correctnessSecurityPassScope) && !failedPrompts.has(instructions) if (isFirstCorrectnessSecurityRequest) { - failedPrompts.add(systemPrompt) - throw Object.assign(new Error("HTTP 500"), { statusCode: 500 }) + failedPrompts.add(instructions) + return makePendingModelResultStub( + Promise.reject(Object.assign(new Error("HTTP 500"), { statusCode: 500 })), + ) } - return response + return makeModelResultStub(response) }) const client = createOpenRouterClient({ - sdk: { chat: { send }, generations: { getGeneration: vi.fn() } }, + agent: { callModel }, requestTimeoutMs: 900_000, remainingReviewMs: () => Infinity, retryDelayMs: 0, @@ -4934,7 +4920,7 @@ describe("staged phases", () => { { phase: "conventions-tests", status: "completed" }, { phase: "subtle-bugs", status: "completed" }, ]) - expect(send).toHaveBeenCalledTimes(4) + expect(callModel).toHaveBeenCalledTimes(4) expect(logsWithMessage(generateLogger, "review attempt failed")).toEqual([ { level: "warn", diff --git a/src/main.ts b/src/main.ts index 8cdb469d..2b39c6ff 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,6 +1,6 @@ import * as core from "@actions/core" import { context, getOctokit } from "@actions/github" -import { OpenRouter } from "@openrouter/sdk" +import { OpenRouter as OpenRouterAgent } from "@openrouter/agent" import envVar from "env-var" import { parseConfig, type RawInputs } from "./config.js" import { createContextReader } from "./context/workspace.js" @@ -12,6 +12,8 @@ import { createPromptedGenerateFindings, orchestrate } from "./orchestrate.js" const actionStartedAt = performance.now() const logger = createLogger("umm-actually") +// The action's own unawaited promises cannot reject, so a rejection here comes +// from a promise inside an SDK. Logging it keeps Node from exiting mid-review process.on("unhandledRejection", (error) => { logger.warn("unhandled promise rejection (likely SDK internal)", { error: describeError(error), @@ -20,6 +22,7 @@ process.on("unhandledRejection", (error) => { // Cleanups to run when the job is cancelled. It holds at most one entry, the // open check run's completion call, registered only while that run is open. +// A Set lets the unregister callback delete exactly the cleanup it added. // Mutable on purpose — signal handlers can only reach shared state const cancellationCleanups = new Set<() => Promise>() @@ -136,7 +139,7 @@ try { // createOpenRouterClient takes no logger because each review phase // passes its own phase-tagged logger to requestReview openrouterClient: createOpenRouterClient({ - sdk: new OpenRouter({ apiKey: config.openrouterApiKey }), + agent: new OpenRouterAgent({ apiKey: config.openrouterApiKey }), requestTimeoutMs: config.requestTimeoutSeconds * 1000, remainingReviewMs, }), diff --git a/src/openrouter/__tests__/client.test.ts b/src/openrouter/__tests__/client.test.ts index 7e61c5b8..de7b5ea3 100644 --- a/src/openrouter/__tests__/client.test.ts +++ b/src/openrouter/__tests__/client.test.ts @@ -1,113 +1,167 @@ import { readFileSync } from "node:fs" +import { openResponsesResultFromJSON } from "@openrouter/sdk/models" +import { + BadRequestResponseError, + OpenRouterError, + ResponseValidationError, + UnauthorizedResponseError, +} from "@openrouter/sdk/models/errors" import { describe, expect, it, vi } from "vitest" +import { z } from "zod" import { createTestLogger, logsWithMessage } from "../../__tests__/test-logger.js" import { reviewResponseJsonSchema } from "../../review/finding.js" import { createOpenRouterClient, ReviewRequestError, - type ChatRequestSubset, - type OpenRouterLike, + type AgentLike, + type ResponsesResult, + type ReviewRequest, } from "../client.js" const readJsonFixture = (name: string): unknown => { return JSON.parse(readFileSync(new URL(`../../../fixtures/${name}`, import.meta.url), "utf8")) } -const acceptedChatResult = readJsonFixture("openrouter.chat-result.json") -const invalidJsonChatResult = readJsonFixture("openrouter.invalid-json.json") -const schemaMismatchChatResult = readJsonFixture("openrouter.schema-mismatch.json") +// Raw Responses API payloads captured from live requests. The two variants +// are the accepted payload with only its message text edited +const acceptedWirePayload = readJsonFixture("openrouter.responses-accepted.json") +const invalidJsonWirePayload = readJsonFixture("openrouter.responses-invalid-json.json") +const schemaMismatchWirePayload = readJsonFixture("openrouter.responses-schema-mismatch.json") +const overflowErrorBody = readJsonFixture("openrouter.responses-overflow-body.json") +const badKeyErrorBody = readJsonFixture("openrouter.responses-bad-key-body.json") + +/** The generation id and usage the accepted wire payload carries. */ +const ACCEPTED_GENERATION_ID = "gen-1794440085-n7DSIWy7fBrBUcqMEvla" +const ACCEPTED_USAGE = { promptTokens: 54, completionTokens: 102, costUsd: 0.000021036 } -/** The review JSON embedded in the accepted chat-result fixture, parsed. */ +/** The review JSON in the accepted wire payload's message text, parsed. */ const acceptedReview = { - analysis: "src/greeter.ts: verified the empty-key guard and traced register() callers.", + analysis: + 'Reviewed the single-line change `export const greeting = "hello"`. This is a simple, valid module-level constant export. The syntax is correct, the value is a static string with no side effects, and naming follows common conventions. No security, correctness, style, or maintainability concerns were identified. As requested, there are no findings to report.', findings: [], } -/** Accepted-shape chat result whose usage carries no cost — the case that - * routes the client to the generation cost lookup. */ -const makeNoCostChatResult = (): unknown => ({ - id: "gen-no-cost", - model: "openai/gpt-5-mini", - choices: [{ message: { content: JSON.stringify(acceptedReview) } }], - usage: { promptTokens: 12000, completionTokens: 800, cost: null }, +/** The output items of a wire payload; every other field passes through. */ +const wirePayloadSchema = z.looseObject({ + output: z.array( + z.looseObject({ + type: z.string(), + content: z.array(z.looseObject({ type: z.string(), text: z.string() })).optional(), + }), + ), }) -type StubOptions = { signal?: AbortSignal } | undefined +/** The accepted payload also carries a priced usage block the variants edit. */ +const acceptedPayloadSchema = wirePayloadSchema.extend({ + usage: z.looseObject({ cost: z.number() }), +}) -/** `pending` hands the call's options to a custom promise — for a request - * that never settles, settles late, or honours the abort signal. */ -type StubResponse = - { value: unknown } | { error: unknown } | { pending: (options: StubOptions) => Promise } +const parseAcceptedPayload = (): z.infer => { + return acceptedPayloadSchema.parse(acceptedWirePayload) +} -const makeSdkStub = ({ - sendResponses, - generationResponses = [], -}: { - sendResponses: StubResponse[] - generationResponses?: StubResponse[] -}) => { - const sendCalls: { - chatRequest: ChatRequestSubset - options: StubOptions - }[] = [] - const generationCalls: { - id: string - options: StubOptions +/** The text the Agent SDK extracts from a payload: the first message item's + * output text, or "" when the output holds no message item. This mirrors the + * SDK's convention rather than calling it, so the tests pin the client's + * text-to-outcome mapping, not the SDK's extraction. */ +const extractMessageText = (payload: unknown): string => { + const message = wirePayloadSchema.parse(payload).output.find((item) => item.type === "message") + const outputText = message?.content?.find((part) => part.type === "output_text") + + if (!outputText) return "" + return outputText.text +} + +/** What the Agent SDK hands the client for a wire payload: the payload run + * through the SDK's own transform. The repo's @openrouter/sdk copy runs it + * here as a drift guard on hand-edited fixtures; the Agent SDK's bundled + * copy parses every field the client reads identically. */ +const transformWirePayload = (payload: unknown): ResponsesResult => { + const transformed = openResponsesResultFromJSON(JSON.stringify(payload)) + + if (!transformed.ok) throw transformed.error + return transformed.value +} + +/** What one `callModel` stub call resolves: the transformed response and the + * SDK-extracted text, as the client reads them. */ +type ModelCallResult = { response: ResponsesResult; text: string } + +const toModelCallResult = (payload: unknown): ModelCallResult => ({ + response: transformWirePayload(payload), + text: extractMessageText(payload), +}) + +const acceptedResult = toModelCallResult(acceptedWirePayload) +const invalidJsonResult = toModelCallResult(invalidJsonWirePayload) +const schemaMismatchResult = toModelCallResult(schemaMismatchWirePayload) + +/** The accepted payload with its usage cost removed. */ +const makeNoCostResult = (): ModelCallResult => { + const { usage, ...payload } = parseAcceptedPayload() + const usageWithoutCost = Object.fromEntries( + Object.entries(usage).filter(([key]) => key !== "cost"), + ) + return toModelCallResult({ ...payload, usage: usageWithoutCost }) +} + +/** The accepted payload with no usage block at all. */ +const makeNoUsageResult = (): ModelCallResult => { + return toModelCallResult({ ...parseAcceptedPayload(), usage: null }) +} + +/** The accepted payload with only its reasoning item, so the SDK extracts no text. */ +const makeReasoningOnlyResult = (): ModelCallResult => { + const payload = parseAcceptedPayload() + const reasoningItems = payload.output.filter((item) => item.type === "reasoning") + return toModelCallResult({ ...payload, output: reasoningItems }) +} + +/** `pending` hands the call's request to a custom promise — for a request + * that never settles, settles late, or honours the abort signal. */ +type StubModelResult = + | { value: ModelCallResult } + | { error: unknown } + | { pending: (request: { signal: AbortSignal }) => Promise } + +const makeClientStubs = ({ modelResults }: { modelResults: StubModelResult[] }) => { + const modelCalls: { + request: ReviewRequest & { signal: AbortSignal } + options: { retries?: { strategy: "backoff" | "none" } } | undefined }[] = [] - const takeNext = ({ - queue, - callCount, - operation, - options, - }: { - queue: StubResponse[] - callCount: number - operation: string - options: StubOptions - }): unknown => { - const next = queue[callCount - 1] + const takeNextModelResult = (request: { signal: AbortSignal }): Promise => { + const next = modelResults.at(modelCalls.length - 1) if (!next) { - throw new Error(`stub: unexpected ${operation} call #${callCount}`) + throw new Error(`stub: unexpected callModel call #${modelCalls.length}`) } if ("error" in next) throw next.error - if ("pending" in next) return next.pending(options) - return next.value + if ("pending" in next) return next.pending(request) + return Promise.resolve(next.value) } - const sdk: OpenRouterLike = { - chat: { - send: async (request, options) => { - sendCalls.push({ ...request, options }) - return takeNext({ - queue: sendResponses, - callCount: sendCalls.length, - operation: "chat.send", - options, - }) - }, - }, - generations: { - getGeneration: async (request, options) => { - generationCalls.push({ ...request, options }) - return takeNext({ - queue: generationResponses, - callCount: generationCalls.length, - operation: "generations.getGeneration", - options, - }) - }, + const agent: AgentLike = { + callModel: (request, options) => { + modelCalls.push({ request, options }) + + // Promise.try settles a synchronous throw from the queue as a rejection, + // the way the SDK's materialized stream rejects + const settled = Promise.try(takeNextModelResult, request) + return { + getResponse: async () => (await settled).response, + getText: async () => (await settled).text, + } }, } - return { sdk, sendCalls, generationCalls } + return { agent, modelCalls } } -const makeClient = (stub: { sdk: OpenRouterLike }) => { +const makeClient = (stub: { agent: AgentLike }) => { const logger = createTestLogger() const client = createOpenRouterClient({ - sdk: stub.sdk, + agent: stub.agent, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, retryDelayMs: 0, @@ -115,30 +169,87 @@ const makeClient = (stub: { sdk: OpenRouterLike }) => { return { client, logger } } -const makeStatusError = (statusCode: number): Error => { - return Object.assign(new Error(`HTTP ${statusCode}`), { statusCode }) +const RESPONSES_URL = "https://openrouter.ai/api/v1/responses" + +/** The HTTP metadata the SDK's error classes read their status and body from. */ +const makeHttpMeta = ( + statusCode: number, + body: string, +): { response: Response; request: Request; body: string } => ({ + response: new Response(body, { status: statusCode }), + request: new Request(RESPONSES_URL), + body, +}) + +/** An SDK error of the given status, as the SDK's base class carries it. The + * repo's @openrouter/sdk copy stands in for the copy bundled inside + * @openrouter/agent, which throws the production errors; both set + * `statusCode` from the HTTP response. An error without `statusCode` takes + * the status-less path the network-error test pins. */ +const makeStatusError = (statusCode: number, message = `HTTP ${statusCode}`): OpenRouterError => { + return new OpenRouterError(message, makeHttpMeta(statusCode, "")) } +/** The error the SDK throws when a non-streamed 200 body fails its result + * schema, built the way the SDK's response matcher builds it. */ +const makeResponseValidationError = (): ResponseValidationError => { + const body = JSON.stringify({ object: "response" }) + const message = "Response validation failed" + + return new ResponseValidationError(message, { + ...makeHttpMeta(200, body), + cause: new Error("output: Invalid input: expected array, received undefined"), + rawValue: JSON.parse(body), + rawMessage: message, + }) +} + +/** The `{ error: { code, message } }` body OpenRouter sends on a rejected request. */ +const errorBodySchema = z.object({ error: z.object({ code: z.number(), message: z.string() }) }) + +/** The SDK's 400 error built from a captured error body. */ +const makeBadRequestError = (body: unknown): BadRequestResponseError => { + return new BadRequestResponseError( + errorBodySchema.parse(body), + makeHttpMeta(400, JSON.stringify(body)), + ) +} + +/** The numbers in the captured overflow body's message: the window, the + * requested total, the input estimate and the output ceiling. */ +const OVERFLOW_NUMBERS_PATTERN = + /maximum context length is \d+ tokens\. However, you requested about \d+ tokens \(\d+ of text input, \d+ in the output\)/ + /** OpenRouter's 400 when the prompt plus the requested output ceiling exceeds - * the routed endpoint's context window, worded as the live API words it. */ + * the routed endpoint's context window: the captured body's wording with + * only its numbers replaced, so every fitted-retry test runs on the live + * wording the detector must match. */ const makeContextOverflowError = ({ contextLength, inputTokens, }: { contextLength: number inputTokens: number -}): Error => { - const message = `This endpoint's maximum context length is ${contextLength} tokens. However, you requested about ${inputTokens + 128_000} tokens (${inputTokens} of text input, 128000 in the output). Please reduce the length of either one, or use the context-compression plugin to compress your prompt automatically.` - return Object.assign(new Error(message), { statusCode: 400 }) +}): BadRequestResponseError => { + const capturedMessage = errorBodySchema.parse(overflowErrorBody).error.message + + if (!OVERFLOW_NUMBERS_PATTERN.test(capturedMessage)) { + throw new Error("the captured overflow body no longer carries the expected numbers") + } + const message = capturedMessage.replace( + OVERFLOW_NUMBERS_PATTERN, + `maximum context length is ${contextLength} tokens. However, you requested about ${inputTokens + 128_000} tokens (${inputTokens} of text input, 128000 in the output)`, + ) + return makeBadRequestError({ error: { code: 400, message } }) } -/** The model and output ceiling of every chat request sent, in order. */ +/** The model and output ceiling of every review request sent, in order. */ const sentCeilings = (stub: { - sendCalls: { chatRequest: ChatRequestSubset }[] -}): { model: string; maxCompletionTokens: number }[] => { - return stub.sendCalls.map(({ chatRequest }) => ({ - model: chatRequest.model, - maxCompletionTokens: chatRequest.maxCompletionTokens, + modelCalls: { request: ReviewRequest }[] +}): { model: string; maxOutputTokens: number }[] => { + return stub.modelCalls.map(({ request }) => ({ + model: request.model, + maxOutputTokens: request.maxOutputTokens, })) } @@ -157,16 +268,34 @@ const captureRejection = async (request: Promise): Promise => const requestParams = { systemPrompt: "system prompt", userPrompt: "user prompt", - model: "openai/gpt-5-mini", - fallbackModel: "anthropic/claude-haiku-4.5", + model: "z-ai/glm-5.3-flash", + fallbackModel: "deepseek/deepseek-v4.1-flash", } +describe("captured Responses fixtures", () => { + it("the overflow and bad-key bodies build the SDK's error classes with their status and message", () => { + const overflowError = makeBadRequestError(overflowErrorBody) + const badKeyError = new UnauthorizedResponseError( + errorBodySchema.parse(badKeyErrorBody), + makeHttpMeta(401, JSON.stringify(badKeyErrorBody)), + ) + + expect([overflowError, badKeyError].map((error) => [error.statusCode, error.message])).toEqual([ + [ + 400, + "This endpoint's maximum context length is 8192 tokens. However, you requested about 11593 tokens (4593 of text input, 7000 in the output). Please reduce the length of either one, or use the context-compression plugin to compress your prompt automatically.", + ], + [401, "User not found."], + ]) + }) +}) + describe("requestReview", () => { it("does not start an attempt when the review deadline has expired", async () => { - const stub = makeSdkStub({ sendResponses: [] }) + const stub = makeClientStubs({ modelResults: [] }) const logger = createTestLogger() const client = createOpenRouterClient({ - sdk: stub.sdk, + agent: stub.agent, requestTimeoutMs: 45_000, remainingReviewMs: () => 0, }) @@ -184,8 +313,7 @@ describe("requestReview", () => { keyRejected: false, deadlineExceeded: true, }) - expect(stub.sendCalls).toEqual([]) - expect(stub.generationCalls).toEqual([]) + expect(stub.modelCalls).toEqual([]) }) it.each(["primary", "fallback"])( @@ -193,9 +321,9 @@ describe("requestReview", () => { async (rung) => { vi.useFakeTimers() try { - const late = Promise.withResolvers() - const stub = makeSdkStub({ - sendResponses: [ + const late = Promise.withResolvers() + const stub = makeClientStubs({ + modelResults: [ ...(rung === "fallback" ? [{ error: makeStatusError(400) }] : []), { pending: () => late.promise }, ], @@ -203,7 +331,7 @@ describe("requestReview", () => { const deadline = performance.now() + 100 const logger = createTestLogger() const client = createOpenRouterClient({ - sdk: stub.sdk, + agent: stub.agent, requestTimeoutMs: 45_000, remainingReviewMs: () => deadline - performance.now(), }) @@ -246,9 +374,9 @@ describe("requestReview", () => { deadlineExceeded: true, }) expect( - stub.sendCalls.map(({ chatRequest, options }) => ({ - model: chatRequest.model, - aborted: options?.signal?.aborted, + stub.modelCalls.map(({ request }) => ({ + model: request.model, + aborted: request.signal.aborted, })), ).toEqual([ ...(rung === "fallback" ? [{ model: requestParams.model, aborted: false }] : []), @@ -257,10 +385,9 @@ describe("requestReview", () => { aborted: true, }, ]) - late.resolve(acceptedChatResult) + late.resolve(acceptedResult) await vi.advanceTimersByTimeAsync(1000) expect(error.attempts).toEqual(expectedAttempts) - expect(stub.generationCalls).toEqual([]) expect(vi.getTimerCount()).toBe(0) } finally { vi.useRealTimers() @@ -271,22 +398,11 @@ describe("requestReview", () => { it("expires during retry sleep without losing billed usage or starting another attempt", async () => { vi.useFakeTimers() try { - const stub = makeSdkStub({ - sendResponses: [ - { - value: { - id: "invalid", - model: requestParams.model, - choices: [{ message: { content: "not json" } }], - usage: { promptTokens: 10, completionTokens: 20, cost: 0.01 }, - }, - }, - ], - }) + const stub = makeClientStubs({ modelResults: [{ value: invalidJsonResult }] }) const deadline = performance.now() + 100 const logger = createTestLogger() const client = createOpenRouterClient({ - sdk: stub.sdk, + agent: stub.agent, requestTimeoutMs: 45_000, remainingReviewMs: () => deadline - performance.now(), }) @@ -306,218 +422,175 @@ describe("requestReview", () => { { model: requestParams.model, outcome: "invalid_json", - promptTokens: 10, - completionTokens: 20, - costUsd: 0.01, + ...ACCEPTED_USAGE, errorSummary: "response content is not valid JSON", }, ], }) - expect(stub.sendCalls.map(({ chatRequest }) => chatRequest.model)).toEqual([ - requestParams.model, - ]) + expect(stub.modelCalls.map(({ request }) => request.model)).toEqual([requestParams.model]) expect(vi.getTimerCount()).toBe(0) } finally { vi.useRealTimers() } }) - it("skips cost lookup when an accepted response exhausts the remaining review time", async () => { - const remainingReviewMs = vi.fn(() => 100) - const stub = makeSdkStub({ - sendResponses: [ + it("returns an accepted review whose response arrives as the review deadline runs out", async () => { + // The response spends the rest of this test's controllable deadline + const deadline = { remainingMs: 100 } + const stub = makeClientStubs({ + modelResults: [ { pending: async () => { - remainingReviewMs.mockReturnValue(0) - return makeNoCostChatResult() + deadline.remainingMs = 0 + return acceptedResult }, }, ], }) const logger = createTestLogger() const client = createOpenRouterClient({ - sdk: stub.sdk, + agent: stub.agent, requestTimeoutMs: 45_000, - remainingReviewMs, + remainingReviewMs: () => deadline.remainingMs, }) - expect(await client.requestReview(requestParams, logger)).toEqual({ + + const result = await client.requestReview(requestParams, logger) + + expect(result).toEqual({ review: acceptedReview, modelUsed: requestParams.model, attempts: [ { model: requestParams.model, outcome: "accepted", - promptTokens: 12000, - completionTokens: 800, - costUsd: null, + ...ACCEPTED_USAGE, errorSummary: null, }, ], }) - expect(stub.sendCalls.map(({ chatRequest }) => chatRequest.model)).toEqual([ - requestParams.model, - ]) - expect(stub.generationCalls).toEqual([]) - expect(logsWithMessage(logger, "generation cost lookup skipped")).toEqual([ - { - level: "debug", - message: "generation cost lookup skipped", - data: { - operation: "generation cost lookup", - generationId: "gen-no-cost", - reason: "review deadline exceeded", - }, - }, - ]) - }) - - it("retains an accepted review when cost lookup reaches the review deadline and ignores its late response", async () => { - vi.useFakeTimers() - try { - const late = Promise.withResolvers() - const stub = makeSdkStub({ - sendResponses: [{ value: makeNoCostChatResult() }], - generationResponses: [{ pending: () => late.promise }], - }) - const deadline = performance.now() + 100 - const logger = createTestLogger() - const client = createOpenRouterClient({ - sdk: stub.sdk, - requestTimeoutMs: 45_000, - remainingReviewMs: () => deadline - performance.now(), - }) - const request = client.requestReview(requestParams, logger) - await vi.advanceTimersByTimeAsync(100) - const result = await request - const expected = { - review: acceptedReview, - modelUsed: requestParams.model, - attempts: [ - { - model: requestParams.model, - outcome: "accepted", - promptTokens: 12000, - completionTokens: 800, - costUsd: null, - errorSummary: null, - }, - ], - } - expect(result).toEqual(expected) - expect( - stub.generationCalls.map(({ id, options }) => ({ - id, - aborted: options?.signal?.aborted, - })), - ).toEqual([{ id: "gen-no-cost", aborted: true }]) - late.resolve({ data: { totalCost: 42 } }) - await vi.advanceTimersByTimeAsync(1000) - expect(result).toEqual(expected) - expect(stub.sendCalls.map(({ chatRequest }) => chatRequest.model)).toEqual([ - requestParams.model, - ]) - expect(vi.getTimerCount()).toBe(0) - } finally { - vi.useRealTimers() - } + expect(stub.modelCalls.map(({ request }) => request.model)).toEqual([requestParams.model]) }) - it("sends the strict json_schema response format with the review schema and per-attempt timeout", async () => { - const stub = makeSdkStub({ sendResponses: [{ value: acceptedChatResult }] }) + it("sends one Responses request with the prompts, the full output ceiling, the strict review schema and an abort signal", async () => { + const stub = makeClientStubs({ modelResults: [{ value: acceptedResult }] }) const { client, logger } = makeClient(stub) await client.requestReview(requestParams, logger) - expect(stub.sendCalls).toEqual([ + expect(stub.modelCalls).toEqual([ { - chatRequest: { - model: "openai/gpt-5-mini", - messages: [ - { role: "system", content: "system prompt" }, - { role: "user", content: "user prompt" }, - ], - maxCompletionTokens: 128_000, - responseFormat: { - type: "json_schema", - jsonSchema: { + request: { + model: "z-ai/glm-5.3-flash", + instructions: "system prompt", + input: "user prompt", + maxOutputTokens: 128_000, + text: { + format: { + type: "json_schema", name: "review_response", strict: true, schema: reviewResponseJsonSchema, }, }, - stream: false, - }, - options: { - retries: { strategy: "none" }, signal: expect.any(AbortSignal), }, + options: { retries: { strategy: "none" } }, }, ]) }) it("returns the parsed review and a single accepted attempt on first success", async () => { - const stub = makeSdkStub({ sendResponses: [{ value: acceptedChatResult }] }) + const stub = makeClientStubs({ modelResults: [{ value: acceptedResult }] }) const { client, logger } = makeClient(stub) const result = await client.requestReview(requestParams, logger) expect(result).toEqual({ review: acceptedReview, - modelUsed: "openai/gpt-5-mini", + modelUsed: "z-ai/glm-5.3-flash", attempts: [ { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "accepted", - promptTokens: 12000, - completionTokens: 800, - costUsd: 0.0421, + promptTokens: 54, + completionTokens: 102, + costUsd: 0.000021036, errorSummary: null, }, ], }) }) + it("reports the response's routed model as modelUsed when it differs from the requested model", async () => { + const stub = makeClientStubs({ modelResults: [{ value: acceptedResult }] }) + const { client, logger } = makeClient(stub) + + const result = await client.requestReview( + { ...requestParams, model: "openrouter/auto", fallbackModel: null }, + logger, + ) + + expect(result).toEqual({ + review: acceptedReview, + modelUsed: "z-ai/glm-5.3-flash", + attempts: [ + { + model: "openrouter/auto", + outcome: "accepted", + ...ACCEPTED_USAGE, + errorSummary: null, + }, + ], + }) + expect(logsWithMessage(logger, "review response accepted")).toEqual([ + { + level: "info", + message: "review response accepted", + data: { + model: "openrouter/auto", + routedModel: "z-ai/glm-5.3-flash", + generationId: ACCEPTED_GENERATION_ID, + totalAttemptCount: 1, + }, + }, + ]) + }) + it("retries the same model once on invalid JSON, recording both attempts", async () => { - const stub = makeSdkStub({ - sendResponses: [{ value: invalidJsonChatResult }, { value: acceptedChatResult }], + const stub = makeClientStubs({ + modelResults: [{ value: invalidJsonResult }, { value: acceptedResult }], }) const { client, logger } = makeClient(stub) const result = await client.requestReview(requestParams, logger) - expect(stub.sendCalls.map((sendCall) => sendCall.chatRequest.model)).toEqual([ - "openai/gpt-5-mini", - "openai/gpt-5-mini", + expect(stub.modelCalls.map((modelCall) => modelCall.request.model)).toEqual([ + "z-ai/glm-5.3-flash", + "z-ai/glm-5.3-flash", ]) expect(result.attempts).toEqual([ { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "invalid_json", - promptTokens: 11800, - completionTokens: 42, - costUsd: null, + promptTokens: 54, + completionTokens: 102, + costUsd: 0.000021036, errorSummary: "response content is not valid JSON", }, { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "accepted", - promptTokens: 12000, - completionTokens: 800, - costUsd: 0.0421, + promptTokens: 54, + completionTokens: 102, + costUsd: 0.000021036, errorSummary: null, }, ]) }) - it("retries the same model once on empty content", async () => { - const emptyContentResult = { - id: "gen-empty", - model: "openai/gpt-5-mini", - choices: [{ message: { content: "" } }], - usage: { promptTokens: 10, completionTokens: 0, cost: null }, - } - const stub = makeSdkStub({ - sendResponses: [{ value: emptyContentResult }, { value: acceptedChatResult }], + it("records empty_content and retries the same model once when the SDK extracts no text", async () => { + const stub = makeClientStubs({ + modelResults: [{ value: makeReasoningOnlyResult() }, { value: acceptedResult }], }) const { client, logger } = makeClient(stub) @@ -525,48 +598,46 @@ describe("requestReview", () => { expect(result.attempts).toEqual([ { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "empty_content", - promptTokens: 10, - completionTokens: 0, - costUsd: null, + ...ACCEPTED_USAGE, errorSummary: "response had no text content", }, { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "accepted", - promptTokens: 12000, - completionTokens: 800, - costUsd: 0.0421, + promptTokens: 54, + completionTokens: 102, + costUsd: 0.000021036, errorSummary: null, }, ]) }) it("advances to the fallback model after two schema mismatches", async () => { - const stub = makeSdkStub({ - sendResponses: [ - { value: schemaMismatchChatResult }, - { value: schemaMismatchChatResult }, - { value: acceptedChatResult }, + const stub = makeClientStubs({ + modelResults: [ + { value: schemaMismatchResult }, + { value: schemaMismatchResult }, + { value: acceptedResult }, ], }) const { client, logger } = makeClient(stub) const result = await client.requestReview(requestParams, logger) - expect(stub.sendCalls.map((sendCall) => sendCall.chatRequest.model)).toEqual([ - "openai/gpt-5-mini", - "openai/gpt-5-mini", - "anthropic/claude-haiku-4.5", + expect(stub.modelCalls.map((modelCall) => modelCall.request.model)).toEqual([ + "z-ai/glm-5.3-flash", + "z-ai/glm-5.3-flash", + "deepseek/deepseek-v4.1-flash", ]) const schemaMismatchAttempt = { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "schema_mismatch", - promptTokens: 11900, - completionTokens: 12, - costUsd: 0.001, + promptTokens: 54, + completionTokens: 102, + costUsd: 0.000021036, errorSummary: "schema mismatch at analysis: Invalid input: expected string, received undefined", } @@ -574,92 +645,150 @@ describe("requestReview", () => { schemaMismatchAttempt, schemaMismatchAttempt, { - model: "anthropic/claude-haiku-4.5", + model: "deepseek/deepseek-v4.1-flash", outcome: "accepted", - promptTokens: 12000, - completionTokens: 800, - costUsd: 0.0421, + promptTokens: 54, + completionTokens: 102, + costUsd: 0.000021036, errorSummary: null, }, ]) }) - it("retries the same model once on a 429", async () => { - const stub = makeSdkStub({ - sendResponses: [{ error: makeStatusError(429) }, { value: acceptedChatResult }], + it.each([408, 429])("retries the same model once on a %i", async (statusCode) => { + const stub = makeClientStubs({ + modelResults: [{ error: makeStatusError(statusCode) }, { value: acceptedResult }], }) const { client, logger } = makeClient(stub) const result = await client.requestReview(requestParams, logger) - expect(stub.sendCalls.map((sendCall) => sendCall.chatRequest.model)).toEqual([ - "openai/gpt-5-mini", - "openai/gpt-5-mini", + expect(stub.modelCalls.map((modelCall) => modelCall.request.model)).toEqual([ + "z-ai/glm-5.3-flash", + "z-ai/glm-5.3-flash", ]) - expect(result.attempts[0]).toEqual({ - model: "openai/gpt-5-mini", - outcome: "api_error", - promptTokens: null, - completionTokens: null, - costUsd: null, - errorSummary: "HTTP 429: HTTP 429", + expect(result.attempts).toEqual([ + { + model: "z-ai/glm-5.3-flash", + outcome: "api_error", + promptTokens: null, + completionTokens: null, + costUsd: null, + errorSummary: `HTTP ${statusCode}: HTTP ${statusCode}`, + }, + { + model: "z-ai/glm-5.3-flash", + outcome: "accepted", + ...ACCEPTED_USAGE, + errorSummary: null, + }, + ]) + }) + + it("retries the same model once when the SDK rejects a non-streamed 200 body", async () => { + const stub = makeClientStubs({ + modelResults: [{ error: makeResponseValidationError() }, { value: acceptedResult }], }) + const { client, logger } = makeClient(stub) + + const result = await client.requestReview(requestParams, logger) + + expect(stub.modelCalls.map((modelCall) => modelCall.request.model)).toEqual([ + "z-ai/glm-5.3-flash", + "z-ai/glm-5.3-flash", + ]) + expect(result.attempts).toEqual([ + { + model: "z-ai/glm-5.3-flash", + outcome: "api_error", + promptTokens: null, + completionTokens: null, + costUsd: null, + errorSummary: "HTTP 200: Response validation failed", + }, + { + model: "z-ai/glm-5.3-flash", + outcome: "accepted", + ...ACCEPTED_USAGE, + errorSummary: null, + }, + ]) }) it("retries a status-less network error and records its bare message", async () => { - const stub = makeSdkStub({ - sendResponses: [{ error: new Error("socket hang up") }, { value: acceptedChatResult }], + const stub = makeClientStubs({ + modelResults: [{ error: new Error("socket hang up") }, { value: acceptedResult }], }) const { client, logger } = makeClient(stub) const result = await client.requestReview(requestParams, logger) - expect(stub.sendCalls.map((sendCall) => sendCall.chatRequest.model)).toEqual([ - "openai/gpt-5-mini", - "openai/gpt-5-mini", + expect(stub.modelCalls.map((modelCall) => modelCall.request.model)).toEqual([ + "z-ai/glm-5.3-flash", + "z-ai/glm-5.3-flash", + ]) + expect(result.attempts).toEqual([ + { + model: "z-ai/glm-5.3-flash", + outcome: "api_error", + promptTokens: null, + completionTokens: null, + costUsd: null, + errorSummary: "socket hang up", + }, + { + model: "z-ai/glm-5.3-flash", + outcome: "accepted", + ...ACCEPTED_USAGE, + errorSummary: null, + }, ]) - expect(result.attempts[0]).toEqual({ - model: "openai/gpt-5-mini", - outcome: "api_error", - promptTokens: null, - completionTokens: null, - costUsd: null, - errorSummary: "socket hang up", - }) }) it("retries an SDK-thrown TimeoutError as an api_error through the action's ladder, not the SDK's", async () => { const timeoutError = new Error("Request timed out") timeoutError.name = "TimeoutError" - const stub = makeSdkStub({ - sendResponses: [{ error: timeoutError }, { value: acceptedChatResult }], + const stub = makeClientStubs({ + modelResults: [{ error: timeoutError }, { value: acceptedResult }], }) const { client, logger } = makeClient(stub) const result = await client.requestReview(requestParams, logger) - expect(stub.sendCalls.map((sendCall) => sendCall.chatRequest.model)).toEqual([ - "openai/gpt-5-mini", - "openai/gpt-5-mini", - ]) - expect(stub.sendCalls[0]?.options).toEqual({ - retries: { strategy: "none" }, + const actionLadderCall = { + model: "z-ai/glm-5.3-flash", signal: expect.any(AbortSignal), - }) - expect(result.attempts[0]).toEqual({ - model: "openai/gpt-5-mini", - outcome: "api_error", - promptTokens: null, - completionTokens: null, - costUsd: null, - errorSummary: "Request timed out", - }) + options: { retries: { strategy: "none" } }, + } + expect( + stub.modelCalls.map(({ request, options }) => ({ + model: request.model, + signal: request.signal, + options, + })), + ).toEqual([actionLadderCall, actionLadderCall]) + expect(result.attempts).toEqual([ + { + model: "z-ai/glm-5.3-flash", + outcome: "api_error", + promptTokens: null, + completionTokens: null, + costUsd: null, + errorSummary: "Request timed out", + }, + { + model: "z-ai/glm-5.3-flash", + outcome: "accepted", + ...ACCEPTED_USAGE, + errorSummary: null, + }, + ]) expect(logsWithMessage(logger, "review attempt failed")).toEqual([ { level: "warn", message: "review attempt failed", data: { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", modelAttemptNumber: 1, outcome: "api_error", errorSummary: "Request timed out", @@ -669,15 +798,14 @@ describe("requestReview", () => { }) it("passes a live (non-aborted) AbortSignal to the SDK call", async () => { - const stub = makeSdkStub({ - sendResponses: [{ value: acceptedChatResult }], - generationResponses: [{ value: { data: { totalCost: 0.01 } } }], + const stub = makeClientStubs({ + modelResults: [{ value: acceptedResult }], }) const { client, logger } = makeClient(stub) await client.requestReview(requestParams, logger) - const signal = stub.sendCalls[0]?.options?.signal + const signal = stub.modelCalls[0]?.request.signal expect(signal).toBeInstanceOf(AbortSignal) expect(signal?.aborted).toBe(false) }) @@ -687,9 +815,9 @@ describe("requestReview", () => { try { // Shaped like the SDK's rejection: its own error wrapping the // DOMException the aborted fetch raised - const rejectOnAbort = (options: StubOptions): Promise => { + const rejectOnAbort = (request: { signal: AbortSignal }): Promise => { return new Promise((_resolve, reject) => { - options?.signal?.addEventListener("abort", () => { + request.signal.addEventListener("abort", () => { reject( Object.assign(new Error("Request aborted by client"), { name: "RequestAbortedError", @@ -699,12 +827,12 @@ describe("requestReview", () => { }) }) } - const stub = makeSdkStub({ - sendResponses: [{ pending: rejectOnAbort }, { pending: rejectOnAbort }], + const stub = makeClientStubs({ + modelResults: [{ pending: rejectOnAbort }, { pending: rejectOnAbort }], }) const logger = createTestLogger() const client = createOpenRouterClient({ - sdk: stub.sdk, + agent: stub.agent, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, retryDelayMs: 0, @@ -727,10 +855,10 @@ describe("requestReview", () => { await expect(reviewPromise).rejects.toThrow( new ReviewRequestError({ message: - "review request failed after 2 attempt(s): openai/gpt-5-mini: timeout (no response within 45s); openai/gpt-5-mini: timeout (no response within 45s)", + "review request failed after 2 attempt(s): z-ai/glm-5.3-flash: timeout (no response within 45s); z-ai/glm-5.3-flash: timeout (no response within 45s)", attempts: [ { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "timeout", promptTokens: null, completionTokens: null, @@ -738,7 +866,7 @@ describe("requestReview", () => { errorSummary: "no response within 45s", }, { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "timeout", promptTokens: null, completionTokens: null, @@ -749,15 +877,15 @@ describe("requestReview", () => { keyRejected: false, }), ) - expect(stub.sendCalls[0]?.options?.signal?.aborted).toBe(true) + expect(stub.modelCalls[0]?.request.signal.aborted).toBe(true) // One orphan settlement per timed-out attempt const abortedSettlement = { level: "warn", message: "deadline-elapsed request settled", data: { - operation: "chat request", - model: "openai/gpt-5-mini", + operation: "review request", + model: "z-ai/glm-5.3-flash", elapsedMs: 45_000, timeoutMs: 45_000, settledWith: "abort_error", @@ -776,15 +904,12 @@ describe("requestReview", () => { it("advances to the fallback model at the deadline even when the SDK ignores the abort signal", async () => { vi.useFakeTimers() try { - const stub = makeSdkStub({ - sendResponses: [ - { pending: () => new Promise(() => undefined) }, - { value: acceptedChatResult }, - ], + const stub = makeClientStubs({ + modelResults: [{ pending: () => new Promise(() => undefined) }, { value: acceptedResult }], }) const logger = createTestLogger() const client = createOpenRouterClient({ - sdk: stub.sdk, + agent: stub.agent, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, retryDelayMs: 0, @@ -794,13 +919,13 @@ describe("requestReview", () => { await vi.advanceTimersByTimeAsync(45_000) const result = await reviewPromise - expect(stub.sendCalls.map((sendCall) => sendCall.chatRequest.model)).toEqual([ - "openai/gpt-5-mini", - "anthropic/claude-haiku-4.5", + expect(stub.modelCalls.map((modelCall) => modelCall.request.model)).toEqual([ + "z-ai/glm-5.3-flash", + "deepseek/deepseek-v4.1-flash", ]) expect(result.attempts).toEqual([ { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "timeout", promptTokens: null, completionTokens: null, @@ -808,11 +933,11 @@ describe("requestReview", () => { errorSummary: "no response within 45s", }, { - model: "anthropic/claude-haiku-4.5", + model: "deepseek/deepseek-v4.1-flash", outcome: "accepted", - promptTokens: 12000, - completionTokens: 800, - costUsd: 0.0421, + promptTokens: 54, + completionTokens: 102, + costUsd: 0.000021036, errorSummary: null, }, ]) @@ -823,8 +948,8 @@ describe("requestReview", () => { level: "info", message: "advancing to fallback model without same-model retry", data: { - from: "openai/gpt-5-mini", - to: "anthropic/claude-haiku-4.5", + from: "z-ai/glm-5.3-flash", + to: "deepseek/deepseek-v4.1-flash", }, }, ]) @@ -833,8 +958,8 @@ describe("requestReview", () => { level: "warn", message: "request deadline elapsed", data: { - operation: "chat request", - model: "openai/gpt-5-mini", + operation: "review request", + model: "z-ai/glm-5.3-flash", timeoutMs: 45_000, }, }, @@ -844,7 +969,7 @@ describe("requestReview", () => { level: "warn", message: "review attempt failed", data: { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", modelAttemptNumber: 1, outcome: "timeout", errorSummary: "no response within 45s", @@ -859,17 +984,17 @@ describe("requestReview", () => { it("logs a late response when the orphaned request completes after the deadline", async () => { vi.useFakeTimers() try { - const resolveLate = (): Promise => { + const resolveLate = (): Promise => { return new Promise((resolve) => { - setTimeout(() => resolve(acceptedChatResult), 120_000) + setTimeout(() => resolve(acceptedResult), 120_000) }) } - const stub = makeSdkStub({ - sendResponses: [{ pending: resolveLate }, { value: acceptedChatResult }], + const stub = makeClientStubs({ + modelResults: [{ pending: resolveLate }, { value: acceptedResult }], }) const logger = createTestLogger() const client = createOpenRouterClient({ - sdk: stub.sdk, + agent: stub.agent, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, retryDelayMs: 0, @@ -879,8 +1004,8 @@ describe("requestReview", () => { await vi.advanceTimersByTimeAsync(45_000) const result = await reviewPromise expect(result.attempts.map((attempt) => [attempt.model, attempt.outcome])).toEqual([ - ["openai/gpt-5-mini", "timeout"], - ["anthropic/claude-haiku-4.5", "accepted"], + ["z-ai/glm-5.3-flash", "timeout"], + ["deepseek/deepseek-v4.1-flash", "accepted"], ]) await vi.advanceTimersByTimeAsync(75_000) @@ -890,8 +1015,8 @@ describe("requestReview", () => { level: "warn", message: "deadline-elapsed request settled", data: { - operation: "chat request", - model: "openai/gpt-5-mini", + operation: "review request", + model: "z-ai/glm-5.3-flash", elapsedMs: 120_000, timeoutMs: 45_000, settledWith: "response", @@ -906,17 +1031,17 @@ describe("requestReview", () => { it("tags a late settlement with the caller's logger context after the request has rejected", async () => { vi.useFakeTimers() try { - const resolveLate = (): Promise => { + const resolveLate = (): Promise => { return new Promise((resolve) => { - setTimeout(() => resolve(acceptedChatResult), 120_000) + setTimeout(() => resolve(acceptedResult), 120_000) }) } - const stub = makeSdkStub({ - sendResponses: [{ pending: resolveLate }, { error: makeStatusError(400) }], + const stub = makeClientStubs({ + modelResults: [{ pending: resolveLate }, { error: makeStatusError(400) }], }) const logger = createTestLogger() const client = createOpenRouterClient({ - sdk: stub.sdk, + agent: stub.agent, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, retryDelayMs: 0, @@ -933,7 +1058,7 @@ describe("requestReview", () => { if (!(error instanceof ReviewRequestError)) throw error expect(error.message).toBe( - "review request failed after 2 attempt(s): openai/gpt-5-mini: timeout (no response within 45s); openai/gpt-5-mini: api_error (HTTP 400: HTTP 400)", + "review request failed after 2 attempt(s): z-ai/glm-5.3-flash: timeout (no response within 45s); z-ai/glm-5.3-flash: api_error (HTTP 400: HTTP 400)", ) expect(logsWithMessage(logger, "deadline-elapsed request settled")).toEqual([]) @@ -945,8 +1070,8 @@ describe("requestReview", () => { message: "deadline-elapsed request settled", data: { phase: "subtle-bugs", - operation: "chat request", - model: "openai/gpt-5-mini", + operation: "review request", + model: "z-ai/glm-5.3-flash", elapsedMs: 120_000, timeoutMs: 45_000, settledWith: "response", @@ -961,17 +1086,17 @@ describe("requestReview", () => { it("logs a late non-abort rejection without raising an unhandled rejection", async () => { vi.useFakeTimers() try { - const rejectLate = (): Promise => { + const rejectLate = (): Promise => { return new Promise((_resolve, reject) => { setTimeout(() => reject(new Error("socket hang up")), 120_000) }) } - const stub = makeSdkStub({ - sendResponses: [{ pending: rejectLate }, { value: acceptedChatResult }], + const stub = makeClientStubs({ + modelResults: [{ pending: rejectLate }, { value: acceptedResult }], }) const logger = createTestLogger() const client = createOpenRouterClient({ - sdk: stub.sdk, + agent: stub.agent, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, retryDelayMs: 0, @@ -981,8 +1106,8 @@ describe("requestReview", () => { await vi.advanceTimersByTimeAsync(45_000) const result = await reviewPromise expect(result.attempts.map((attempt) => [attempt.model, attempt.outcome])).toEqual([ - ["openai/gpt-5-mini", "timeout"], - ["anthropic/claude-haiku-4.5", "accepted"], + ["z-ai/glm-5.3-flash", "timeout"], + ["deepseek/deepseek-v4.1-flash", "accepted"], ]) await vi.advanceTimersByTimeAsync(75_000) @@ -991,8 +1116,8 @@ describe("requestReview", () => { level: "warn", message: "deadline-elapsed request settled", data: { - operation: "chat request", - model: "openai/gpt-5-mini", + operation: "review request", + model: "z-ai/glm-5.3-flash", elapsedMs: 120_000, timeoutMs: 45_000, settledWith: "error", @@ -1008,15 +1133,12 @@ describe("requestReview", () => { it("retries a timeout on the same model when no fallback is configured", async () => { vi.useFakeTimers() try { - const stub = makeSdkStub({ - sendResponses: [ - { pending: () => new Promise(() => undefined) }, - { value: acceptedChatResult }, - ], + const stub = makeClientStubs({ + modelResults: [{ pending: () => new Promise(() => undefined) }, { value: acceptedResult }], }) const logger = createTestLogger() const client = createOpenRouterClient({ - sdk: stub.sdk, + agent: stub.agent, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, retryDelayMs: 0, @@ -1034,7 +1156,7 @@ describe("requestReview", () => { expect(result.attempts).toEqual([ { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "timeout", promptTokens: null, completionTokens: null, @@ -1042,11 +1164,11 @@ describe("requestReview", () => { errorSummary: "no response within 45s", }, { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "accepted", - promptTokens: 12000, - completionTokens: 800, - costUsd: 0.0421, + promptTokens: 54, + completionTokens: 102, + costUsd: 0.000021036, errorSummary: null, }, ]) @@ -1061,9 +1183,9 @@ describe("requestReview", () => { it("exhausts the ladder in order when the fallback also times out, retrying only the last rung", async () => { vi.useFakeTimers() try { - const neverSettle = (): Promise => new Promise(() => undefined) - const stub = makeSdkStub({ - sendResponses: [ + const neverSettle = (): Promise => new Promise(() => undefined) + const stub = makeClientStubs({ + modelResults: [ { pending: neverSettle }, { pending: neverSettle }, { pending: neverSettle }, @@ -1071,7 +1193,7 @@ describe("requestReview", () => { }) const logger = createTestLogger() const client = createOpenRouterClient({ - sdk: stub.sdk, + agent: stub.agent, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, retryDelayMs: 0, @@ -1085,10 +1207,10 @@ describe("requestReview", () => { await expect(reviewPromise).rejects.toThrow( new ReviewRequestError({ message: - "review request failed after 3 attempt(s): openai/gpt-5-mini: timeout (no response within 45s); anthropic/claude-haiku-4.5: timeout (no response within 45s); anthropic/claude-haiku-4.5: timeout (no response within 45s)", + "review request failed after 3 attempt(s): z-ai/glm-5.3-flash: timeout (no response within 45s); deepseek/deepseek-v4.1-flash: timeout (no response within 45s); deepseek/deepseek-v4.1-flash: timeout (no response within 45s)", attempts: [ { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "timeout", promptTokens: null, completionTokens: null, @@ -1096,7 +1218,7 @@ describe("requestReview", () => { errorSummary: "no response within 45s", }, { - model: "anthropic/claude-haiku-4.5", + model: "deepseek/deepseek-v4.1-flash", outcome: "timeout", promptTokens: null, completionTokens: null, @@ -1104,7 +1226,7 @@ describe("requestReview", () => { errorSummary: "no response within 45s", }, { - model: "anthropic/claude-haiku-4.5", + model: "deepseek/deepseek-v4.1-flash", outcome: "timeout", promptTokens: null, completionTokens: null, @@ -1115,10 +1237,10 @@ describe("requestReview", () => { keyRejected: false, }), ) - expect(stub.sendCalls.map((sendCall) => sendCall.chatRequest.model)).toEqual([ - "openai/gpt-5-mini", - "anthropic/claude-haiku-4.5", - "anthropic/claude-haiku-4.5", + expect(stub.modelCalls.map((modelCall) => modelCall.request.model)).toEqual([ + "z-ai/glm-5.3-flash", + "deepseek/deepseek-v4.1-flash", + "deepseek/deepseek-v4.1-flash", ]) } finally { vi.useRealTimers() @@ -1129,15 +1251,12 @@ describe("requestReview", () => { vi.useFakeTimers() try { const retryDelayMs = 50 - const stub = makeSdkStub({ - sendResponses: [ - { pending: () => new Promise(() => undefined) }, - { value: acceptedChatResult }, - ], + const stub = makeClientStubs({ + modelResults: [{ pending: () => new Promise(() => undefined) }, { value: acceptedResult }], }) const logger = createTestLogger() const client = createOpenRouterClient({ - sdk: stub.sdk, + agent: stub.agent, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, retryDelayMs, @@ -1159,120 +1278,30 @@ describe("requestReview", () => { } }) - it("reports the review deadline as the cost lookup's failure when that deadline cuts the lookup short", async () => { + it("clears the review attempt timeout timer after a successful response", async () => { vi.useFakeTimers() try { - const stub = makeSdkStub({ - sendResponses: [{ value: makeNoCostChatResult() }], - generationResponses: [{ pending: () => new Promise(() => undefined) }], - }) - const logger = createTestLogger() - const deadline = performance.now() + 100 - const client = createOpenRouterClient({ - sdk: stub.sdk, - requestTimeoutMs: 45_000, - remainingReviewMs: () => deadline - performance.now(), - }) - - const reviewPromise = client.requestReview(requestParams, logger) - await vi.advanceTimersByTimeAsync(100) - await reviewPromise - - expect(logger.messages.filter((entry) => entry.level === "warn")).toEqual([ - { - level: "warn", - message: "request deadline elapsed", - data: { - operation: "generation cost lookup", - generationId: "gen-no-cost", - timeoutMs: 100, - }, - }, - { - level: "warn", - message: "generation cost lookup failed", - data: { - operation: "generation cost lookup", - generationId: "gen-no-cost", - error: "review deadline exceeded", - }, - }, - ]) - } finally { - vi.useRealTimers() - } - }) - - it("degrades to a null cost with both warnings when the generation lookup exceeds the deadline", async () => { - vi.useFakeTimers() - try { - const stub = makeSdkStub({ - sendResponses: [{ value: makeNoCostChatResult() }], - generationResponses: [{ pending: () => new Promise(() => undefined) }], - }) - const logger = createTestLogger() - const client = createOpenRouterClient({ - sdk: stub.sdk, - remainingReviewMs: () => Infinity, - requestTimeoutMs: 45_000, - retryDelayMs: 0, + const stub = makeClientStubs({ + modelResults: [{ value: acceptedResult }], }) + const { client, logger } = makeClient(stub) - const reviewPromise = client.requestReview(requestParams, logger) - await vi.advanceTimersByTimeAsync(45_000) - const result = await reviewPromise + await client.requestReview(requestParams, logger) - expect(result.attempts[0]?.costUsd).toBeNull() - expect(logsWithMessage(logger, "request deadline elapsed")).toEqual([ - { - level: "warn", - message: "request deadline elapsed", - data: { - operation: "generation cost lookup", - generationId: "gen-no-cost", - timeoutMs: 45_000, - }, - }, - ]) - expect(logsWithMessage(logger, "generation cost lookup failed")).toEqual([ - { - level: "warn", - message: "generation cost lookup failed", - data: { - operation: "generation cost lookup", - generationId: "gen-no-cost", - error: "no response within 45s", - }, - }, - ]) + expect(vi.getTimerCount()).toBe(0) } finally { vi.useRealTimers() } }) - it("clears the chat attempt timeout timer after a successful response", async () => { - const stub = makeSdkStub({ - sendResponses: [{ value: acceptedChatResult }], - }) - const { client, logger } = makeClient(stub) - - const clearTimeoutSpy = vi.spyOn(globalThis, "clearTimeout") - try { - await client.requestReview(requestParams, logger) - expect(clearTimeoutSpy).toHaveBeenCalled() - } finally { - clearTimeoutSpy.mockRestore() - } - }) - it("does not sleep after the final attempt when retries are exhausted", async () => { const retryDelayMs = 50 - const stub = makeSdkStub({ - sendResponses: [{ error: makeStatusError(429) }, { error: makeStatusError(429) }], + const stub = makeClientStubs({ + modelResults: [{ error: makeStatusError(429) }, { error: makeStatusError(429) }], }) const logger = createTestLogger() const client = createOpenRouterClient({ - sdk: stub.sdk, + agent: stub.agent, remainingReviewMs: () => Infinity, requestTimeoutMs: 45_000, retryDelayMs, @@ -1285,10 +1314,10 @@ describe("requestReview", () => { ).rejects.toThrow( new ReviewRequestError({ message: - "review request failed after 2 attempt(s): openai/gpt-5-mini: api_error (HTTP 429: HTTP 429); openai/gpt-5-mini: api_error (HTTP 429: HTTP 429)", + "review request failed after 2 attempt(s): z-ai/glm-5.3-flash: api_error (HTTP 429: HTTP 429); z-ai/glm-5.3-flash: api_error (HTTP 429: HTTP 429)", attempts: [ { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "api_error", promptTokens: null, completionTokens: null, @@ -1296,7 +1325,7 @@ describe("requestReview", () => { errorSummary: "HTTP 429: HTTP 429", }, { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "api_error", promptTokens: null, completionTokens: null, @@ -1313,45 +1342,53 @@ describe("requestReview", () => { // after the final attempt. Only one sleep (between attempts 1 and 2). const retrySleepCalls = setTimeoutSpy.mock.calls.filter((call) => call[1] === retryDelayMs) expect(retrySleepCalls).toHaveLength(1) - expect(stub.sendCalls).toHaveLength(2) + expect(stub.modelCalls).toHaveLength(2) } finally { setTimeoutSpy.mockRestore() } }) it("truncates an error message longer than 200 characters in the attempt summary", async () => { - const stub = makeSdkStub({ - sendResponses: [ - { - error: Object.assign(new Error("x".repeat(250)), { - statusCode: 429, - }), - }, - { value: acceptedChatResult }, - ], + const stub = makeClientStubs({ + modelResults: [{ error: makeStatusError(429, "x".repeat(250)) }, { value: acceptedResult }], }) const { client, logger } = makeClient(stub) const result = await client.requestReview(requestParams, logger) - expect(result.attempts[0]?.errorSummary).toBe(`HTTP 429: ${"x".repeat(200)}…`) + expect(result.attempts).toEqual([ + { + model: "z-ai/glm-5.3-flash", + outcome: "api_error", + promptTokens: null, + completionTokens: null, + costUsd: null, + errorSummary: `HTTP 429: ${"x".repeat(200)}…`, + }, + { + model: "z-ai/glm-5.3-flash", + outcome: "accepted", + ...ACCEPTED_USAGE, + errorSummary: null, + }, + ]) }) it("advances to the fallback model without a retry on a 404", async () => { - const stub = makeSdkStub({ - sendResponses: [{ error: makeStatusError(404) }, { value: acceptedChatResult }], + const stub = makeClientStubs({ + modelResults: [{ error: makeStatusError(404) }, { value: acceptedResult }], }) const { client, logger } = makeClient(stub) const result = await client.requestReview(requestParams, logger) - expect(stub.sendCalls.map((sendCall) => sendCall.chatRequest.model)).toEqual([ - "openai/gpt-5-mini", - "anthropic/claude-haiku-4.5", + expect(stub.modelCalls.map((modelCall) => modelCall.request.model)).toEqual([ + "z-ai/glm-5.3-flash", + "deepseek/deepseek-v4.1-flash", ]) expect(result.attempts).toEqual([ { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "api_error", promptTokens: null, completionTokens: null, @@ -1359,21 +1396,21 @@ describe("requestReview", () => { errorSummary: "HTTP 404: HTTP 404", }, { - model: "anthropic/claude-haiku-4.5", + model: "deepseek/deepseek-v4.1-flash", outcome: "accepted", - promptTokens: 12000, - completionTokens: 800, - costUsd: 0.0421, + promptTokens: 54, + completionTokens: 102, + costUsd: 0.000021036, errorSummary: null, }, ]) }) it("retries the same model with an output ceiling that fits after a context-overflow 400", async () => { - const stub = makeSdkStub({ - sendResponses: [ + const stub = makeClientStubs({ + modelResults: [ { error: makeContextOverflowError({ contextLength: 262_144, inputTokens: 135_762 }) }, - { value: acceptedChatResult }, + { value: acceptedResult }, ], }) const { client, logger } = makeClient(stub) @@ -1382,15 +1419,15 @@ describe("requestReview", () => { // 262,144 window − 135,762 input − 8,192 margin expect(sentCeilings(stub)).toEqual([ - { model: "openai/gpt-5-mini", maxCompletionTokens: 128_000 }, - { model: "openai/gpt-5-mini", maxCompletionTokens: 118_190 }, + { model: "z-ai/glm-5.3-flash", maxOutputTokens: 128_000 }, + { model: "z-ai/glm-5.3-flash", maxOutputTokens: 118_190 }, ]) expect(result).toEqual({ review: acceptedReview, - modelUsed: "openai/gpt-5-mini", + modelUsed: "z-ai/glm-5.3-flash", attempts: [ { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "api_error", promptTokens: null, completionTokens: null, @@ -1399,11 +1436,11 @@ describe("requestReview", () => { "HTTP 400: This endpoint's maximum context length is 262144 tokens. However, you requested about 263762 tokens (135762 of text input, 128000 in the output). Please reduce the length of either one, or use the con…", }, { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "accepted", - promptTokens: 12000, - completionTokens: 800, - costUsd: 0.0421, + promptTokens: 54, + completionTokens: 102, + costUsd: 0.000021036, errorSummary: null, }, ], @@ -1412,17 +1449,17 @@ describe("requestReview", () => { { level: "info", message: CEILING_RETRY_LOG, - data: { model: "openai/gpt-5-mini", maxCompletionTokens: 118_190 }, + data: { model: "z-ai/glm-5.3-flash", maxOutputTokens: 118_190 }, }, ]) }) it("starts the fallback model at the full output ceiling after the primary lowered it", async () => { - const stub = makeSdkStub({ - sendResponses: [ + const stub = makeClientStubs({ + modelResults: [ { error: makeContextOverflowError({ contextLength: 262_144, inputTokens: 135_762 }) }, { error: makeStatusError(500) }, - { value: acceptedChatResult }, + { value: acceptedResult }, ], }) const { client, logger } = makeClient(stub) @@ -1430,17 +1467,17 @@ describe("requestReview", () => { await client.requestReview(requestParams, logger) expect(sentCeilings(stub)).toEqual([ - { model: "openai/gpt-5-mini", maxCompletionTokens: 128_000 }, - { model: "openai/gpt-5-mini", maxCompletionTokens: 118_190 }, - { model: "anthropic/claude-haiku-4.5", maxCompletionTokens: 128_000 }, + { model: "z-ai/glm-5.3-flash", maxOutputTokens: 128_000 }, + { model: "z-ai/glm-5.3-flash", maxOutputTokens: 118_190 }, + { model: "deepseek/deepseek-v4.1-flash", maxOutputTokens: 128_000 }, ]) }) it("advances to the fallback without a retry when a context-overflow 400 leaves no room for output", async () => { - const stub = makeSdkStub({ - sendResponses: [ + const stub = makeClientStubs({ + modelResults: [ { error: makeContextOverflowError({ contextLength: 262_144, inputTokens: 258_000 }) }, - { value: acceptedChatResult }, + { value: acceptedResult }, ], }) const { client, logger } = makeClient(stub) @@ -1448,17 +1485,17 @@ describe("requestReview", () => { await client.requestReview(requestParams, logger) expect(sentCeilings(stub)).toEqual([ - { model: "openai/gpt-5-mini", maxCompletionTokens: 128_000 }, - { model: "anthropic/claude-haiku-4.5", maxCompletionTokens: 128_000 }, + { model: "z-ai/glm-5.3-flash", maxOutputTokens: 128_000 }, + { model: "deepseek/deepseek-v4.1-flash", maxOutputTokens: 128_000 }, ]) }) it("advances to the fallback without a retry when the fitted output ceiling is too small for a review", async () => { // 262,144 window − 240,000 input − 8,192 margin = 13,952, under the 32,768 floor - const stub = makeSdkStub({ - sendResponses: [ + const stub = makeClientStubs({ + modelResults: [ { error: makeContextOverflowError({ contextLength: 262_144, inputTokens: 240_000 }) }, - { value: acceptedChatResult }, + { value: acceptedResult }, ], }) const { client, logger } = makeClient(stub) @@ -1466,17 +1503,17 @@ describe("requestReview", () => { await client.requestReview(requestParams, logger) expect(sentCeilings(stub)).toEqual([ - { model: "openai/gpt-5-mini", maxCompletionTokens: 128_000 }, - { model: "anthropic/claude-haiku-4.5", maxCompletionTokens: 128_000 }, + { model: "z-ai/glm-5.3-flash", maxOutputTokens: 128_000 }, + { model: "deepseek/deepseek-v4.1-flash", maxOutputTokens: 128_000 }, ]) }) it("retries at a fitted output ceiling exactly at the 32,768-token floor", async () => { // 262,144 window − 221,184 input − 8,192 margin = 32,768, the floor itself - const stub = makeSdkStub({ - sendResponses: [ + const stub = makeClientStubs({ + modelResults: [ { error: makeContextOverflowError({ contextLength: 262_144, inputTokens: 221_184 }) }, - { value: acceptedChatResult }, + { value: acceptedResult }, ], }) const { client, logger } = makeClient(stub) @@ -1484,18 +1521,18 @@ describe("requestReview", () => { await client.requestReview(requestParams, logger) expect(sentCeilings(stub)).toEqual([ - { model: "openai/gpt-5-mini", maxCompletionTokens: 128_000 }, - { model: "openai/gpt-5-mini", maxCompletionTokens: 32_768 }, + { model: "z-ai/glm-5.3-flash", maxOutputTokens: 128_000 }, + { model: "z-ai/glm-5.3-flash", maxOutputTokens: 32_768 }, ]) }) it("advances to the fallback without a retry when the rejected ceiling already fit the endpoint's window", async () => { // 262,144 window − 125,952 input − 8,192 margin = 128,000, the rejected // ceiling itself, so lowering it cannot fix the 400 - const stub = makeSdkStub({ - sendResponses: [ + const stub = makeClientStubs({ + modelResults: [ { error: makeContextOverflowError({ contextLength: 262_144, inputTokens: 125_952 }) }, - { value: acceptedChatResult }, + { value: acceptedResult }, ], }) const { client, logger } = makeClient(stub) @@ -1503,18 +1540,18 @@ describe("requestReview", () => { await client.requestReview(requestParams, logger) expect(sentCeilings(stub)).toEqual([ - { model: "openai/gpt-5-mini", maxCompletionTokens: 128_000 }, - { model: "anthropic/claude-haiku-4.5", maxCompletionTokens: 128_000 }, + { model: "z-ai/glm-5.3-flash", maxOutputTokens: 128_000 }, + { model: "deepseek/deepseek-v4.1-flash", maxOutputTokens: 128_000 }, ]) expect(logsWithMessage(logger, CEILING_RETRY_LOG)).toEqual([]) }) it("does not lower the ceiling when the context-overflow 400 was the model's last attempt", async () => { - const stub = makeSdkStub({ - sendResponses: [ + const stub = makeClientStubs({ + modelResults: [ { error: makeStatusError(500) }, { error: makeContextOverflowError({ contextLength: 262_144, inputTokens: 135_762 }) }, - { value: acceptedChatResult }, + { value: acceptedResult }, ], }) const { client, logger } = makeClient(stub) @@ -1522,59 +1559,118 @@ describe("requestReview", () => { await client.requestReview(requestParams, logger) expect(sentCeilings(stub)).toEqual([ - { model: "openai/gpt-5-mini", maxCompletionTokens: 128_000 }, - { model: "openai/gpt-5-mini", maxCompletionTokens: 128_000 }, - { model: "anthropic/claude-haiku-4.5", maxCompletionTokens: 128_000 }, + { model: "z-ai/glm-5.3-flash", maxOutputTokens: 128_000 }, + { model: "z-ai/glm-5.3-flash", maxOutputTokens: 128_000 }, + { model: "deepseek/deepseek-v4.1-flash", maxOutputTokens: 128_000 }, ]) expect(logsWithMessage(logger, CEILING_RETRY_LOG)).toEqual([]) }) it("keeps the full ceiling when a non-400 error carries context-overflow wording", async () => { - const overflowWordedGatewayError = Object.assign( - makeContextOverflowError({ contextLength: 262_144, inputTokens: 135_762 }), - { statusCode: 502 }, + const overflowWordedGatewayError = makeStatusError( + 502, + makeContextOverflowError({ contextLength: 262_144, inputTokens: 135_762 }).message, ) - const stub = makeSdkStub({ - sendResponses: [{ error: overflowWordedGatewayError }, { value: acceptedChatResult }], + const stub = makeClientStubs({ + modelResults: [{ error: overflowWordedGatewayError }, { value: acceptedResult }], }) const { client, logger } = makeClient(stub) await client.requestReview(requestParams, logger) expect(sentCeilings(stub)).toEqual([ - { model: "openai/gpt-5-mini", maxCompletionTokens: 128_000 }, - { model: "openai/gpt-5-mini", maxCompletionTokens: 128_000 }, + { model: "z-ai/glm-5.3-flash", maxOutputTokens: 128_000 }, + { model: "z-ai/glm-5.3-flash", maxOutputTokens: 128_000 }, ]) }) it("advances to the fallback without a retry on a 400 that is not a context overflow", async () => { - const stub = makeSdkStub({ - sendResponses: [{ error: makeStatusError(400) }, { value: acceptedChatResult }], + const stub = makeClientStubs({ + modelResults: [{ error: makeStatusError(400) }, { value: acceptedResult }], }) const { client, logger } = makeClient(stub) await client.requestReview(requestParams, logger) expect(sentCeilings(stub)).toEqual([ - { model: "openai/gpt-5-mini", maxCompletionTokens: 128_000 }, - { model: "anthropic/claude-haiku-4.5", maxCompletionTokens: 128_000 }, + { model: "z-ai/glm-5.3-flash", maxOutputTokens: 128_000 }, + { model: "deepseek/deepseek-v4.1-flash", maxOutputTokens: 128_000 }, ]) }) + it("advances to the fallback on the captured overflow body, whose 8K window leaves no room for a retry", async () => { + const stub = makeClientStubs({ + modelResults: [{ error: makeBadRequestError(overflowErrorBody) }, { value: acceptedResult }], + }) + const { client, logger } = makeClient(stub) + + const result = await client.requestReview(requestParams, logger) + + expect(sentCeilings(stub)).toEqual([ + { model: "z-ai/glm-5.3-flash", maxOutputTokens: 128_000 }, + { model: "deepseek/deepseek-v4.1-flash", maxOutputTokens: 128_000 }, + ]) + expect(result.attempts).toEqual([ + { + model: "z-ai/glm-5.3-flash", + outcome: "api_error", + promptTokens: null, + completionTokens: null, + costUsd: null, + errorSummary: + "HTTP 400: This endpoint's maximum context length is 8192 tokens. However, you requested about 11593 tokens (4593 of text input, 7000 in the output). Please reduce the length of either one, or use the context-co…", + }, + { + model: "deepseek/deepseek-v4.1-flash", + outcome: "accepted", + ...ACCEPTED_USAGE, + errorSummary: null, + }, + ]) + }) + + it("aborts the ladder after exactly one call on the captured bad-key body", async () => { + const badKeyError = new UnauthorizedResponseError( + errorBodySchema.parse(badKeyErrorBody), + makeHttpMeta(401, JSON.stringify(badKeyErrorBody)), + ) + const stub = makeClientStubs({ modelResults: [{ error: badKeyError }] }) + const { client, logger } = makeClient(stub) + + await expect(client.requestReview(requestParams, logger)).rejects.toThrow( + new ReviewRequestError({ + message: + "OpenRouter auth/credit error — aborting without fallback: z-ai/glm-5.3-flash: api_error (HTTP 401: User not found.)", + attempts: [ + { + model: "z-ai/glm-5.3-flash", + outcome: "api_error", + promptTokens: null, + completionTokens: null, + costUsd: null, + errorSummary: "HTTP 401: User not found.", + }, + ], + keyRejected: true, + }), + ) + expect(stub.modelCalls).toHaveLength(1) + }) + it.each([401, 402, 403])( "aborts the ladder after exactly one call on a %d", async (statusCode) => { - const stub = makeSdkStub({ - sendResponses: [{ error: makeStatusError(statusCode) }], + const stub = makeClientStubs({ + modelResults: [{ error: makeStatusError(statusCode) }], }) const { client, logger } = makeClient(stub) await expect(client.requestReview(requestParams, logger)).rejects.toThrow( new ReviewRequestError({ - message: `OpenRouter auth/credit error — aborting without fallback: openai/gpt-5-mini: api_error (HTTP ${statusCode}: HTTP ${statusCode})`, + message: `OpenRouter auth/credit error — aborting without fallback: z-ai/glm-5.3-flash: api_error (HTTP ${statusCode}: HTTP ${statusCode})`, attempts: [ { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "api_error", promptTokens: null, completionTokens: null, @@ -1585,13 +1681,13 @@ describe("requestReview", () => { keyRejected: true, }), ) - expect(stub.sendCalls).toHaveLength(1) + expect(stub.modelCalls).toHaveLength(1) }, ) it("throws a per-attempt summary after exhausting a fallback-less ladder", async () => { - const stub = makeSdkStub({ - sendResponses: [{ error: makeStatusError(500) }, { error: makeStatusError(503) }], + const stub = makeClientStubs({ + modelResults: [{ error: makeStatusError(500) }, { error: makeStatusError(503) }], }) const { client, logger } = makeClient(stub) @@ -1600,10 +1696,10 @@ describe("requestReview", () => { ).rejects.toThrow( new ReviewRequestError({ message: - "review request failed after 2 attempt(s): openai/gpt-5-mini: api_error (HTTP 500: HTTP 500); openai/gpt-5-mini: api_error (HTTP 503: HTTP 503)", + "review request failed after 2 attempt(s): z-ai/glm-5.3-flash: api_error (HTTP 500: HTTP 500); z-ai/glm-5.3-flash: api_error (HTTP 503: HTTP 503)", attempts: [ { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "api_error", promptTokens: null, completionTokens: null, @@ -1611,7 +1707,7 @@ describe("requestReview", () => { errorSummary: "HTTP 500: HTTP 500", }, { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "api_error", promptTokens: null, completionTokens: null, @@ -1622,12 +1718,12 @@ describe("requestReview", () => { keyRejected: false, }), ) - expect(stub.sendCalls).toHaveLength(2) + expect(stub.modelCalls).toHaveLength(2) }) it("carries every billed attempt on the error when the ladder is exhausted", async () => { - const stub = makeSdkStub({ - sendResponses: [{ error: makeStatusError(500) }, { error: makeStatusError(503) }], + const stub = makeClientStubs({ + modelResults: [{ error: makeStatusError(500) }, { error: makeStatusError(503) }], }) const { client, logger } = makeClient(stub) @@ -1641,7 +1737,7 @@ describe("requestReview", () => { expect(failure.keyRejected).toBe(false) expect(failure.attempts).toEqual([ { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "api_error", promptTokens: null, completionTokens: null, @@ -1649,7 +1745,7 @@ describe("requestReview", () => { errorSummary: "HTTP 500: HTTP 500", }, { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "api_error", promptTokens: null, completionTokens: null, @@ -1660,8 +1756,8 @@ describe("requestReview", () => { }) it("marks the error as key-rejected on an auth/credit failure", async () => { - const stub = makeSdkStub({ - sendResponses: [{ error: makeStatusError(402) }], + const stub = makeClientStubs({ + modelResults: [{ error: makeStatusError(402) }], }) const { client, logger } = makeClient(stub) @@ -1673,7 +1769,7 @@ describe("requestReview", () => { expect(failure.keyRejected).toBe(true) expect(failure.attempts).toEqual([ { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "api_error", promptTokens: null, completionTokens: null, @@ -1684,12 +1780,12 @@ describe("requestReview", () => { }) it("makes at most four calls across both ladder models", async () => { - const stub = makeSdkStub({ - sendResponses: [ - { value: invalidJsonChatResult }, - { value: invalidJsonChatResult }, - { value: invalidJsonChatResult }, - { value: invalidJsonChatResult }, + const stub = makeClientStubs({ + modelResults: [ + { value: invalidJsonResult }, + { value: invalidJsonResult }, + { value: invalidJsonResult }, + { value: invalidJsonResult }, ], }) const { client, logger } = makeClient(stub) @@ -1697,178 +1793,79 @@ describe("requestReview", () => { await expect(client.requestReview(requestParams, logger)).rejects.toThrow( new ReviewRequestError({ message: - "review request failed after 4 attempt(s): openai/gpt-5-mini: invalid_json (response content is not valid JSON); openai/gpt-5-mini: invalid_json (response content is not valid JSON); anthropic/claude-haiku-4.5: invalid_json (response content is not valid JSON); anthropic/claude-haiku-4.5: invalid_json (response content is not valid JSON)", + "review request failed after 4 attempt(s): z-ai/glm-5.3-flash: invalid_json (response content is not valid JSON); z-ai/glm-5.3-flash: invalid_json (response content is not valid JSON); deepseek/deepseek-v4.1-flash: invalid_json (response content is not valid JSON); deepseek/deepseek-v4.1-flash: invalid_json (response content is not valid JSON)", attempts: [ { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "invalid_json", - promptTokens: 11800, - completionTokens: 42, - costUsd: null, + promptTokens: 54, + completionTokens: 102, + costUsd: 0.000021036, errorSummary: "response content is not valid JSON", }, { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "invalid_json", - promptTokens: 11800, - completionTokens: 42, - costUsd: null, + promptTokens: 54, + completionTokens: 102, + costUsd: 0.000021036, errorSummary: "response content is not valid JSON", }, { - model: "anthropic/claude-haiku-4.5", + model: "deepseek/deepseek-v4.1-flash", outcome: "invalid_json", - promptTokens: 11800, - completionTokens: 42, - costUsd: null, + promptTokens: 54, + completionTokens: 102, + costUsd: 0.000021036, errorSummary: "response content is not valid JSON", }, { - model: "anthropic/claude-haiku-4.5", + model: "deepseek/deepseek-v4.1-flash", outcome: "invalid_json", - promptTokens: 11800, - completionTokens: 42, - costUsd: null, + promptTokens: 54, + completionTokens: 102, + costUsd: 0.000021036, errorSummary: "response content is not valid JSON", }, ], keyRejected: false, }), ) - expect(stub.sendCalls).toHaveLength(4) + expect(stub.modelCalls).toHaveLength(4) }) - it("treats an unrecognized chat response shape as a retryable api_error", async () => { - const stub = makeSdkStub({ - sendResponses: [{ value: { unexpected: true } }, { value: acceptedChatResult }], + it("records null token counts and a null cost when the response carries no usage block", async () => { + const stub = makeClientStubs({ + modelResults: [{ value: makeNoUsageResult() }], }) const { client, logger } = makeClient(stub) const result = await client.requestReview(requestParams, logger) - expect(result.attempts[0]).toEqual({ - model: "openai/gpt-5-mini", - outcome: "api_error", - promptTokens: null, - completionTokens: null, - costUsd: null, - errorSummary: "unexpected chat response shape", - }) - }) - - it("looks up the generation cost when the accepted attempt has no usage cost", async () => { - const stub = makeSdkStub({ - sendResponses: [{ value: makeNoCostChatResult() }], - generationResponses: [{ value: { data: { totalCost: 0.0399 } } }], - }) - const { client, logger } = makeClient(stub) - - const result = await client.requestReview(requestParams, logger) - - expect(stub.generationCalls).toEqual([ - { - id: "gen-no-cost", - options: { - retries: { strategy: "none" }, - signal: expect.any(AbortSignal), - }, - }, - ]) - expect(result.attempts[0]?.costUsd).toBe(0.0399) - }) - - it("degrades to a null cost with a warning when the generation lookup fails", async () => { - const stub = makeSdkStub({ - sendResponses: [{ value: makeNoCostChatResult() }], - generationResponses: [{ error: makeStatusError(500) }], - }) - const { client, logger } = makeClient(stub) - - const result = await client.requestReview(requestParams, logger) - - expect(result.attempts[0]?.costUsd).toBeNull() - expect(logsWithMessage(logger, "generation cost lookup failed")).toEqual([ - { - level: "warn", - message: "generation cost lookup failed", - data: { - operation: "generation cost lookup", - generationId: "gen-no-cost", - error: "HTTP 500", - }, - }, - ]) - }) - - it("degrades to a null cost with a warning when the generation lookup throws synchronously", async () => { - const sdkWithThrowingLookup: OpenRouterLike = { - chat: { send: async () => makeNoCostChatResult() }, - generations: { - getGeneration: () => { - throw new Error("lookup threw before returning a promise") - }, - }, - } - const { client, logger } = makeClient({ sdk: sdkWithThrowingLookup }) - - const result = await client.requestReview(requestParams, logger) - - expect(result.attempts[0]?.costUsd).toBeNull() - expect(logsWithMessage(logger, "generation cost lookup failed")).toEqual([ - { - level: "warn", - message: "generation cost lookup failed", - data: { - operation: "generation cost lookup", - generationId: "gen-no-cost", - error: "lookup threw before returning a promise", - }, - }, - ]) - }) - - it("degrades to a null cost with a warning when the generation response has an unexpected shape", async () => { - const stub = makeSdkStub({ - sendResponses: [{ value: makeNoCostChatResult() }], - generationResponses: [{ value: { data: {} } }], - }) - const { client, logger } = makeClient(stub) - - const result = await client.requestReview(requestParams, logger) - - expect(result.attempts[0]?.costUsd).toBeNull() - expect(logsWithMessage(logger, "unexpected generation response shape")).toEqual([ + expect(result.attempts).toEqual([ { - level: "warn", - message: "unexpected generation response shape", - data: { operation: "generation cost lookup", generationId: "gen-no-cost" }, + model: "z-ai/glm-5.3-flash", + outcome: "accepted", + promptTokens: null, + completionTokens: null, + costUsd: null, + errorSummary: null, }, ]) }) - it("records null token counts when the response omits the usage block", async () => { - const noUsageResult = { - id: "gen-no-usage", - model: "openai/gpt-5-mini", - choices: [{ message: { content: JSON.stringify(acceptedReview) } }], - } - const stub = makeSdkStub({ - sendResponses: [{ value: noUsageResult }], - generationResponses: [{ value: { data: { totalCost: 0.0399 } } }], - }) + it("records a null cost with its token counts when the usage block carries no cost", async () => { + const stub = makeClientStubs({ modelResults: [{ value: makeNoCostResult() }] }) const { client, logger } = makeClient(stub) const result = await client.requestReview(requestParams, logger) - // costUsd from the lookup proves the missing usage block routed the - // accepted attempt through the generation-cost fallback expect(result.attempts).toEqual([ { - model: "openai/gpt-5-mini", + model: "z-ai/glm-5.3-flash", outcome: "accepted", - promptTokens: null, - completionTokens: null, - costUsd: 0.0399, + ...ACCEPTED_USAGE, + costUsd: null, errorSummary: null, }, ]) diff --git a/src/openrouter/client.ts b/src/openrouter/client.ts index 0b951857..bef359a7 100644 --- a/src/openrouter/client.ts +++ b/src/openrouter/client.ts @@ -1,5 +1,4 @@ import { DateTime } from "luxon" -import { z } from "zod" import type { Logger } from "../logger.js" import { reviewResponseJsonSchema, @@ -11,9 +10,10 @@ export type AttemptOutcome = "accepted" | "api_error" | "timeout" | "empty_content" | "invalid_json" | "schema_mismatch" export type ModelAttempt = { - /** The model this attempt requested. The ladder is the primary model, then - * the fallback when one is set. OpenRouter's routed model is reported only - * for the accepted attempt, as StructuredReviewResult.modelUsed. */ + /** The model this attempt requested from the model ladder, which tries the + * primary model and then the fallback when one is set. OpenRouter's routed + * model is reported only for the accepted attempt, as + * StructuredReviewResult.modelUsed. */ model: string outcome: AttemptOutcome promptTokens: number | null @@ -31,46 +31,55 @@ export type StructuredReviewResult = { attempts: ModelAttempt[] } -/** Structural subset of the SDK's ChatRequest — exactly what we send. */ -export type ChatRequestSubset = { +/** Structural subset of the `callModel` input of the Agent SDK + * (`@openrouter/agent`) — exactly what we send, as one request with no tools + * to OpenRouter's Responses API (`/responses`). */ +export type ReviewRequest = { model: string - messages: { role: "system" | "user"; content: string }[] - maxCompletionTokens: number - responseFormat: { - type: "json_schema" - jsonSchema: { + instructions: string + input: string + maxOutputTokens: number + text: { + format: { + type: "json_schema" name: string strict: boolean schema: Record } } - stream: false +} + +/** The fields read from the Responses result the Agent SDK takes from the + * stream's completion event. The SDK's own schema validates the result + * before it resolves, so the client reads it through this structural type + * rather than parsing it again; the SDK types `usage` as both nullable and + * optional. */ +export type ResponsesResult = { + id: string + model: string + usage?: + | { + inputTokens: number + outputTokens: number + cost?: number | null | undefined + } + | null + | undefined } /** - * Structural subset of the OpenRouter SDK — test stubs are plain objects. - * Method syntax keeps the real SDK assignable under strictFunctionTypes; - * responses are `unknown` on purpose: Zod parses across the boundary. + * Structural subset of the Agent SDK client. + * - Test stubs are plain objects; method syntax keeps the real client + * assignable under strictFunctionTypes. + * - `getText` is the SDK's own extraction of the message text. + * - The abort signal goes in the request, where the SDK takes the signal that + * cancels the whole call and aborts its in-flight stream. */ -export type OpenRouterLike = { - chat: { - send( - request: { chatRequest: ChatRequestSubset }, - options?: { - retries?: { strategy: "backoff" | "none" } - signal?: AbortSignal - }, - ): Promise - } - generations: { - getGeneration( - request: { id: string }, - options?: { - retries?: { strategy: "backoff" | "none" } - signal?: AbortSignal - }, - ): Promise - } +export type AgentLike = { + callModel( + request: ReviewRequest & { signal: AbortSignal }, + options?: { retries?: { strategy: "backoff" | "none" } }, + ): { getResponse(): Promise; getText(): Promise } } export type OpenRouterClient = { @@ -88,31 +97,14 @@ export type OpenRouterClient = { ) => Promise } -const chatResultSchema = z.object({ - id: z.string(), - model: z.string(), - choices: z.array(z.object({ message: z.object({ content: z.unknown() }) })), - usage: z - .object({ - promptTokens: z.number(), - completionTokens: z.number(), - cost: z.number().nullable().optional(), - }) - .optional(), -}) - -const generationResponseSchema = z.object({ - data: z.object({ totalCost: z.number() }), -}) - /** Auth and credit statuses. The fallback model shares the key, so these end * the ladder without trying it. */ const KEY_REJECTED_STATUSES = new Set([401, 402, 403]) /** The transient 4xx statuses — HTTP request timeout (408) and rate limit - * (429). attemptOnce also marks 5xx and status-less network errors - * retryable, and a context-overflow 400 retries with a smaller output - * ceiling; any other 4xx is structural and skips the retry. */ + * (429). Any other 4xx is structural and skips the retry, except a + * context-overflow 400, which retries with a smaller output ceiling; + * attemptOnce retries every failure outside the 4xx range. */ const RETRYABLE_4XX_STATUSES = new Set([408, 429]) /** Cap on same-model attempts for failures that settle quickly (HTTP errors, @@ -126,10 +118,12 @@ const MAX_ATTEMPTS_PER_MODEL = 2 * rate-limit burst has a recovery window. */ const RETRY_DELAY_MS = 1_000 -/** The output ceiling, sent as the request's `maxCompletionTokens`, for every +/** The output ceiling, sent as the request's `maxOutputTokens`, for every * attempt except a context-overflow retry. Without one, some providers apply - * a default output limit that cuts a large review off mid-JSON. */ -const MAX_COMPLETION_TOKENS = 128_000 + * a default output limit that cuts a large review off mid-JSON. 128_000 is + * several times what a review usually spends, so in practice only a runaway + * response reaches it. */ +const MAX_OUTPUT_TOKENS = 128_000 /** OpenRouter's 400 when the prompt plus the output ceiling exceeds the routed * endpoint's context window, e.g. "This endpoint's maximum context length is @@ -147,10 +141,12 @@ const CONTEXT_FIT_MARGIN_TOKENS = 8_192 * of thousands of output tokens, reasoning included, so a smaller ceiling * would cut many off mid-JSON and still bill the full prompt. Below it, the * 400 fails like any other structural 4xx. */ -const MIN_FITTED_MAX_COMPLETION_TOKENS = 32_768 +const MIN_FITTED_MAX_OUTPUT_TOKENS = 32_768 -/** OpenRouter SDK errors carry a numeric `statusCode` — duck-typed so stubs - * and future SDK versions need no instanceof on SDK internals. */ +/** HTTP errors from both SDKs carry the response's numeric `statusCode`. + * `@openrouter/agent` throws them from its own bundled copy of + * `@openrouter/sdk`, whose error classes differ from the repo's copy, so the + * check is duck-typed rather than an instanceof. */ const errorStatusCode = (error: unknown): number | undefined => { if (typeof error !== "object" || error === null) return undefined if (!("statusCode" in error) || typeof error.statusCode !== "number") { @@ -162,12 +158,12 @@ const errorStatusCode = (error: unknown): number | undefined => { /** The largest output ceiling that fits the endpoint a context-overflow error * names, or null when the error is something else or no usable smaller * ceiling fits. */ -const fittedMaxCompletionTokens = ({ +const fittedMaxOutputTokens = ({ error, - rejectedMaxCompletionTokens, + rejectedMaxOutputTokens, }: { error: unknown - rejectedMaxCompletionTokens: number + rejectedMaxOutputTokens: number }): number | null => { if (errorStatusCode(error) !== 400 || !(error instanceof Error)) return null @@ -180,11 +176,11 @@ const fittedMaxCompletionTokens = ({ const fittedCeiling = Number(contextLength) - Number(inputTokens) - CONTEXT_FIT_MARGIN_TOKENS // The endpoint leaves too little room for a review to finish its JSON - if (fittedCeiling < MIN_FITTED_MAX_COMPLETION_TOKENS) return null + if (fittedCeiling < MIN_FITTED_MAX_OUTPUT_TOKENS) return null // The estimated input plus the rejected ceiling already fit the window, so // that ceiling did not cause this 400 and lowering it cannot fix it - if (fittedCeiling >= rejectedMaxCompletionTokens) return null + if (fittedCeiling >= rejectedMaxOutputTokens) return null return fittedCeiling } @@ -302,32 +298,29 @@ const parseJsonOrNull = (text: string): { parsed: unknown } | null => { } } -const buildChatRequest = ({ +const buildReviewRequest = ({ systemPrompt, userPrompt, model, - maxCompletionTokens, + maxOutputTokens, }: { systemPrompt: string userPrompt: string model: string - maxCompletionTokens: number -}): ChatRequestSubset => ({ + maxOutputTokens: number +}): ReviewRequest => ({ model, - messages: [ - { role: "system", content: systemPrompt }, - { role: "user", content: userPrompt }, - ], - maxCompletionTokens, - responseFormat: { - type: "json_schema", - jsonSchema: { + instructions: systemPrompt, + input: userPrompt, + maxOutputTokens, + text: { + format: { + type: "json_schema", name: "review_response", strict: true, schema: reviewResponseJsonSchema, }, }, - stream: false, }) type SingleAttempt = @@ -348,7 +341,7 @@ type SingleAttempt = | { kind: "context_overflow" attempt: ModelAttempt - fittedMaxCompletionTokens: number + fittedMaxOutputTokens: number } /** Thrown when the model ladder ends without an accepted review; retains every @@ -356,7 +349,7 @@ type SingleAttempt = export class ReviewRequestError extends Error { readonly attempts: ModelAttempt[] /** An auth or credit status (401/402/403) stopped the ladder. The key fails - * for every model, so the stage runner skips the stages after this one. */ + * for every model, so `runStages` skips the review stages after this one. */ readonly keyRejected: boolean /** The whole review's time budget ran out, as reported by the * `remainingReviewMs` that createOpenRouterClient takes. */ @@ -393,12 +386,12 @@ const summarizeAttempts = (attempts: ModelAttempt[]): string => { /** The client holds no logger. Each requestReview call passes its own, so * concurrent callers can tag their requests' lines with their own props. */ export const createOpenRouterClient = ({ - sdk, + agent, requestTimeoutMs, remainingReviewMs, retryDelayMs = RETRY_DELAY_MS, }: { - sdk: OpenRouterLike + agent: AgentLike /** Per-attempt deadline. When it elapses the attempt records outcome * `timeout` and the ladder moves on — to the next model when one * exists, otherwise a same-model retry while attempts remain — whether @@ -406,7 +399,7 @@ export const createOpenRouterClient = ({ * best-effort. */ requestTimeoutMs: number /** Milliseconds left in the whole review's time budget; 0 or less once - * it has run out. Caps every attempt, cost lookup, and retry delay. */ + * it has run out. Caps every attempt and retry delay. */ remainingReviewMs: () => number retryDelayMs?: number }): OpenRouterClient => { @@ -424,26 +417,43 @@ export const createOpenRouterClient = ({ } } - /** Sends one chat request under the per-attempt deadline. Its only log lines - * are withDeadline's; requestReview logs each attempt's outcome. */ + /** Sends one review request under the per-attempt deadline. Its only log + * lines are withDeadline's; requestReview logs each attempt's outcome. */ const attemptOnce = async ( - chatRequest: ChatRequestSubset, + reviewRequest: ReviewRequest, logger: Logger, ): Promise => { - const { model } = chatRequest + const { model } = reviewRequest // No remaining-time guard here. requestReview calls ensureReviewTimeRemaining // before every attempt, so some review time is always left const callTimeout = getCallTimeout(remainingReviewMs()) + // The Agent SDK starts the request's stream on the first read and takes + // the Responses result from the stream's completion event. + // - A stream that closes without a completion event rejects with "Stream + // ended without completion event", a status-less error that retries + // like a network error. + // - When every model fails that way, the `@openrouter/sdk` copy bundled + // inside `@openrouter/agent` rejected a changed wire field or ignored a + // stream error event; only the raw stream tells those apart. const sendResult = await withDeadline( { - start: (signal) => { - return sdk.chat.send({ chatRequest }, { retries: { strategy: "none" }, signal }) + start: async (signal) => { + const modelResult = agent.callModel( + { ...reviewRequest, signal }, + { retries: { strategy: "none" } }, + ) + const response = await modelResult.getResponse() + + // getText reads the response getResponse already awaited, so it + // sends no second request + const text = await modelResult.getText() + return { response, text } }, timeoutMs: callTimeout.timeoutMs, }, - logger.child({ operation: "chat request", model }), + logger.child({ operation: "review request", model }), ) // Retryable even when the review deadline caused the timeout, because @@ -478,52 +488,36 @@ export const createOpenRouterClient = ({ ? `HTTP ${statusCode}: ${summarizeError(sendResult.error)}` : summarizeError(sendResult.error), } - const fittedCeiling = fittedMaxCompletionTokens({ + const fittedCeiling = fittedMaxOutputTokens({ error: sendResult.error, - rejectedMaxCompletionTokens: chatRequest.maxCompletionTokens, + rejectedMaxOutputTokens: reviewRequest.maxOutputTokens, }) if (fittedCeiling) { - return { kind: "context_overflow", attempt, fittedMaxCompletionTokens: fittedCeiling } + return { kind: "context_overflow", attempt, fittedMaxOutputTokens: fittedCeiling } } - const keyRejected = statusCode ? KEY_REJECTED_STATUSES.has(statusCode) : false + const keyRejected = false - // An unknown status, a 5xx, a 408, or a 429 is transient and retries; - // any other 4xx, such as a 400 bad request or a 401/402/403 auth or - // credit status, is structural and does not - const retryable = !statusCode || statusCode >= 500 || RETRYABLE_4XX_STATUSES.has(statusCode) - - return { kind: "failed", attempt, retryable, keyRejected } - } + // A 4xx other than a 408 or a 429, such as a 400 bad request or a + // 401/402/403 auth or credit status, is structural and does not retry. + // Every other failure retries. That includes the status-200 error the SDK + // throws when a non-streamed body fails its result schema, because a + // second call can come back well-formed + const isStructuralClientError = statusCode + ? statusCode >= 400 && statusCode < 500 && !RETRYABLE_4XX_STATUSES.has(statusCode) + : false - const parsedResult = chatResultSchema.safeParse(sendResult.value) - - // A malformed response body points at a provider or gateway glitch rather - // than the request, so a second call can come back well-formed - if (!parsedResult.success) { - return { - kind: "failed", - attempt: { - model, - outcome: "api_error", - promptTokens: null, - completionTokens: null, - costUsd: null, - errorSummary: "unexpected chat response shape", - }, - retryable: true, - keyRejected: false, - } + return { kind: "failed", attempt, retryable: !isStructuralClientError, keyRejected } } - const chatResult = parsedResult.data + const responsesResult = sendResult.value.response // Failed-validation attempts still carry usage — they were billed too const usage = { - promptTokens: chatResult.usage?.promptTokens ?? null, - completionTokens: chatResult.usage?.completionTokens ?? null, - costUsd: chatResult.usage?.cost ?? null, + promptTokens: responsesResult.usage?.inputTokens ?? null, + completionTokens: responsesResult.usage?.outputTokens ?? null, + costUsd: responsesResult.usage?.cost ?? null, } const failedValidation = (outcome: AttemptOutcome, errorSummary: string): SingleAttempt => ({ kind: "failed", @@ -532,9 +526,11 @@ export const createOpenRouterClient = ({ keyRejected: false, }) - const content = chatResult.choices[0]?.message.content + // The SDK extracts "" when the output holds no message item, such as a + // reasoning-only response + const content = sendResult.value.text - if (typeof content !== "string" || content === "") { + if (content === "") { return failedValidation("empty_content", "response had no text content") } @@ -559,63 +555,12 @@ export const createOpenRouterClient = ({ return { kind: "accepted", review: parsedReview.data, - routedModel: chatResult.model, - generationId: chatResult.id, - attempt: { model, outcome: "accepted", ...usage, errorSummary: null }, - } - } - - /** A cost lookup failure must never fail a completed review, so every - * failure degrades to a null cost. */ - const lookupGenerationCost = async ( - generationId: string, - logger: Logger, - ): Promise => { - const remainingMs = remainingReviewMs() - - // Every lookup line names the generation, so a failure ties back to it - const lookupLogger = logger.child({ operation: "generation cost lookup", generationId }) - - if (remainingMs <= 0) { - lookupLogger.debug("generation cost lookup skipped", { reason: "review deadline exceeded" }) - return null - } - - const callTimeout = getCallTimeout(remainingMs) - - const lookup = await withDeadline( - { - start: (signal) => { - return sdk.generations.getGeneration( - { id: generationId }, - { retries: { strategy: "none" }, signal }, - ) - }, - timeoutMs: callTimeout.timeoutMs, - }, - lookupLogger, - ) + routedModel: responsesResult.model, - if (lookup.status === "timed_out") { - lookupLogger.warn("generation cost lookup failed", { error: callTimeout.timeoutSummary }) - return null - } - if (lookup.status === "rejected") { - lookupLogger.warn("generation cost lookup failed", { - error: summarizeError(lookup.error), - }) - return null - } - - const parsed = generationResponseSchema.safeParse(lookup.value) - - if (!parsed.success) { - lookupLogger.warn("unexpected generation response shape") - return null + // A Responses result's `id` is its generation id (`gen-…`) + generationId: responsesResult.id, + attempt: { model, outcome: "accepted", ...usage, errorSummary: null }, } - - // parsed.data = Zod safeParse result; .data = API envelope's data property - return parsed.data.data.totalCost } const requestReview: OpenRouterClient["requestReview"] = async ( @@ -634,27 +579,31 @@ export const createOpenRouterClient = ({ }) } + // Each model's attempts end in one of four ways: + // - An accepted review returns. + // - A rejected key or a spent review deadline throws, ending the ladder. + // - A timeout with a next model left, or a structural failure, breaks to + // the next model. + // - A failed last attempt ends the while loop and moves to the next model. + // When the last model's attempts end without a review, the throw after the + // loops reports every attempt for (const [ladderIndex, ladderModel] of modelLadder.entries()) { const nextLadderModel = modelLadder[ladderIndex + 1] // Both are reassigned across this model's attempts. Each model starts at // the full ceiling because its own endpoints decide what fits, so a // context overflow lowers the ceiling only for the same model's retry - let maxCompletionTokens = MAX_COMPLETION_TOKENS + let maxOutputTokens = MAX_OUTPUT_TOKENS let modelAttemptNumber = 1 while (modelAttemptNumber <= MAX_ATTEMPTS_PER_MODEL) { ensureReviewTimeRemaining() const attemptResult = await attemptOnce( - buildChatRequest({ systemPrompt, userPrompt, model: ladderModel, maxCompletionTokens }), + buildReviewRequest({ systemPrompt, userPrompt, model: ladderModel, maxOutputTokens }), logger, ) if (attemptResult.kind === "accepted") { - // The generation lookup runs only when the response's usage omits the cost - const costUsd = - attemptResult.attempt.costUsd ?? - (await lookupGenerationCost(attemptResult.generationId, logger)) - attempts.push({ ...attemptResult.attempt, costUsd }) + attempts.push(attemptResult.attempt) logger.info("review response accepted", { model: ladderModel, routedModel: attemptResult.routedModel, @@ -713,10 +662,10 @@ export const createOpenRouterClient = ({ // fitted ceiling, so the loop ends and any next model starts at the // full ceiling if (retryRemains && attemptResult.kind === "context_overflow") { - maxCompletionTokens = attemptResult.fittedMaxCompletionTokens + maxOutputTokens = attemptResult.fittedMaxOutputTokens logger.info("retrying with an output ceiling that fits the endpoint's context window", { model: ladderModel, - maxCompletionTokens, + maxOutputTokens, }) } diff --git a/src/orchestrate.ts b/src/orchestrate.ts index e165fb4b..5fbd574d 100644 --- a/src/orchestrate.ts +++ b/src/orchestrate.ts @@ -1054,8 +1054,7 @@ const runReviewPipeline = async ( const completedPhases = phaseOutcomes.filter((outcome) => outcome.status === "completed") const phases = phaseOutcomes.map(describePhaseOutcome) - // Only a phase that failed on the deadline lost coverage. A completed phase - // whose cost lookup the deadline skipped still delivered its findings. + // Only a phase that failed on the deadline lost coverage const coverageLostToReviewDeadline = phaseOutcomes.some( (outcome) => outcome.status === "failed" && outcome.deadlineExceeded, )