From fa669cca54bdfa1442a133b4d6fdd97eb1f80d76 Mon Sep 17 00:00:00 2001 From: Homen Shum Date: Wed, 14 Oct 2026 11:52:00 -0700 Subject: [PATCH 1/8] chore(models): refresh stale model defaults (NodeBenchAI) - PR 1 of 3, request shapes and fail-closed resolver No wire-id moves except the free lane registration (Codex v3 N2). Makes the later id moves (PR 2) testable and keeps an unknown model id from ever becoming a paid default. - Anthropic request shapes: delete temperature (events.ts x3, modelRouter, benchmark judge, mcp-local llmTools/researchWritingTools), max_tokens 700/260/300 -> 2048, extendedThinking budget_tokens -> adaptive + output_config.effort, text read by block type in the 16 generateWithProvider copies, explicit ANTHROPIC_WIRE_IDS table instead of dotted aliases on the wire. - Gemini: thinkingBudget:0 -> thinkingLevel "low" (triSearch, judge script). - Resolver fails closed (A.0.10): unknown free-looking -> FREE_LANE_MODEL, unknown otherwise -> UnknownModelError unless allowPaidFallback === true; pipeline launch boundary asserts before admission; parseModelKey no longer forwards arbitrary ids. - FREE_LANE_MODEL "nemotron-3-super-free" registered (approved alias, spec, UI row, legacy alias for the wire id, zero-price row); web mirror gets a drift test. - Eval harness: runner --tier/--seeds, real Messages call in agentInvoker, run-comprehensive-eval aborts on unknown --models and records resolvedModelId. - Nine request-shape / boundary tests, added to the CI runtime-smoke list. Spec: model-defaults SPEC Part A s.A.2 (NodeBenchAI) Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/ci.yml | 9 + .../shared/llm/approvedModels.drift.test.ts | 52 +++++ apps/web/src/shared/llm/approvedModels.ts | 30 ++- .../shared/llm/pipelineModelRoutes.test.ts | 6 +- .../web/src/shared/llm/pipelineModelRoutes.ts | 4 +- .../convex/domains/agents/fastAgentChat.ts | 4 +- .../domains/agents/mcp_tools/models/index.ts | 7 + .../agents/mcp_tools/models/migration.ts | 12 +- .../mcp_tools/models/modelResolver.test.ts | 138 +++++++++++- .../agents/mcp_tools/models/modelResolver.ts | 179 +++++++++++++--- backend/convex/domains/ai/ai.ts | 4 +- .../convex/domains/ai/models/modelRouter.ts | 19 +- backend/convex/domains/ai/morningDigest.ts | 4 +- .../domains/evaluation/personaEpisodeEval.ts | 6 +- .../domains/knowledge/entityInsights.ts | 3 +- backend/convex/domains/pipelines/piRuntime.ts | 27 +-- .../pipelines/pipelineModelBoundary.test.ts | 102 +++++++++ .../domains/pipelines/pipelineWorkflow.ts | 5 + .../domains/product/extendedThinking.ts | 19 +- .../research/dailyBriefPersonalOverlay.ts | 3 +- .../domains/research/dailyBriefWorker.ts | 3 +- backend/convex/domains/research/dealFlow.ts | 3 +- .../domains/research/modelComparison.ts | 3 +- .../convex/domains/research/paperDetails.ts | 3 +- backend/convex/domains/research/repoScout.ts | 3 +- .../convex/domains/research/stackImpact.ts | 3 +- .../domains/research/strategyMetrics.ts | 3 +- .../convex/domains/search/fusion/benchmark.ts | 5 +- .../convex/domains/social/postDedupAction.ts | 5 +- .../verification/claimVerificationAction.ts | 4 +- backend/convex/events.requestShape.test.ts | 113 ++++++++++ backend/convex/events.ts | 22 +- backend/convex/tags_actions.ts | 4 +- packages/mcp-local/src/tools/llmTools.ts | 2 +- .../src/tools/researchWritingTools.ts | 2 +- production-agent.json | 2 +- .../__tests__/comprehensiveEvalModels.test.ts | 42 ++++ scripts/__tests__/toolGenRequestShape.test.ts | 67 ++++++ scripts/eval-harness/agentInvoker.ts | 136 +++++++++--- scripts/eval-harness/evalHarness.test.ts | 189 +++++++++++++++++ scripts/eval-harness/runner.ts | 198 ++++++++++-------- scripts/eval-harness/types.ts | 5 +- scripts/run-comprehensive-eval.ts | 57 ++++- scripts/tool-gen/analyze-gap.mjs | 43 ++-- scripts/tool-gen/generate-tool.mjs | 6 +- scripts/ui/judgeScratchnodeChatVideo.mjs | 2 +- shared/llm/anthropicWireIds.test.ts | 123 +++++++++++ shared/llm/modelCatalog.ts | 70 ++++++- shared/search/triSearch.test.ts | 24 +++ shared/search/triSearch.ts | 12 +- .../pipeline/extendedThinkingRunner.test.ts | 42 ++++ .../node/pipeline/extendedThinkingRunner.ts | 41 ++++ 52 files changed, 1618 insertions(+), 252 deletions(-) create mode 100644 apps/web/src/shared/llm/approvedModels.drift.test.ts create mode 100644 backend/convex/domains/pipelines/pipelineModelBoundary.test.ts create mode 100644 backend/convex/events.requestShape.test.ts create mode 100644 scripts/__tests__/comprehensiveEvalModels.test.ts create mode 100644 scripts/__tests__/toolGenRequestShape.test.ts create mode 100644 scripts/eval-harness/evalHarness.test.ts create mode 100644 shared/llm/anthropicWireIds.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d056dcaa5..43d87559a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,6 +100,15 @@ jobs: backend/convex/domains/mcp/mcpSourcingDraft.integration.test.ts backend/convex/domains/mcp/mcpExecutionTraceEndpoints.integration.test.ts backend/convex/domains/agents/mcp_tools/models/modelResolver.test.ts + backend/convex/domains/pipelines/pipelineModelBoundary.test.ts + backend/convex/events.requestShape.test.ts + shared/llm/anthropicWireIds.test.ts + shared/search/triSearch.test.ts + workers/node/pipeline/extendedThinkingRunner.test.ts + scripts/eval-harness/evalHarness.test.ts + scripts/__tests__/toolGenRequestShape.test.ts + scripts/__tests__/comprehensiveEvalModels.test.ts + apps/web/src/shared/llm/approvedModels.drift.test.ts apps/web/src/features/agents/components/FastAgentPanel/__tests__/RunState.test.tsx backend/convex/domains/agents/autonomy/autonomy.integration.test.ts backend/convex/domains/agents/autonomy/autonomyTasteBench.integration.test.ts diff --git a/apps/web/src/shared/llm/approvedModels.drift.test.ts b/apps/web/src/shared/llm/approvedModels.drift.test.ts new file mode 100644 index 000000000..a9ca129ad --- /dev/null +++ b/apps/web/src/shared/llm/approvedModels.drift.test.ts @@ -0,0 +1,52 @@ +/** + * Drift guard: apps/web/src/shared/llm/approvedModels.ts is a hand-kept mirror of + * backend/convex/domains/agents/mcp_tools/models/modelResolver.ts (Convex cannot import + * from the web tree). The two had already drifted - MODEL_PRIORITY_ORDER had 16 entries + * against the resolver's 24 - so the selector could offer or order models the backend + * would route differently (SPEC-v9 A.2.10 PR 1). Failing here means one side was edited + * without the other: regenerate the web side from the resolver, never loosen this test. + */ + +import { describe, expect, it } from "vitest"; + +import * as web from "./approvedModels"; +import * as backend from "../../../../../backend/convex/domains/agents/mcp_tools/models/modelResolver"; + +describe("web approvedModels mirrors modelResolver", () => { + it("lists the same approved ids in the same order", () => { + expect([...web.APPROVED_MODELS]).toEqual([...backend.APPROVED_MODELS]); + }); + + it("has a UI row for every approved id, agreeing on provider and free status", () => { + expect(Object.keys(web.MODEL_UI_INFO).sort()).toEqual([...backend.APPROVED_MODELS].sort()); + for (const id of backend.APPROVED_MODELS) { + const ui = web.MODEL_UI_INFO[id]; + expect(ui.id, id).toBe(id); + expect(ui.provider, `${id} provider`).toBe(backend.MODEL_SPECS[id].provider); + expect(Boolean(ui.isFree), `${id} isFree`).toBe(backend.isFreeModel(id)); + } + }); + + it("agrees on the default, fallback and free-lane constants", () => { + expect(web.DEFAULT_MODEL).toBe(backend.DEFAULT_MODEL); + expect(web.FALLBACK_MODEL).toBe(backend.FALLBACK_MODEL); + expect(web.MODEL_UI_INFO[backend.FREE_LANE_MODEL].isFree).toBe(true); + }); + + it("agrees on the priority order and the advisor/executor/background lanes", () => { + expect(web.MODEL_PRIORITY_ORDER).toEqual(backend.MODEL_PRIORITY_ORDER); + expect(web.NODEBENCH_ADVISOR_MODEL).toBe(backend.NODEBENCH_ADVISOR_MODEL); + expect(web.NODEBENCH_EXECUTOR_MODELS).toEqual(backend.NODEBENCH_EXECUTOR_MODELS); + expect(web.NODEBENCH_BACKGROUND_MODELS).toEqual(backend.NODEBENCH_BACKGROUND_MODELS); + }); + + it("normalizes every approved id to the same lane model", () => { + for (const profile of ["advisor", "executor", "background"] as const) { + for (const id of backend.APPROVED_MODELS) { + expect(web.normalizeNodeBenchRuntimeModel(id, profile), `${profile}/${id}`).toBe( + backend.normalizeNodeBenchRuntimeModel(id, profile), + ); + } + } + }); +}); diff --git a/apps/web/src/shared/llm/approvedModels.ts b/apps/web/src/shared/llm/approvedModels.ts index a978a6cfb..fd7361788 100644 --- a/apps/web/src/shared/llm/approvedModels.ts +++ b/apps/web/src/shared/llm/approvedModels.ts @@ -32,6 +32,7 @@ export const APPROVED_MODELS = [ "qwen3-235b", "minimax-m2.7", "mistral-large", + "nemotron-3-super-free", "laguna-s-2.1-free", "laguna-xs-2.1-free", "qwen3-coder-free", @@ -293,6 +294,16 @@ export const MODEL_UI_INFO: Record = { contextWindow: "131K", icon: "OR", }, + "nemotron-3-super-free": { + id: "nemotron-3-super-free", + name: "Nemotron 3 Super (Free)", + provider: "openrouter", + description: "Verified free lane with tools and structured outputs", + tier: "powerful", + contextWindow: "262K", + icon: "OR", + isFree: true, + }, "laguna-s-2.1-free": { id: "laguna-s-2.1-free", name: "Laguna S 2.1 (Free)", @@ -478,23 +489,32 @@ export const MODEL_UI_INFO: Record = { export const DEFAULT_MODEL: ApprovedModel = "kimi-k2.6"; export const FALLBACK_MODEL: ApprovedModel = "gpt-5.4-mini"; +// Mirrors modelResolver.ts; approvedModels.drift.test.ts fails when they diverge. export const MODEL_PRIORITY_ORDER: ApprovedModel[] = [ "kimi-k2.6", - "minimax-m2.7", "gemini-3.1-flash-lite-preview", - "gpt-5.4-mini", "gemini-3-flash-preview", "gemini-3.1-pro-preview", + "minimax-m2.7", + "gpt-5.4-mini", "gpt-5.4", "claude-sonnet-4.6", "claude-opus-4.7", "claude-haiku-4.5", "gpt-5.4-nano", + "laguna-s-2.1-free", + "laguna-xs-2.1-free", "qwen3-coder-free", "step-3.5-flash-free", "gpt-oss-120b-free", "trinity-large-free", "nemotron-3-nano-free", + "qwen3-next-free", + "llama-3.3-70b-free", + "glm-4.5-air-free", + "deepseek-r1-free", + "deepseek-chimera-free", + "venice-dolphin-free", ]; export type NodeBenchRuntimeProfile = "advisor" | "executor" | "background"; @@ -503,16 +523,16 @@ export const NODEBENCH_ADVISOR_MODEL: ApprovedModel = "kimi-k2.6"; export const NODEBENCH_EXECUTOR_MODELS: ApprovedModel[] = [ "gemini-3.1-flash-lite-preview", - "gpt-5.4-mini", - "minimax-m2.7", "gemini-3-flash-preview", + "minimax-m2.7", + "gpt-5.4-mini", "kimi-k2.6", ]; export const NODEBENCH_BACKGROUND_MODELS: ApprovedModel[] = [ "gemini-3.1-pro-preview", - "kimi-k2.6", "gpt-5.4", + "kimi-k2.6", "gemini-3-flash-preview", ]; diff --git a/apps/web/src/shared/llm/pipelineModelRoutes.test.ts b/apps/web/src/shared/llm/pipelineModelRoutes.test.ts index 37e9fce69..d541b1990 100644 --- a/apps/web/src/shared/llm/pipelineModelRoutes.test.ts +++ b/apps/web/src/shared/llm/pipelineModelRoutes.test.ts @@ -25,10 +25,12 @@ describe("pipeline model routes", () => { expect(option).toMatchObject({ label: "Auto free", provider: "openrouter", - resolvedModelId: "laguna-s-2.1-free", + // Was laguna-s-2.1-free: that lane expires on OpenRouter 2026-10-31, so auto-free + // resolves to FREE_LANE_MODEL (SPEC-v9 A.2.10 PR 1; NodeVideo's measured pick). + resolvedModelId: "nemotron-3-super-free", isFree: true, }); - expect(getPipelineModelRuntimeId(PIPELINE_MODEL_AUTO_FREE)).toBe("laguna-s-2.1-free"); + expect(getPipelineModelRuntimeId(PIPELINE_MODEL_AUTO_FREE)).toBe("nemotron-3-super-free"); }); it("keeps direct models and route models in one selector bank", () => { diff --git a/apps/web/src/shared/llm/pipelineModelRoutes.ts b/apps/web/src/shared/llm/pipelineModelRoutes.ts index ee73a34ae..2baf6f667 100644 --- a/apps/web/src/shared/llm/pipelineModelRoutes.ts +++ b/apps/web/src/shared/llm/pipelineModelRoutes.ts @@ -47,7 +47,9 @@ const AUTO_ROUTE_OPTIONS: PipelineModelOption[] = [ shortLabel: "Auto free", detail: "Free-capable OpenRouter route for budget-sensitive runs.", provider: "openrouter", - resolvedModelId: "laguna-s-2.1-free", + // Must equal routeAliasToModel("auto-free") in modelResolver.ts (FREE_LANE_MODEL): + // laguna-s-2.1:free expires on OpenRouter 2026-10-31 (SPEC-v9 A.1). + resolvedModelId: "nemotron-3-super-free", isRoute: true, isFree: true, }, diff --git a/backend/convex/domains/agents/fastAgentChat.ts b/backend/convex/domains/agents/fastAgentChat.ts index 325a75c35..abe45cbbc 100644 --- a/backend/convex/domains/agents/fastAgentChat.ts +++ b/backend/convex/domains/agents/fastAgentChat.ts @@ -12,7 +12,7 @@ import OpenAI from "openai"; import Anthropic from "@anthropic-ai/sdk"; import { createGoogleGenerativeAI } from "@ai-sdk/google"; import { generateText } from "ai"; -import { getLlmModel, resolveModelAlias, getModelWithFailover } from "../../../../shared/llm/modelCatalog"; +import { getLlmModel, resolveModelAlias, getModelWithFailover, anthropicText } from "../../../../shared/llm/modelCatalog"; /** * Modern fast agent chat action @@ -326,7 +326,7 @@ async function handleChatResponse( system: systemPrompt, messages: [{ role: "user", content: message }], }); - response = completion.content[0]?.type === "text" ? completion.content[0].text : "I apologize, but I couldn't generate a response."; + response = anthropicText(completion.content) || "I apologize, but I couldn't generate a response."; } else if (provider === "gemini") { const google = createGoogleGenerativeAI({ apiKey: process.env.GEMINI_API_KEY }); const result = await generateText({ diff --git a/backend/convex/domains/agents/mcp_tools/models/index.ts b/backend/convex/domains/agents/mcp_tools/models/index.ts index 89e60dc0a..61d461c5f 100644 --- a/backend/convex/domains/agents/mcp_tools/models/index.ts +++ b/backend/convex/domains/agents/mcp_tools/models/index.ts @@ -23,11 +23,18 @@ export { LEGACY_ALIASES, DEFAULT_MODEL, FALLBACK_MODEL, + FREE_LANE_MODEL, MODEL_PRIORITY_ORDER, NODEBENCH_ADVISOR_MODEL, NODEBENCH_EXECUTOR_MODELS, NODEBENCH_BACKGROUND_MODELS, + // Fail-closed resolution (SPEC-v9 A.0.10) + UnknownModelError, + type UnknownModelOptions, + assertKnownPipelineModel, + getPaidFallbackCount, + // Core resolver functions getLanguageModel, getLanguageModelOrThrow, diff --git a/backend/convex/domains/agents/mcp_tools/models/migration.ts b/backend/convex/domains/agents/mcp_tools/models/migration.ts index 63217fbd8..99d94399d 100644 --- a/backend/convex/domains/agents/mcp_tools/models/migration.ts +++ b/backend/convex/domains/agents/mcp_tools/models/migration.ts @@ -10,7 +10,7 @@ import { action, internalAction, query, internalMutation, internalQuery } from "../../../../_generated/server"; import { internal } from "../../../../_generated/api"; import { v } from "convex/values"; -import { normalizeModelInput, isApprovedModel } from "./modelResolver"; +import { resolveModelAlias, isApprovedModel } from "./modelResolver"; import type { Id } from "../../../../_generated/dataModel"; // ═══════════════════════════════════════════════════════════════════════════ @@ -40,8 +40,10 @@ export const getLegacyThreads = internalQuery({ const legacyThreads: Array<{ _id: Id<"chatThreadsStream">; model: string; normalized: string }> = []; for (const thread of threads) { if (thread.model && !isApprovedModel(thread.model)) { - const normalized = normalizeModelInput(thread.model); - if (normalized !== thread.model) { + // Unknown persisted strings are left as they are: normalizeModelInput now + // throws for them, and rows are never rewritten to a paid default. + const normalized = resolveModelAlias(thread.model); + if (normalized && normalized !== thread.model) { legacyThreads.push({ _id: thread._id, model: thread.model, normalized }); } } @@ -59,8 +61,8 @@ export const getLegacyMessages = internalQuery({ const legacyMessages: Array<{ _id: Id<"chatMessagesStream">; model: string; normalized: string }> = []; for (const msg of messages) { if (msg.model && !isApprovedModel(msg.model)) { - const normalized = normalizeModelInput(msg.model); - if (normalized !== msg.model) { + const normalized = resolveModelAlias(msg.model); + if (normalized && normalized !== msg.model) { legacyMessages.push({ _id: msg._id, model: msg.model, normalized }); } } diff --git a/backend/convex/domains/agents/mcp_tools/models/modelResolver.test.ts b/backend/convex/domains/agents/mcp_tools/models/modelResolver.test.ts index b136d4343..143a3e173 100644 --- a/backend/convex/domains/agents/mcp_tools/models/modelResolver.test.ts +++ b/backend/convex/domains/agents/mcp_tools/models/modelResolver.test.ts @@ -1,6 +1,18 @@ -import { describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { + APPROVED_MODELS, + DEFAULT_MODEL, + FREE_LANE_MODEL, + MODEL_SPECS, + UnknownModelError, + assertKnownPipelineModel, + getLanguageModel, + getLanguageModelSafe, + getPaidFallbackCount, + isApprovedModel, + isFreeModel, + normalizeModelInput, resolveModelAlias, resolvePipelineModelSelection, } from "./modelResolver"; @@ -13,17 +25,17 @@ describe("pipeline model route resolution", () => { routeTier: "balanced", isFreeRoute: false, }); + // Was laguna-s-2.1-free: that lane expires on OpenRouter 2026-10-31, so auto-free + // resolves to FREE_LANE_MODEL (SPEC-v9 A.2.10 PR 1; NodeVideo's measured pick). expect(resolvePipelineModelSelection("nodebench:auto-free")).toMatchObject({ - resolvedModelId: "laguna-s-2.1-free", + resolvedModelId: FREE_LANE_MODEL, routeTier: "free", isFreeRoute: true, }); }); it("accepts Kilo-style route aliases and old provider-prefixed values", () => { - expect(resolvePipelineModelSelection("kilo-auto/free").resolvedModelId).toBe( - "laguna-s-2.1-free", - ); + expect(resolvePipelineModelSelection("kilo-auto/free").resolvedModelId).toBe(FREE_LANE_MODEL); expect(resolvePipelineModelSelection("kilo-auto/public-research")).toMatchObject({ resolvedModelId: "glm-4.7-flash", routeTier: "balanced", @@ -42,3 +54,119 @@ describe("pipeline model route resolution", () => { expect(resolveModelAlias("poolside/laguna-s-2.1:free")).toBe("laguna-s-2.1-free"); }); }); + +// --------------------------------------------------------------------------- +// Free lane registration and fail-closed resolution (SPEC-v9 A.0.10, A.2.10 PR 1) +// --------------------------------------------------------------------------- + +const FREE_LANE_WIRE_ID = "nvidia/nemotron-3-super-120b-a12b:free"; + +describe("free lane registration (Codex v3 N2)", () => { + beforeEach(() => { + vi.stubEnv("OPENROUTER_API_KEY", "test-key-not-real"); + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "warn").mockImplementation(() => {}); + }); + afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + }); + + it("registers FREE_LANE_MODEL as an approved, zero-priced alias", () => { + expect(isApprovedModel(FREE_LANE_MODEL)).toBe(true); + expect(APPROVED_MODELS).toContain(FREE_LANE_MODEL); + expect(isFreeModel(FREE_LANE_MODEL)).toBe(true); + expect(MODEL_SPECS[FREE_LANE_MODEL]).toMatchObject({ + provider: "openrouter", + sdkId: FREE_LANE_WIRE_ID, + pricing: { inputPerMillion: 0, outputPerMillion: 0 }, + }); + }); + + it("resolves the OpenRouter wire id to the typed alias", () => { + expect(resolveModelAlias(FREE_LANE_WIRE_ID)).toBe(FREE_LANE_MODEL); + }); + + it("builds an OpenRouter model whose modelId is the wire string", () => { + const model = getLanguageModel(FREE_LANE_MODEL) as { modelId: string }; + expect(model.modelId).toBe(FREE_LANE_WIRE_ID); + }); +}); + +describe("unknown model ids fail closed", () => { + beforeEach(() => { + vi.stubEnv("OPENROUTER_API_KEY", "test-key-not-real"); + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "warn").mockImplementation(() => {}); + }); + afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + }); + + it("sends an unknown free-looking alias to the free lane, never a paid id", () => { + for (const input of ["retired-model-free", "someone/new-model:free", "auto-free", "openrouter/free", "kilo-auto/free"]) { + const model = getLanguageModelSafe(input) as { modelId: string }; + expect(model.modelId).toBe(FREE_LANE_WIRE_ID); + expect(normalizeModelInput(input)).toBe(FREE_LANE_MODEL); + } + }); + + it("throws for an unknown paid alias instead of using DEFAULT_MODEL", () => { + expect(() => getLanguageModelSafe("gpt-9-ultra")).toThrow(UnknownModelError); + expect(() => normalizeModelInput("claude-opus-9")).toThrow(UnknownModelError); + // 'free' only counts as a suffix; it must not be a substring loophole + expect(() => getLanguageModelSafe("freeloader-pro")).toThrow(UnknownModelError); + }); + + it("keeps empty input on the default and known aliases on themselves", () => { + expect(normalizeModelInput(undefined)).toBe(DEFAULT_MODEL); + expect(normalizeModelInput("")).toBe(DEFAULT_MODEL); + expect(normalizeModelInput("gpt-4o-mini")).toBe("gpt-5.4-mini"); + }); + + it("allowPaidFallback falls back to DEFAULT_MODEL and is counted", () => { + const before = getPaidFallbackCount(); + const model = getLanguageModelSafe("gpt-9-ultra", { allowPaidFallback: true }) as { modelId: string }; + expect(model.modelId).toBe(MODEL_SPECS[DEFAULT_MODEL].sdkId); + expect(getPaidFallbackCount()).toBe(before + 1); + expect(normalizeModelInput("gpt-9-ultra", { allowPaidFallback: true })).toBe(DEFAULT_MODEL); + expect(getPaidFallbackCount()).toBe(before + 2); + // a known alias does not touch the counter + getLanguageModelSafe("gpt-5.4", { allowPaidFallback: true }); + expect(getPaidFallbackCount()).toBe(before + 2); + }); + + it("resolves wikiDreamingGraph's provider-prefixed strings instead of rejecting them", () => { + // They used to be unknown and silently ran on DEFAULT_MODEL; the strings are also persisted. + expect(resolveModelAlias("google:models/gemini-3-flash-preview")).toBe("gemini-3-flash-preview"); + expect(resolveModelAlias("google:models/gemini-3.1-pro-preview")).toBe("gemini-3.1-pro-preview"); + }); + + it("allowPaidFallback must be exactly true", () => { + expect(() => getLanguageModelSafe("gpt-9-ultra", { allowPaidFallback: undefined })).toThrow(UnknownModelError); + expect(() => getLanguageModelSafe("gpt-9-ultra", {} as never)).toThrow(UnknownModelError); + }); + + it("resolvePipelineModelSelection throws for unknown ids and keeps free ids free", () => { + expect(() => resolvePipelineModelSelection("not-a-model")).toThrow(UnknownModelError); + expect(() => resolvePipelineModelSelection("openai:gpt-9")).toThrow(UnknownModelError); + expect(resolvePipelineModelSelection("x:free")).toMatchObject({ + resolvedModelId: FREE_LANE_MODEL, + isFreeRoute: true, + routeTier: "free", + usedDefault: false, + }); + // usedDefault is true only for empty input + expect(resolvePipelineModelSelection("")).toMatchObject({ resolvedModelId: DEFAULT_MODEL, usedDefault: true }); + expect(resolvePipelineModelSelection(undefined).usedDefault).toBe(true); + expect(resolvePipelineModelSelection("gpt-5.4").usedDefault).toBe(false); + }); + + it("assertKnownPipelineModel applies the same rules at the launch boundary", () => { + expect(() => assertKnownPipelineModel("not-a-model")).toThrow(UnknownModelError); + expect(() => assertKnownPipelineModel(undefined)).not.toThrow(); + expect(() => assertKnownPipelineModel("nodebench:auto-balanced")).not.toThrow(); + expect(() => assertKnownPipelineModel("x:free")).not.toThrow(); + }); +}); diff --git a/backend/convex/domains/agents/mcp_tools/models/modelResolver.ts b/backend/convex/domains/agents/mcp_tools/models/modelResolver.ts index 367f895c2..14f5c370a 100644 --- a/backend/convex/domains/agents/mcp_tools/models/modelResolver.ts +++ b/backend/convex/domains/agents/mcp_tools/models/modelResolver.ts @@ -109,6 +109,7 @@ export const APPROVED_MODELS = [ "minimax-m2.7", // MiniMax M2.7 - agentic executor lane $0.30/$1.20 "mistral-large", // Mistral Large 2411 - function calling $2/M // OpenRouter free-tier models (availability is re-verified before routing changes) + "nemotron-3-super-free", // NVIDIA Nemotron 3 Super 120B A12B - 262K, tools + structured outputs; FREE_LANE_MODEL "laguna-s-2.1-free", // Poolside Laguna S 2.1 - 262K, current free judge lane "laguna-xs-2.1-free", // Poolside Laguna XS 2.1 - 262K, free fallback judge lane "qwen3-coder-free", // Qwen3 Coder 480B MoE (A3.5B) - 262K, agentic coding @@ -139,6 +140,38 @@ export const DEFAULT_MODEL: ApprovedModel = "kimi-k2.6"; // Fallback model when the primary lane is unavailable or over budget export const FALLBACK_MODEL: ApprovedModel = "gpt-5.4-mini"; +/** + * The one verified free lane. This is the typed resolver boundary (an approved + * alias): code that hands a value to the resolver writes this constant. The + * provider wire string is read from `MODEL_SPECS[FREE_LANE_MODEL].sdkId` and is + * never typed a second time. Evidence: NodeVideo's measured free-model benchmark + * (config/openrouter-free-routing.json), the only measured free pick in the portfolio. + * SPEC-v9 A.2.10 PR 1 (Codex v3 N2). + */ +export const FREE_LANE_MODEL: ApprovedModel = "nemotron-3-super-free"; + +/** + * Thrown when an id is not an approved model, not a legacy alias, and not + * free-looking. Unknown ids never become a paid default (SPEC-v9 A.0.10). + */ +export class UnknownModelError extends Error { + readonly input: string; + constructor(input: string) { + super(`Unknown model "${input}". Allowed models: ${APPROVED_MODELS.join(", ")}`); + this.name = "UnknownModelError"; + this.input = input; + } +} + +/** + * Opt-in for the single kind of caller allowed to turn an unknown id into + * DEFAULT_MODEL: one that, in the same function, awaits a financial reservation + * for the resolved model (`reserveLlmRequestInternal`) before dispatch. + */ +export interface UnknownModelOptions { + allowPaidFallback?: boolean; +} + // Model priority order for production-first routing export const MODEL_PRIORITY_ORDER: ApprovedModel[] = [ // PRIMARY SHIPPING LANES @@ -257,6 +290,15 @@ function stripProviderPrefix(input: string): string { return match ? match[2].trim() : input; } +/** + * An unknown id is "free-looking" when it carries the OpenRouter free suffix + * (`-free` / `:free`) or is one of the free route aliases. + */ +function isFreeLookingModelInput(input: string): boolean { + const normalized = input.trim().toLowerCase(); + return /(-|:)free$/.test(normalized) || routeAliasToModel(normalized)?.routeTier === "free"; +} + function routeAliasToModel(input: string): { routeTier: NodeBenchModelRouteTier; modelId: ApprovedModel; @@ -324,12 +366,37 @@ export function resolvePipelineModelSelection( resolveModelAlias(normalized) ?? resolveModelAlias(stripped); - return { - requestedModelId, - resolvedModelId: resolved ?? DEFAULT_MODEL, - isFreeRoute: resolved ? isFreeModel(resolved) : false, - usedDefault: !resolved, - }; + if (resolved) { + return { + requestedModelId, + resolvedModelId: resolved, + isFreeRoute: isFreeModel(resolved), + usedDefault: false, + }; + } + + // Fail closed (SPEC-v9 A.0.10): a free-looking unknown id stays in the free + // lane; any other unknown id throws. It never becomes DEFAULT_MODEL, because + // the pipelines domain holds a launch counter, not a financial reservation. + if (isFreeLookingModelInput(requestedModelId) || isFreeLookingModelInput(stripped)) { + return { + requestedModelId, + resolvedModelId: FREE_LANE_MODEL, + routeTier: "free", + isFreeRoute: true, + usedDefault: false, + }; + } + throw new UnknownModelError(requestedModelId); +} + +/** + * Boundary guard for pipeline launches: throws `UnknownModelError` for an + * unknown, non-free-looking model id, using exactly the rules of + * `resolvePipelineModelSelection`. Empty input is the default route and passes. + */ +export function assertKnownPipelineModel(input: string | undefined | null): void { + resolvePipelineModelSelection(input); } // ═══════════════════════════════════════════════════════════════════════════ @@ -561,6 +628,17 @@ export const MODEL_SPECS: Record = { // Free availability is temporal; callers that require continuity must use an // explicit, provenance-preserving fallback rather than assuming this list is live. // ═══════════════════════════════════════════════════════════════════════════ + // Free lane (FREE_LANE_MODEL). Wire id and 262,144-token context from NodeVideo's + // measured benchmark (config/openrouter-free-routing.json, 8/8 successes); + // tools + structured outputs per SPEC-v9 s1.2. isFreeModel() derives "free" + // from the zero pricing below, which is what costClass: free means in this repo. + "nemotron-3-super-free": { + alias: "nemotron-3-super-free", + provider: "openrouter", + sdkId: "nvidia/nemotron-3-super-120b-a12b:free", + capabilities: { vision: false, toolUse: true, streaming: true, structuredOutputs: true, maxContext: 262_144 }, + pricing: { inputPerMillion: 0.00, outputPerMillion: 0.00 }, + }, "laguna-s-2.1-free": { alias: "laguna-s-2.1-free", provider: "openrouter", @@ -731,6 +809,11 @@ export const LEGACY_ALIASES: Record = { "gemini-flash": "gemini-3-flash-preview", "gemini-pro": "gemini-3.1-pro-preview", "gemini": "gemini-3-flash-preview", + // Provider-prefixed strings that wikiDreamingGraph.ts passes (and stores as modelUsed). + // They were unknown and silently ran on DEFAULT_MODEL; failing closed would break that + // job, so they resolve to the Gemini ids they name. SPEC-v9 A.2.10 PR 2 moves the ids. + "google:models/gemini-3-flash-preview": "gemini-3-flash-preview", + "google:models/gemini-3.1-pro-preview": "gemini-3.1-pro-preview", // Retired free models → redirect to best available free replacement "mimo-v2-flash-free": "qwen3-coder-free", @@ -762,6 +845,8 @@ export const LEGACY_ALIASES: Record = { "nvidia/llama-3.1-nemotron-70b-instruct:free": "nemotron-3-nano-free", "nemotron": "nemotron-3-nano-free", "nemotron-70b": "nemotron-3-nano-free", + // Free lane: the OpenRouter wire id resolves to the typed alias (SPEC-v9 A.2.10 PR 1) + "nvidia/nemotron-3-super-120b-a12b:free": "nemotron-3-super-free", // Current free model aliases (verified July 22, 2026) "poolside/laguna-s-2.1:free": "laguna-s-2.1-free", "poolside/laguna-s-2.1": "laguna-s-2.1-free", @@ -902,19 +987,59 @@ export function getLanguageModelOrThrow(input: string): LanguageModel { return getLanguageModel(resolved); } +// Unknown ids that were turned into DEFAULT_MODEL under `allowPaidFallback`. +// A plain number, so it cannot grow: it is a count for the trace, not a store. +let paidFallbackCount = 0; + +/** Number of unknown ids resolved to DEFAULT_MODEL via `allowPaidFallback` in this process. */ +export function getPaidFallbackCount(): number { + return paidFallbackCount; +} + /** - * Get a language model from untrusted input with fallback to default + * Resolve untrusted input to an approved alias or fail closed (SPEC-v9 A.0.10): + * - empty input -> DEFAULT_MODEL + * - known alias -> that alias + * - unknown, free-looking -> FREE_LANE_MODEL (never a paid id) + * - unknown, allowPaidFallback===true -> DEFAULT_MODEL, counted and annotated + * - unknown otherwise -> throws UnknownModelError */ -export function getLanguageModelSafe(input: string | undefined | null): LanguageModel { - if (!input) { - return getLanguageModel(DEFAULT_MODEL); - } +function resolveOrFailClosed( + input: string | undefined | null, + opts: UnknownModelOptions | undefined, + caller: string, +): ApprovedModel { + if (!input) return DEFAULT_MODEL; const resolved = resolveModelAlias(input); - if (!resolved) { - console.warn(`[modelResolver] Unknown model "${input}", falling back to ${DEFAULT_MODEL}`); - return getLanguageModel(DEFAULT_MODEL); + if (resolved) return resolved; + if (isFreeLookingModelInput(input)) { + console.warn(`[modelResolver] ${caller}: unknown free-looking model "${input}" -> ${FREE_LANE_MODEL}`); + return FREE_LANE_MODEL; } - return getLanguageModel(resolved); + if (opts?.allowPaidFallback === true) { + paidFallbackCount += 1; + console.warn( + `[modelResolver] ${caller}: unknown model "${input}" -> ${DEFAULT_MODEL} (allowPaidFallback; count=${paidFallbackCount})`, + ); + return DEFAULT_MODEL; + } + throw new UnknownModelError(input); +} + +/** + * Get a language model from untrusted input. Fails closed for unknown ids; see + * `resolveOrFailClosed`. `allowPaidFallback: true` is permitted only where the + * same function awaits `reserveLlmRequestInternal` for the resolved model. + */ +export function getLanguageModelSafe( + input: string | undefined | null, + opts?: UnknownModelOptions, +): LanguageModel { + const resolved = resolveOrFailClosed(input, opts, "getLanguageModelSafe"); + const substituted = Boolean(input) && resolveModelAlias(input as string) === null; + const spec = MODEL_SPECS[resolved]; + logModelResolution(String(input ?? ""), spec, "getLanguageModelSafe", substituted ? "unknown_input" : undefined); + return buildLanguageModel(spec); } /** @@ -945,12 +1070,16 @@ export interface ModelResolutionEvent { caller: string; success: boolean; errorMessage?: string; + /** Set when the requested id was unknown and the resolver substituted another. */ + fallback?: "unknown_input"; + paidFallbackCount?: number; } function logModelResolution( requestedAlias: string, spec: ModelSpec, - caller: string + caller: string, + fallback?: "unknown_input", ): void { const event: ModelResolutionEvent = { event: "ModelResolution", @@ -961,6 +1090,7 @@ function logModelResolution( provider: spec.provider, caller, success: true, + ...(fallback ? { fallback, paidFallbackCount } : {}), }; console.log("[ModelResolver]", JSON.stringify(event)); } @@ -1024,14 +1154,11 @@ export function getModelsWithCapabilities( * Safely resolve a model alias and return normalized alias * Used for API boundary normalization */ -export function normalizeModelInput(input: string | undefined | null): ApprovedModel { - if (!input) return DEFAULT_MODEL; - const resolved = resolveModelAlias(input); - if (!resolved) { - console.warn(`[modelResolver] Normalizing unknown model "${input}" to ${DEFAULT_MODEL}`); - return DEFAULT_MODEL; - } - return resolved; +export function normalizeModelInput( + input: string | undefined | null, + opts?: UnknownModelOptions, +): ApprovedModel { + return resolveOrFailClosed(input, opts, "normalizeModelInput"); } // ═══════════════════════════════════════════════════════════════════════════ @@ -1231,7 +1358,9 @@ export async function executeWithModelFallback( * Free availability is temporal; production callers should retain an explicit fallback. */ export function getBestFreeModel(): ApprovedModel { - return "laguna-s-2.1-free"; + // laguna-s-2.1:free expires on OpenRouter 2026-10-31 (SPEC-v9 A.1); the free + // lane is the registered, measured FREE_LANE_MODEL. + return FREE_LANE_MODEL; } /** diff --git a/backend/convex/domains/ai/ai.ts b/backend/convex/domains/ai/ai.ts index 2f0cca2fa..34dd9542f 100644 --- a/backend/convex/domains/ai/ai.ts +++ b/backend/convex/domains/ai/ai.ts @@ -10,7 +10,7 @@ import { generateText } from "ai"; import { Id } from "../../_generated/dataModel"; import { z } from "zod"; import { zodToJsonSchema } from "zod-to-json-schema"; -import { getLlmModel, resolveModelAlias, getModelWithFailover } from "../../../../shared/llm/modelCatalog"; +import { getLlmModel, resolveModelAlias, getModelWithFailover, anthropicText } from "../../../../shared/llm/modelCatalog"; // Legacy import - agentTools not used with new Convex Agent system // import { agentToolsOpenAI } from "../agents/agentTools"; @@ -198,7 +198,7 @@ export const generateResponse = action({ system: messages.filter((m: any) => m.role === "system").map((m: any) => m.content).join("\n"), messages: messages.filter((m: any) => m.role !== "system"), }); - responseMessage = { content: response.content[0]?.type === "text" ? response.content[0].text : "" }; + responseMessage = { content: anthropicText(response.content) }; toolCalls = undefined; } else if (provider === "gemini") { const google = createGoogleGenerativeAI({ apiKey: process.env.GEMINI_API_KEY }); diff --git a/backend/convex/domains/ai/models/modelRouter.ts b/backend/convex/domains/ai/models/modelRouter.ts index 14139e73b..5ae533607 100644 --- a/backend/convex/domains/ai/models/modelRouter.ts +++ b/backend/convex/domains/ai/models/modelRouter.ts @@ -16,7 +16,7 @@ import { v } from "convex/values"; import { internalAction } from "../../../_generated/server"; import { internal } from "../../../_generated/api"; import { generateText } from "ai"; -import { getLanguageModelSafe } from "../../agents/mcp_tools/models"; +import { getLanguageModelSafe, getProviderForModel, resolveModelAlias } from "../../agents/mcp_tools/models"; // B-PR4: capability-aware fallback chain. // resolveChain replaces the static `[...TIER_MODELS[tier]]` candidate list // with a tier-floor-enforced ordering that respects requested capabilities @@ -98,17 +98,24 @@ export interface RouteResult { }; } +/** Claude 4.7+/5.x 400 on non-default sampling and Gemini 3.x deprecates it. */ +function acceptsSampling(modelId: string): boolean { + const alias = resolveModelAlias(modelId); + const provider = alias ? getProviderForModel(alias) : null; + return provider !== "anthropic" && provider !== "google"; +} + // ═══════════════════════════════════════════════════════════════════════════ // MODEL TIERS — which models serve which tier // ═══════════════════════════════════════════════════════════════════════════ const TIER_MODELS: Record = { + // openai-gpt-oss-120b-free, arcee-trinity-large-free and nemotron-3-nano-30b-free + // were dropped: they are not approved aliases, so getLanguageModelSafe used to + // route them to the paid DEFAULT_MODEL (SPEC-v9 A.2.10 PR 1). free: [ "qwen3-coder-free", "step-3.5-flash-free", - "openai-gpt-oss-120b-free", - "arcee-trinity-large-free", - "nemotron-3-nano-30b-free", "llama-3.3-70b-free", ], cheap: [ @@ -431,7 +438,9 @@ export const route = internalAction({ content: m.content, })), maxOutputTokens: args.maxTokens ?? 2048, - temperature: args.temperature ?? 0.3, + // Claude 4.7+/5.x reject non-default sampling (400) and Gemini 3.x + // deprecates it, so only the other providers receive temperature. + ...(acceptsSampling(modelId) ? { temperature: args.temperature ?? 0.3 } : {}), }); const inputTokens = result.usage?.inputTokens ?? 0; diff --git a/backend/convex/domains/ai/morningDigest.ts b/backend/convex/domains/ai/morningDigest.ts index d2e40b8e7..55ad44209 100644 --- a/backend/convex/domains/ai/morningDigest.ts +++ b/backend/convex/domains/ai/morningDigest.ts @@ -6,7 +6,7 @@ import OpenAI from "openai"; import Anthropic from "@anthropic-ai/sdk"; import { createGoogleGenerativeAI } from "@ai-sdk/google"; import { generateText } from "ai"; -import { getLlmModel, getProviderForModel, resolveModelAlias, getModelWithFailover } from "../../../../shared/llm/modelCatalog"; +import { getLlmModel, getProviderForModel, resolveModelAlias, getModelWithFailover, anthropicText } from "../../../../shared/llm/modelCatalog"; // NOTE: Query is in morningDigestQueries.ts (Convex requires queries in non-Node files) // Import it via: api.domains.ai.morningDigestQueries.getDigestData @@ -28,7 +28,7 @@ async function generateWithProvider( system: systemPrompt, messages: [{ role: "user", content: userPrompt }], }); - return response.content[0]?.type === "text" ? response.content[0].text : ""; + return anthropicText(response.content); } if (provider === "gemini") { diff --git a/backend/convex/domains/evaluation/personaEpisodeEval.ts b/backend/convex/domains/evaluation/personaEpisodeEval.ts index 36c21c051..27f584d9a 100644 --- a/backend/convex/domains/evaluation/personaEpisodeEval.ts +++ b/backend/convex/domains/evaluation/personaEpisodeEval.ts @@ -4,7 +4,7 @@ import { v } from "convex/values"; import { action } from "../../_generated/server"; import { internal, components } from "../../_generated/api"; import { Agent, stepCountIs } from "@convex-dev/agent"; -import { getLanguageModelSafe, DEFAULT_MODEL } from "../agents/mcp_tools/models"; +import { getLanguageModelSafe, normalizeModelInput, DEFAULT_MODEL } from "../agents/mcp_tools/models"; import { PERSONAS, type Persona, GROUND_TRUTH_ENTITIES } from "./groundTruth"; import { calculateRequestCost } from "../../../../shared/llm/modelCatalog"; @@ -1020,6 +1020,9 @@ export const runPersonaEpisodeEval = action({ } const model = String(args.model ?? DEFAULT_MODEL); + // What the resolver chose for `model`; throws UnknownModelError for an unknown id, + // so a run can never be labelled with a candidate it did not use (SPEC-v9 A.0.3). + const resolvedModelId = normalizeModelInput(model); const suite = args.suite ?? "core"; const useDbScenarios = args.useDbScenarios === true; @@ -1354,6 +1357,7 @@ export const runPersonaEpisodeEval = action({ return { ok: total > 0 && passed === total, model, + resolvedModelId, elapsedMs: Date.now() - startedAt, window: { suite, diff --git a/backend/convex/domains/knowledge/entityInsights.ts b/backend/convex/domains/knowledge/entityInsights.ts index 0e930c5ac..0c459afd7 100644 --- a/backend/convex/domains/knowledge/entityInsights.ts +++ b/backend/convex/domains/knowledge/entityInsights.ts @@ -12,6 +12,7 @@ import { getLlmModel, resolveModelAlias, getModelWithFailover, + anthropicText, } from "../../../../shared/llm/modelCatalog"; import { runPiOrAiSdkCompletion } from "../pipelines/piRuntime"; import { linkupSearch } from "../../tools/media/linkupSearch"; @@ -632,7 +633,7 @@ async function generateWithProvider( system: systemPrompt, messages: [{ role: "user", content: userPrompt }], }); - return response.content[0]?.type === "text" ? response.content[0].text : ""; + return anthropicText(response.content); } if (provider === "gemini") { diff --git a/backend/convex/domains/pipelines/piRuntime.ts b/backend/convex/domains/pipelines/piRuntime.ts index be9f076a1..27d1cc8f1 100644 --- a/backend/convex/domains/pipelines/piRuntime.ts +++ b/backend/convex/domains/pipelines/piRuntime.ts @@ -142,27 +142,30 @@ function parseRawModelKey(input: string): { return { provider: "openai", modelId: input, requestedModelId: input }; } -async function parseModelKey(input: string): Promise<{ +// Exported for the fail-closed test (SPEC-v9 A.2.10 PR 1); not a Convex function. +export async function parseModelKey(input: string): Promise<{ provider: PiProviderId; modelId: string; requestedModelId: string; resolvedAlias?: string; }> { + // Only a failure to LOAD the resolver module may fall through to the raw + // parser. An UnknownModelError (or any resolver error) must surface: the raw + // parser would forward an arbitrary id to pi-ai (SPEC-v9 A.0.10). + let resolver: typeof import("../agents/mcp_tools/models/modelResolver"); try { - const { getModelSpec, resolvePipelineModelSelection } = await import( - "../agents/mcp_tools/models/modelResolver" - ); - const selection = resolvePipelineModelSelection(input); - const spec = getModelSpec(selection.resolvedModelId); - return { - provider: spec.provider as PiProviderId, - modelId: spec.sdkId, - requestedModelId: selection.requestedModelId, - resolvedAlias: selection.resolvedModelId, - }; + resolver = await import("../agents/mcp_tools/models/modelResolver"); } catch { return parseRawModelKey(input); } + const selection = resolver.resolvePipelineModelSelection(input); + const spec = resolver.getModelSpec(selection.resolvedModelId); + return { + provider: spec.provider as PiProviderId, + modelId: spec.sdkId, + requestedModelId: selection.requestedModelId, + resolvedAlias: selection.resolvedModelId, + }; } /** diff --git a/backend/convex/domains/pipelines/pipelineModelBoundary.test.ts b/backend/convex/domains/pipelines/pipelineModelBoundary.test.ts new file mode 100644 index 000000000..86258b241 --- /dev/null +++ b/backend/convex/domains/pipelines/pipelineModelBoundary.test.ts @@ -0,0 +1,102 @@ +/// +/** + * Pipeline launches fail closed on an unknown model id (SPEC-v9 A.0.10, A.2.10 PR 1). + * + * Persona: a signed-in user (or a stale client) starts a research pipeline with a model + * id the resolver has never heard of. Before this change the id silently became the + * paid DEFAULT_MODEL, and the only control in front of it was a per-window launch + * counter that knows nothing about models or dollars. Now the launch is refused before + * it consumes an admission slot or starts a workflow, and piRuntime never forwards the + * raw id to pi-ai. + */ + +import { describe, expect, it, vi } from "vitest"; + +import schema from "../../schema"; +import { api } from "../../_generated/api"; +import { UnknownModelError } from "../agents/mcp_tools/models/modelResolver"; +import { parseModelKey } from "./piRuntime"; + +const DIR_SEGMENTS = ["domains", "pipelines"]; +function rerootGlobKey(key: string): string { + const parts = key.replace(/^\.\//, "").split("/"); + const base = [...DIR_SEGMENTS]; + while (parts[0] === "..") { + parts.shift(); + base.pop(); + } + return [...base, ...parts].join("/"); +} + +const convexModules = Object.fromEntries( + Object.entries(import.meta.glob("../../**/*.{ts,js}")).map(([key, loader]) => [rerootGlobKey(key), loader]), +); + +let convexTest: any; +let convexTestAvailable = false; +try { + const mod = await import(/* @vite-ignore */ "convex-test"); + convexTest = mod.convexTest; + convexTestAvailable = typeof convexTest === "function"; +} catch { + convexTestAvailable = false; +} + +const pipelines = (api as any).domains.pipelines; + +describe.skipIf(!convexTestAvailable)("startPipelineRun model boundary", () => { + it("rejects an unknown modelId and leaves pipelineLaunchAdmissions unchanged", async () => { + const t = convexTest(schema, convexModules); + const userId = await t.run(async (ctx: any) => ctx.db.insert("users", { email: "boundary@example.com" })); + const user = t.withIdentity({ subject: String(userId) }); + const countAdmissions = () => t.run(async (ctx: any) => (await ctx.db.query("pipelineLaunchAdmissions").collect()).length); + + const before = await countAdmissions(); + await expect( + user.mutation(pipelines.pipelineWorkflow.startPipelineRun, { + pipelineKind: "research", + spec: "Research a company", + modelId: "gpt-9-ultra", + }), + ).rejects.toThrow(/Unknown model "gpt-9-ultra"/); + expect(await countAdmissions()).toBe(before); + }, 20_000); + + it("rejects an unknown modelId on the trusted internal entry point too", async () => { + const t = convexTest(schema, convexModules); + const countAdmissions = () => t.run(async (ctx: any) => (await ctx.db.query("pipelineLaunchAdmissions").collect()).length); + const before = await countAdmissions(); + await expect( + t.mutation(pipelines.pipelineWorkflow.startPipelineRunInternal, { + pipelineKind: "research", + spec: "Cron research", + modelId: "openai:gpt-9", + ownerKey: "user:cron-owner", + }), + ).rejects.toThrow(/Unknown model/); + expect(await countAdmissions()).toBe(before); + }, 20_000); +}); + +describe("piRuntime.parseModelKey", () => { + it("throws for an unknown provider:model instead of returning a raw handle", async () => { + // Before: the catch fell through to parseRawModelKey, which forwarded "gpt-9" to pi-ai. + await expect(parseModelKey("openai:gpt-9")).rejects.toBeInstanceOf(UnknownModelError); + await expect(parseModelKey("gpt-9-ultra")).rejects.toBeInstanceOf(UnknownModelError); + }); + + it("still resolves approved aliases, route aliases and free-looking ids", async () => { + vi.spyOn(console, "log").mockImplementation(() => {}); + expect(await parseModelKey("openai:gpt-4o-mini")).toMatchObject({ + provider: "openai", + modelId: "gpt-5.4-mini", + resolvedAlias: "gpt-5.4-mini", + }); + expect(await parseModelKey("nodebench:auto-balanced")).toMatchObject({ resolvedAlias: "kimi-k2.6" }); + expect(await parseModelKey("x:free")).toMatchObject({ + provider: "openrouter", + modelId: "nvidia/nemotron-3-super-120b-a12b:free", + }); + vi.restoreAllMocks(); + }); +}); diff --git a/backend/convex/domains/pipelines/pipelineWorkflow.ts b/backend/convex/domains/pipelines/pipelineWorkflow.ts index e5142efaf..7c67b08dd 100644 --- a/backend/convex/domains/pipelines/pipelineWorkflow.ts +++ b/backend/convex/domains/pipelines/pipelineWorkflow.ts @@ -23,6 +23,7 @@ import { v } from "convex/values"; import { internalMutation, mutation } from "../../_generated/server"; import { components, internal } from "../../_generated/api"; import { WorkflowManager } from "@convex-dev/workflow"; +import { assertKnownPipelineModel } from "../agents/mcp_tools/models/modelResolver"; import { requireAuthenticatedPipelineOwnerKey } from "./pipelineOwnership"; import { normalizePipelineLaunchText, @@ -198,6 +199,9 @@ export const startPipelineRun = mutation({ }), handler: async (ctx, args): Promise<{ workflowId: string }> => { const ownerKey = await requireAuthenticatedPipelineOwnerKey(ctx); + // Fail closed before admission: an unknown model id neither consumes a + // launch slot nor starts a workflow (SPEC-v9 A.0.10). + assertKnownPipelineModel(args.modelId); await reservePipelineLaunchAdmission(ctx, ownerKey); return await startPipelineWorkflowForOwner(ctx, args, ownerKey); }, @@ -214,6 +218,7 @@ export const startPipelineRunInternal = internalMutation({ handler: async (ctx, args): Promise<{ workflowId: string }> => { const ownerKey = normalizePipelineOwnerKey(args.ownerKey); const { ownerKey: _ownerKey, ...launchArgs } = args; + assertKnownPipelineModel(args.modelId); if (ownerKey.startsWith("user:")) { await reservePipelineLaunchAdmission(ctx, ownerKey); } diff --git a/backend/convex/domains/product/extendedThinking.ts b/backend/convex/domains/product/extendedThinking.ts index afcf8467c..9748393bf 100644 --- a/backend/convex/domains/product/extendedThinking.ts +++ b/backend/convex/domains/product/extendedThinking.ts @@ -44,11 +44,11 @@ import { import { api, internal } from "../../_generated/api"; import { buildCheckpointPrompt, + buildCheckpointRequestBody, parseCheckpointResponse, shouldContinue, MAX_CHECKPOINTS, MAX_THINKING_BUDGET_TOKENS, - MAX_OUTPUT_TOKENS, } from "../../../../workers/node/pipeline/extendedThinkingRunner"; const DEFAULT_MODEL = "claude-sonnet-4.6"; @@ -333,16 +333,13 @@ export const advanceRun = internalAction({ "anthropic-version": "2023-06-01", }, signal: controller.signal, - body: JSON.stringify({ - model: run.modelName, - max_tokens: MAX_OUTPUT_TOKENS, - thinking: { - type: "enabled", - budget_tokens: Math.min(10_000, run.thinkingBudgetTokens - run.thinkingTokensUsed), - }, - system: prompt.system, - messages: [{ role: "user", content: prompt.user }], - }), + body: JSON.stringify( + buildCheckpointRequestBody({ + modelName: run.modelName, + prompt, + remainingThinkingTokens: run.thinkingBudgetTokens - run.thinkingTokensUsed, + }), + ), }); if (!response.ok) { diff --git a/backend/convex/domains/research/dailyBriefPersonalOverlay.ts b/backend/convex/domains/research/dailyBriefPersonalOverlay.ts index 52f2f045b..aaa060a54 100644 --- a/backend/convex/domains/research/dailyBriefPersonalOverlay.ts +++ b/backend/convex/domains/research/dailyBriefPersonalOverlay.ts @@ -20,6 +20,7 @@ import { getLlmModel, resolveModelAlias, getModelWithFailover, + anthropicText, } from "../../../../shared/llm/modelCatalog"; async function generateWithProvider( @@ -40,7 +41,7 @@ async function generateWithProvider( system: systemPrompt, messages: [{ role: "user", content: userPrompt }], }); - return response.content[0]?.type === "text" ? response.content[0].text : ""; + return anthropicText(response.content); } if (provider === "gemini") { diff --git a/backend/convex/domains/research/dailyBriefWorker.ts b/backend/convex/domains/research/dailyBriefWorker.ts index 64d4810ee..bf074131c 100644 --- a/backend/convex/domains/research/dailyBriefWorker.ts +++ b/backend/convex/domains/research/dailyBriefWorker.ts @@ -19,6 +19,7 @@ import { getLlmModel, resolveModelAlias, getModelWithFailover, + anthropicText, } from "../../../../shared/llm/modelCatalog"; async function generateWithProvider( @@ -39,7 +40,7 @@ async function generateWithProvider( system: systemPrompt, messages: [{ role: "user", content: userPrompt }], }); - return response.content[0]?.type === "text" ? response.content[0].text : ""; + return anthropicText(response.content); } if (provider === "gemini") { diff --git a/backend/convex/domains/research/dealFlow.ts b/backend/convex/domains/research/dealFlow.ts index f903e12af..bf92905b3 100644 --- a/backend/convex/domains/research/dealFlow.ts +++ b/backend/convex/domains/research/dealFlow.ts @@ -12,6 +12,7 @@ import { getLlmModel, resolveModelAlias, getModelWithFailover, + anthropicText, } from "../../../../shared/llm/modelCatalog"; import { linkupSearch } from "../../tools/media/linkupSearch"; @@ -60,7 +61,7 @@ async function generateWithProvider( system: systemPrompt, messages: [{ role: "user", content: userPrompt }], }); - return response.content[0]?.type === "text" ? response.content[0].text : ""; + return anthropicText(response.content); } if (provider === "gemini") { diff --git a/backend/convex/domains/research/modelComparison.ts b/backend/convex/domains/research/modelComparison.ts index 4fb7df35b..750a4af3a 100644 --- a/backend/convex/domains/research/modelComparison.ts +++ b/backend/convex/domains/research/modelComparison.ts @@ -12,6 +12,7 @@ import { resolveModelAlias, getModelWithFailover, modelPricing, + anthropicText, } from "../../../../shared/llm/modelCatalog"; const CACHE_TTL_MS = 12 * 60 * 60 * 1000; @@ -34,7 +35,7 @@ async function generateWithProvider( system: systemPrompt, messages: [{ role: "user", content: userPrompt }], }); - return response.content[0]?.type === "text" ? response.content[0].text : ""; + return anthropicText(response.content); } if (provider === "gemini") { diff --git a/backend/convex/domains/research/paperDetails.ts b/backend/convex/domains/research/paperDetails.ts index 432300a9d..600ae3e73 100644 --- a/backend/convex/domains/research/paperDetails.ts +++ b/backend/convex/domains/research/paperDetails.ts @@ -12,6 +12,7 @@ import { getLlmModel, resolveModelAlias, getModelWithFailover, + anthropicText, } from "../../../../shared/llm/modelCatalog"; import { linkupSearch } from "../../tools/media/linkupSearch"; @@ -35,7 +36,7 @@ async function generateWithProvider( system: systemPrompt, messages: [{ role: "user", content: userPrompt }], }); - return response.content[0]?.type === "text" ? response.content[0].text : ""; + return anthropicText(response.content); } if (provider === "gemini") { diff --git a/backend/convex/domains/research/repoScout.ts b/backend/convex/domains/research/repoScout.ts index 7e32a91c5..999508098 100644 --- a/backend/convex/domains/research/repoScout.ts +++ b/backend/convex/domains/research/repoScout.ts @@ -12,6 +12,7 @@ import { getLlmModel, resolveModelAlias, getModelWithFailover, + anthropicText, } from "../../../../shared/llm/modelCatalog"; import { linkupSearch } from "../../tools/media/linkupSearch"; @@ -47,7 +48,7 @@ async function generateWithProvider( system: systemPrompt, messages: [{ role: "user", content: userPrompt }], }); - return response.content[0]?.type === "text" ? response.content[0].text : ""; + return anthropicText(response.content); } if (provider === "gemini") { diff --git a/backend/convex/domains/research/stackImpact.ts b/backend/convex/domains/research/stackImpact.ts index 7b9b03ca3..70c45233b 100644 --- a/backend/convex/domains/research/stackImpact.ts +++ b/backend/convex/domains/research/stackImpact.ts @@ -12,6 +12,7 @@ import { getLlmModel, resolveModelAlias, getModelWithFailover, + anthropicText, } from "../../../../shared/llm/modelCatalog"; type ReaderContentResult = { @@ -46,7 +47,7 @@ async function generateWithProvider( system: systemPrompt, messages: [{ role: "user", content: userPrompt }], }); - return response.content[0]?.type === "text" ? response.content[0].text : ""; + return anthropicText(response.content); } if (provider === "gemini") { diff --git a/backend/convex/domains/research/strategyMetrics.ts b/backend/convex/domains/research/strategyMetrics.ts index 819642907..640bf2b83 100644 --- a/backend/convex/domains/research/strategyMetrics.ts +++ b/backend/convex/domains/research/strategyMetrics.ts @@ -12,6 +12,7 @@ import { getLlmModel, resolveModelAlias, getModelWithFailover, + anthropicText, } from "../../../../shared/llm/modelCatalog"; const CACHE_TTL_MS = 12 * 60 * 60 * 1000; @@ -34,7 +35,7 @@ async function generateWithProvider( system: systemPrompt, messages: [{ role: "user", content: userPrompt }], }); - return response.content[0]?.type === "text" ? response.content[0].text : ""; + return anthropicText(response.content); } if (provider === "gemini") { diff --git a/backend/convex/domains/search/fusion/benchmark.ts b/backend/convex/domains/search/fusion/benchmark.ts index 14668ec6b..2316d8d57 100644 --- a/backend/convex/domains/search/fusion/benchmark.ts +++ b/backend/convex/domains/search/fusion/benchmark.ts @@ -43,6 +43,7 @@ import { v } from "convex/values"; import { action, internalMutation, internalQuery } from "../../../_generated/server"; import { internal, api } from "../../../_generated/api"; import type { SearchResponse, SearchSource, FusionSearchPayload } from "./types"; +import { anthropicText } from "../../../../../shared/llm/modelCatalog"; import { normalizeModelInput, getModelSpec, @@ -587,7 +588,7 @@ export const evaluateSearch = action({ body: JSON.stringify({ model: modelSpec.sdkId, max_tokens: 1500, - temperature: JUDGE_TEMPERATURE, // Pinned to 0 for determinism + // No temperature: Claude 4.7+/5.x answer 400 to non-default sampling. messages: [ { role: "user", content: `You are an expert search quality evaluator. Respond only with valid JSON.\n\n${prompt}` }, ], @@ -600,7 +601,7 @@ export const evaluateSearch = action({ } const data = await response.json(); - llmResponse = data.content?.[0]?.text || ""; + llmResponse = anthropicText(data.content); } else if (modelSpec.provider === "google") { const response = await fetch( `https://generativelanguage.googleapis.com/v1beta/models/${modelSpec.sdkId}:generateContent?key=${process.env.GOOGLE_GENERATIVE_AI_API_KEY}`, diff --git a/backend/convex/domains/social/postDedupAction.ts b/backend/convex/domains/social/postDedupAction.ts index 9d783d7cf..e73306eb3 100644 --- a/backend/convex/domains/social/postDedupAction.ts +++ b/backend/convex/domains/social/postDedupAction.ts @@ -23,6 +23,7 @@ import { getLlmModel, resolveModelAlias, getModelWithFailover, + anthropicText, } from "../../../../shared/llm/modelCatalog"; import { generateDedupJudgePrompt, type DedupVerdict } from "./postDedup"; @@ -71,9 +72,7 @@ async function callDedupJudge(prompt: string): Promise { messages: [{ role: "user", content: prompt }], }); content = - response.content[0]?.type === "text" - ? response.content[0].text - : undefined; + anthropicText(response.content) || undefined; } else if (provider === "gemini") { const google = createGoogleGenerativeAI({ apiKey: process.env.GEMINI_API_KEY, diff --git a/backend/convex/domains/verification/claimVerificationAction.ts b/backend/convex/domains/verification/claimVerificationAction.ts index 10bdf7799..f4199e523 100644 --- a/backend/convex/domains/verification/claimVerificationAction.ts +++ b/backend/convex/domains/verification/claimVerificationAction.ts @@ -12,7 +12,7 @@ import OpenAI from "openai"; import Anthropic from "@anthropic-ai/sdk"; import { createGoogleGenerativeAI } from "@ai-sdk/google"; import { generateText } from "ai"; -import { getLlmModel, resolveModelAlias, getModelWithFailover } from "../../../../shared/llm/modelCatalog"; +import { getLlmModel, resolveModelAlias, getModelWithFailover, anthropicText } from "../../../../shared/llm/modelCatalog"; // ═══════════════════════════════════════════════════════════════════════════ // TYPES @@ -161,7 +161,7 @@ async function judgeClaimAgainstSource( system: JUDGE_SYSTEM_PROMPT, messages: [{ role: "user", content: userPrompt }], }); - content = response.content[0]?.type === "text" ? response.content[0].text : undefined; + content = anthropicText(response.content) || undefined; } else if (provider === "gemini") { const google = createGoogleGenerativeAI({ apiKey: process.env.GEMINI_API_KEY }); const result = await generateText({ diff --git a/backend/convex/events.requestShape.test.ts b/backend/convex/events.requestShape.test.ts new file mode 100644 index 000000000..f3ffeab7d --- /dev/null +++ b/backend/convex/events.requestShape.test.ts @@ -0,0 +1,113 @@ +// @vitest-environment node +/** + * Request-shape tests for the raw-fetch Anthropic paths in events.ts + * (SPEC-v9 A.0.6, A.2.10 PR 1). + * + * Persona: a ScratchNode event attendee asks a question, and a host triages a join + * request. Each call builds its real request through events.ts's own code and hands + * it to a fake transport, so the assertions are about the bytes that would reach + * api.anthropic.com. Claude 4.7+/5.x answer 400 to a non-default temperature, and + * adaptive thinking counts toward max_tokens, so the body must carry no sampling + * keys and a cap that leaves room to think. No network, no spend. + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { generateBouncerAdvisory, generateProviderAnswer, runJoinBouncer } from "./events"; + +const SAMPLING_KEYS = ["temperature", "top_p", "top_k"] as const; +// Claude Haiku 5.5 runs adaptive thinking by default, so content[0] is a thinking block. +const thinkingThenText = (text: string) => ({ + model: "claude-haiku-4-5-20251001", + content: [ + { type: "thinking", thinking: "…", signature: "sig" }, + { type: "text", text }, + ], + usage: { input_tokens: 10, output_tokens: 20 }, +}); + +interface Captured { + url: string; + body: Record; +} + +function fakeAnthropic(responseBody: unknown): { captured: Captured[]; fetchMock: ReturnType } { + const captured: Captured[] = []; + const fetchMock = vi.fn(async (url: string, init: { body: string }) => { + captured.push({ url: String(url), body: JSON.parse(init.body) }); + return new Response(JSON.stringify(responseBody), { status: 200, headers: { "content-type": "application/json" } }); + }); + vi.stubGlobal("fetch", fetchMock); + return { captured, fetchMock }; +} + +function expectMessagesRequestShape(captured: Captured[], model: string) { + expect(captured).toHaveLength(1); + const { url, body } = captured[0]; + expect(url).toBe("https://api.anthropic.com/v1/messages"); + expect(body.model).toBe(model); + for (const key of SAMPLING_KEYS) expect(body, `${key} must be absent`).not.toHaveProperty(key); + expect(body.max_tokens).toBeGreaterThanOrEqual(1024); + // no prefill: the last turn is the user's + expect(body.messages.at(-1).role).toBe("user"); +} + +describe("events.ts Anthropic request shape", () => { + beforeEach(() => { + vi.stubEnv("ANTHROPIC_API_KEY", "test-key-not-real"); + vi.stubEnv("SCRATCHNODE_BOUNCER_MODEL", ""); + }); + afterEach(() => { + vi.unstubAllGlobals(); + vi.unstubAllEnvs(); + }); + + it("answer path: no sampling keys, room to think, text read by block type", async () => { + const { captured } = fakeAnthropic(thinkingThenText("It starts at 6pm. Next move: arrive early.")); + const result = await generateProviderAnswer({ + eventName: "Demo Night", + question: "When does it start?", + sources: [{ title: "Agenda", excerpt: "Doors 6pm", body: "Doors open at 6pm." }], + model: "claude-haiku-4-5-20251001", + }); + expectMessagesRequestShape(captured, "claude-haiku-4-5-20251001"); + expect(result).toMatchObject({ ok: true, body: "It starts at 6pm. Next move: arrive early." }); + }); + + it("bouncer advisory path: no sampling keys, JSON read past a thinking block", async () => { + const { captured } = fakeAnthropic( + thinkingThenText( + JSON.stringify({ recommendation: "approve", riskScore: 0.1, flags: [], hostSummary: "Looks fine.", guestMessage: "Hi" }), + ), + ); + const result = await generateBouncerAdvisory({ + eventName: "Demo Night", + displayName: "Pat", + note: "friend of the host", + model: "claude-haiku-4-5-20251001", + }); + expectMessagesRequestShape(captured, "claude-haiku-4-5-20251001"); + expect(result).toMatchObject({ provider: "anthropic", recommendation: "approve" }); + }); + + it("join bouncer path: default model, no sampling keys, JSON read past a thinking block", async () => { + const { captured } = fakeAnthropic( + thinkingThenText( + JSON.stringify({ recommendation: "hold", riskScore: 0.4, flags: ["thin_request"], hostSummary: "Thin.", guestMessage: "Wait" }), + ), + ); + const result = await runJoinBouncer({ eventName: "Demo Night", displayName: "Sam", note: "" }); + // PR 1 moves no ids; PR 2 changes this literal with its own request-shape update. + expectMessagesRequestShape(captured, "claude-haiku-4-5-20251001"); + expect(result).toMatchObject({ recommendation: "hold" }); + }); + + it("join bouncer honours SCRATCHNODE_BOUNCER_MODEL (override precedence unchanged)", async () => { + vi.stubEnv("SCRATCHNODE_BOUNCER_MODEL", "claude-sonnet-4-6"); + const { captured } = fakeAnthropic( + thinkingThenText(JSON.stringify({ recommendation: "approve", riskScore: 0, flags: [], hostSummary: "", guestMessage: "" })), + ); + await runJoinBouncer({ eventName: "Demo Night", displayName: "Sam", note: "" }); + expect(captured[0].body.model).toBe("claude-sonnet-4-6"); + }); +}); diff --git a/backend/convex/events.ts b/backend/convex/events.ts index 88c44311a..103e6f68c 100644 --- a/backend/convex/events.ts +++ b/backend/convex/events.ts @@ -670,7 +670,8 @@ async function searchLinkup(question: string) { } } -async function generateProviderAnswer(args: { +// Exported for the request-shape test (SPEC-v9 A.0.6); not a Convex function. +export async function generateProviderAnswer(args: { eventName: string; question: string; sources: Array<{ title: string; excerpt: string; body: string; uri?: string }>; @@ -711,8 +712,9 @@ async function generateProviderAnswer(args: { }, body: JSON.stringify({ model, - max_tokens: 700, - temperature: 0.2, + // No temperature: Claude 4.7+/5.x answer 400 to non-default sampling. + // 2048 leaves room for adaptive thinking, which counts toward max_tokens. + max_tokens: 2048, system, messages: [{ role: "user", content: user }], }), @@ -810,7 +812,8 @@ function deterministicBouncerAdvisory(args: { }; } -async function generateBouncerAdvisory(args: { +// Exported for the request-shape test (SPEC-v9 A.0.6); not a Convex function. +export async function generateBouncerAdvisory(args: { eventName: string; displayName: string; note?: string; @@ -856,8 +859,8 @@ async function generateBouncerAdvisory(args: { }, body: JSON.stringify({ model: args.model, - max_tokens: 260, - temperature: 0, + // No temperature (400 on Claude 4.7+/5.x); 2048 covers thinking tokens. + max_tokens: 2048, system, messages: [{ role: "user", content: user }], }), @@ -2502,7 +2505,8 @@ const JOIN_BOUNCER_SYSTEM = [ 'Output STRICT JSON only: {"recommendation":"approve|hold|deny","riskScore":0.0,"flags":[],"hostSummary":"...","guestMessage":"..."}', ].join(" "); -async function runJoinBouncer(input: { eventName: string; displayName: string; note: string }): Promise< +// Exported for the request-shape test (SPEC-v9 A.0.6); not a Convex function. +export async function runJoinBouncer(input: { eventName: string; displayName: string; note: string }): Promise< | { recommendation: "approve" | "hold" | "deny"; riskScore: number; flags: string[]; hostSummary: string; guestMessage: string } | null > { @@ -2524,8 +2528,8 @@ async function runJoinBouncer(input: { eventName: string; displayName: string; n headers: { "x-api-key": apiKey, "anthropic-version": "2023-06-01", "content-type": "application/json" }, body: JSON.stringify({ model, - max_tokens: 300, - temperature: 0, + // No temperature (400 on Claude 4.7+/5.x); 2048 covers thinking tokens. + max_tokens: 2048, system: JOIN_BOUNCER_SYSTEM, messages: [{ role: "user", content: user }], }), diff --git a/backend/convex/tags_actions.ts b/backend/convex/tags_actions.ts index daa3321b4..5d425d1e8 100644 --- a/backend/convex/tags_actions.ts +++ b/backend/convex/tags_actions.ts @@ -8,7 +8,7 @@ import OpenAI from "openai"; import Anthropic from "@anthropic-ai/sdk"; import { createGoogleGenerativeAI } from "@ai-sdk/google"; import { generateText } from "ai"; -import { getLlmModel, resolveModelAlias, getModelWithFailover } from "../../shared/llm/modelCatalog"; +import { getLlmModel, resolveModelAlias, getModelWithFailover, anthropicText } from "../../shared/llm/modelCatalog"; /** * LLM-powered tag generation for documents @@ -82,7 +82,7 @@ Example: [{"name":"react","kind":"entity","importance":0.9},{"name":"web develop system: systemPrompt, messages: [{ role: "user", content: userPrompt }], }); - responseText = response.content[0]?.type === "text" ? response.content[0].text : "[]"; + responseText = anthropicText(response.content) || "[]"; } else if (provider === "gemini") { const google = createGoogleGenerativeAI({ apiKey: process.env.GEMINI_API_KEY }); const result = await generateText({ diff --git a/packages/mcp-local/src/tools/llmTools.ts b/packages/mcp-local/src/tools/llmTools.ts index ef6e8aad4..af3432721 100644 --- a/packages/mcp-local/src/tools/llmTools.ts +++ b/packages/mcp-local/src/tools/llmTools.ts @@ -108,7 +108,7 @@ async function getAnthropicProvider(): Promise { max_tokens: maxTokens ?? 1024, system: system ?? "", messages: [{ role: "user", content: prompt }], - temperature: temperature ?? 0.7, + // No temperature: Claude 4.7+/5.x answer 400 to non-default sampling. }); const text = result.content .filter((b) => b.type === "text") diff --git a/packages/mcp-local/src/tools/researchWritingTools.ts b/packages/mcp-local/src/tools/researchWritingTools.ts index 8c3e5cc6d..5534b24d5 100644 --- a/packages/mcp-local/src/tools/researchWritingTools.ts +++ b/packages/mcp-local/src/tools/researchWritingTools.ts @@ -100,7 +100,7 @@ async function getAnthropicProvider(): Promise { max_tokens: maxTokens ?? 2048, system: system ?? "", messages: [{ role: "user", content: prompt }], - temperature: temperature ?? 0.3, + // No temperature: Claude 4.7+/5.x answer 400 to non-default sampling. }); const text = result.content.filter((b) => b.type === "text").map((b) => (b as any).text as string).join(""); return { diff --git a/production-agent.json b/production-agent.json index d77f1b81b..a6c7b896f 100644 --- a/production-agent.json +++ b/production-agent.json @@ -112,7 +112,7 @@ }, "modelMigration": { "pinnedModel": "gemini-3.5-flash (nodeagent.yaml provider.model.id)", - "migrationEvalCommand": "npx tsx scripts/eval-harness/runner.ts --task tier1 --seeds 3" + "migrationEvalCommand": "EVAL_BACKEND=anthropic EVAL_MODEL= npx tsx scripts/eval-harness/runner.ts --tier tier1 --seeds 3" }, "tracing": { "traceIdPropagated": true, diff --git a/scripts/__tests__/comprehensiveEvalModels.test.ts b/scripts/__tests__/comprehensiveEvalModels.test.ts new file mode 100644 index 000000000..cbd8cb001 --- /dev/null +++ b/scripts/__tests__/comprehensiveEvalModels.test.ts @@ -0,0 +1,42 @@ +// @vitest-environment node +/** + * run-comprehensive-eval.ts is how a model candidate gets measured + * (`--models --suite full --judge ...`, SPEC-v9 A.2.10). Two ways it used + * to mislead: a `--models` value the resolver did not know was silently served by the + * default model (so "candidate X" measured kimi-k2.6), and its own retry could switch + * models mid-row. These helpers are what makes the run abort and the row disqualify. + * Pure functions; no Convex, no network, no spend. + */ + +import { describe, expect, it } from "vitest"; + +import { identityViolations, unresolvedModels } from "../run-comprehensive-eval"; + +describe("unresolvedModels (--models must resolve or the run aborts)", () => { + it("accepts approved ids and legacy aliases", () => { + expect(unresolvedModels(["kimi-k2.6", "claude-sonnet-4.6", "gpt-4o-mini", "nemotron-3-super-free"])).toEqual([]); + }); + + it("flags ids the resolver does not know, including the removed minimax-m2.1", () => { + expect(unresolvedModels(["kimi-k2.6", "minimax-m2.1", "claude-opus-9"])).toEqual(["minimax-m2.1", "claude-opus-9"]); + }); +}); + +describe("identityViolations (a row is evidence only for the model it requested)", () => { + const row = (over: Record) => + ({ model: "claude-sonnet-4.6", scenarioId: "s1", scenarioName: "s", passed: true, elapsed: 1, failureReasons: [], ...over }) as never; + + it("passes rows that resolved to the requested id without a retry switch", () => { + expect(identityViolations([row({ resolvedModelId: "claude-sonnet-4.6", fallbackUsed: false })])).toEqual([]); + }); + + it("disqualifies a row whose resolved id differs from the request", () => { + const bad = row({ resolvedModelId: "kimi-k2.6" }); + expect(identityViolations([row({ resolvedModelId: "claude-sonnet-4.6" }), bad])).toEqual([bad]); + }); + + it("disqualifies a row whose own retry switched models", () => { + const bad = row({ resolvedModelId: "claude-sonnet-4.6", fallbackUsed: true }); + expect(identityViolations([bad])).toEqual([bad]); + }); +}); diff --git a/scripts/__tests__/toolGenRequestShape.test.ts b/scripts/__tests__/toolGenRequestShape.test.ts new file mode 100644 index 000000000..15bd3a402 --- /dev/null +++ b/scripts/__tests__/toolGenRequestShape.test.ts @@ -0,0 +1,67 @@ +// @vitest-environment node +/** + * Request-shape test for the daily tool-gen job's raw-fetch Anthropic call + * (SPEC-v9 A.0.6, A.2.10 PR 1). The job runs `analyze-gap.mjs` on a schedule with + * a real key, so what it sends is what gets billed: no sampling keys (Claude + * 4.7+/5.x answer 400), an output cap that leaves room for adaptive thinking, and + * the answer read from the text block, because content[0] is a thinking block + * whenever thinking is on. Fake transport; no network, no spend. + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +// @ts-expect-error - plain .mjs script without types +import { buildClaudeRequest, callClaude, claudeText } from "../tool-gen/analyze-gap.mjs"; + +describe("analyze-gap.mjs Anthropic request", () => { + beforeEach(() => { + vi.stubEnv("ANTHROPIC_API_KEY", "test-key-not-real"); + }); + afterEach(() => { + vi.unstubAllGlobals(); + vi.unstubAllEnvs(); + }); + + it("builds a body with no sampling keys, no prefill and room to think", () => { + const body = buildClaudeRequest("system text", "user text"); + // PR 1 moves no ids; PR 2 changes this literal to claude-haiku-5-5 with its own update. + expect(body.model).toBe("claude-haiku-4-5-20251001"); + for (const key of ["temperature", "top_p", "top_k"]) expect(body).not.toHaveProperty(key); + expect(body.max_tokens).toBeGreaterThanOrEqual(1024); + expect(body.system).toBe("system text"); + expect(body.messages).toEqual([{ role: "user", content: "user text" }]); + }); + + it("sends that body over the transport and returns the text block, not content[0]", async () => { + const seen: Array<{ url: string; headers: Record; body: any }> = []; + vi.stubGlobal( + "fetch", + vi.fn(async (url: string, init: { headers: Record; body: string }) => { + seen.push({ url, headers: init.headers, body: JSON.parse(init.body) }); + return new Response( + JSON.stringify({ + content: [ + { type: "thinking", thinking: "weighing the signals", signature: "sig" }, + { type: "text", text: '{"shouldGenerate":false,"reason":"none"}' }, + ], + }), + { status: 200 }, + ); + }), + ); + const text = await callClaude("sys", "usr"); + expect(text).toBe('{"shouldGenerate":false,"reason":"none"}'); + expect(seen).toHaveLength(1); + expect(seen[0].url).toBe("https://api.anthropic.com/v1/messages"); + expect(seen[0].headers["x-api-key"]).toBe("test-key-not-real"); + expect(seen[0].body).toEqual(buildClaudeRequest("sys", "usr")); + }); + + it("claudeText tolerates a response with no text block", () => { + expect(claudeText({ content: [{ type: "thinking", thinking: "x" }] })).toBe(""); + expect(claudeText({})).toBe(""); + }); + + // Importing the module at the top of this file is itself the guard test: an unguarded + // main() would read /tmp/daily-brief.json and call process.exit(1) inside the worker. +}); diff --git a/scripts/eval-harness/agentInvoker.ts b/scripts/eval-harness/agentInvoker.ts index 336d80192..43e0fe8bb 100644 --- a/scripts/eval-harness/agentInvoker.ts +++ b/scripts/eval-harness/agentInvoker.ts @@ -13,6 +13,7 @@ * EVAL_MODEL — model override (default: per-backend) */ +import { createHash } from "node:crypto"; import type { RunConfig, RunTelemetry, Scorecard, ToolCallRecord, EvalTask } from "./types.js"; export type InvokerBackend = "stub" | "anthropic" | "openai"; @@ -22,6 +23,15 @@ interface InvokerOptions { apiKey?: string; model?: string; verbose?: boolean; + /** Transport for the Anthropic backend; tests pass a fake, runs use global fetch. */ + fetchImpl?: typeof fetch; +} + +/** BOUND_READ: a Messages response larger than this is treated as a failed run. */ +const MAX_RESPONSE_CHARS = 4_000_000; + +function sha256(value: string): string { + return createHash("sha256").update(value).digest("hex"); } function getDefaultModel(backend: InvokerBackend): string { @@ -74,7 +84,40 @@ async function invokeStub(config: RunConfig, _task: EvalTask): Promise { + const tools = config.agentMode !== "bare" ? buildMcpToolDefs(config) : []; + return { + model, + max_tokens: 8192, + system: buildSystemPrompt(config, task), + messages: [{ role: "user", content: task.description }], + ...(tools.length > 0 + ? { + tools: tools.map((t) => ({ ...t, input_schema: { type: "object", properties: {} } })), + tool_choice: { type: "auto" }, + } + : {}), + }; +} async function invokeAnthropic( config: RunConfig, @@ -88,29 +131,65 @@ async function invokeAnthropic( const startedAt = new Date().toISOString(); const startMs = Date.now(); const toolCalls: ToolCallRecord[] = []; - - // Build system prompt based on agent mode - const systemPrompt = buildSystemPrompt(config, task); - - // Build tool definitions for MCP modes - const tools = config.agentMode !== "bare" ? buildMcpToolDefs(config) : []; - - // TODO: Replace with real Anthropic API call - // const response = await fetch("https://api.anthropic.com/v1/messages", { - // method: "POST", - // headers: { - // "Content-Type": "application/json", - // "x-api-key": options.apiKey, - // "anthropic-version": "2023-06-01", - // }, - // body: JSON.stringify({ - // model: options.model, - // max_tokens: 8192, - // system: systemPrompt, - // messages: [{ role: "user", content: task.description }], - // tools: tools.length > 0 ? tools : undefined, - // }), - // }); + let error: string | undefined; + let outputHash = ""; + let tokenCount = 0; + let echoedModel: string | undefined; + + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), config.timeout); + try { + const response = await (options.fetchImpl ?? fetch)("https://api.anthropic.com/v1/messages", { + method: "POST", + headers: { + "Content-Type": "application/json", + "x-api-key": options.apiKey, + "anthropic-version": "2023-06-01", + }, + body: JSON.stringify(buildAnthropicRequest(config, task, options.model ?? config.model)), + signal: controller.signal, + }); + const raw = await response.text(); + if (!response.ok) { + error = `Anthropic HTTP ${response.status}: ${raw.slice(0, 240)}`; + } else if (raw.length > MAX_RESPONSE_CHARS) { + error = `Anthropic response exceeded ${MAX_RESPONSE_CHARS} chars`; + } else { + const data = JSON.parse(raw) as { + model?: string; + stop_reason?: string; + content?: AnthropicBlock[]; + usage?: { input_tokens?: number; output_tokens?: number }; + }; + // Read blocks by type: content[0] is a thinking block when adaptive thinking is on. + const blocks = Array.isArray(data.content) ? data.content : []; + const text = blocks + .filter((b) => b.type === "text" && typeof b.text === "string") + .map((b) => b.text as string) + .join(""); + for (const block of blocks) { + if (block.type !== "tool_use" || !block.name) continue; + toolCalls.push({ + toolName: block.name, + durationMs: 0, + success: true, + inputHash: sha256(JSON.stringify(block.input ?? null)), + outputHash: "", + followedQuickRef: null, + timestamp: new Date().toISOString(), + }); + } + outputHash = sha256(text); + tokenCount = (data.usage?.input_tokens ?? 0) + (data.usage?.output_tokens ?? 0); + // The model string the provider returned, not the one requested (SPEC-v9 A.0.3). + echoedModel = data.model; + if (data.stop_reason === "refusal") error = "Anthropic stop_reason: refusal"; + } + } catch (err) { + error = err instanceof Error ? (err.name === "AbortError" ? "request timeout" : err.message) : String(err); + } finally { + clearTimeout(timer); + } const wallClockMs = Date.now() - startMs; const completedAt = new Date().toISOString(); @@ -125,7 +204,7 @@ async function invokeAnthropic( efficiency: { wallClockMs, toolCallCount: toolCalls.length, - tokenCount: 0, + tokenCount, retryThrashRate: 0, }, compounding: { @@ -142,8 +221,11 @@ async function invokeAnthropic( scorecard, toolCalls, verificationCycles: [], - outputHash: "", - error: "SCAFFOLD: Anthropic API call not yet wired — implement real invocation", + outputHash, + echoedModel, + // taskSuccessRate stays 0: acceptance criteria are not graded here, so a + // completed call is recorded without claiming the task passed. + ...(error ? { error } : {}), }; } diff --git a/scripts/eval-harness/evalHarness.test.ts b/scripts/eval-harness/evalHarness.test.ts new file mode 100644 index 000000000..27006e7ef --- /dev/null +++ b/scripts/eval-harness/evalHarness.test.ts @@ -0,0 +1,189 @@ +// @vitest-environment node +/** + * Eval harness wiring (SPEC-v9 A.2.10 PR 1): the harness must be able to name the + * candidate it measures, because a migration receipt that cannot name its candidate + * is rejected (A.0.3). + * + * Persona: an engineer deciding whether to move the Claude agent lane to a new model. + * They run `EVAL_MODEL= runner.ts --tier tier1 --seeds N`. Failure modes: + * the runner measures a model other than the one named (the old hard-coded + * claude-sonnet-4-6), the Messages call is still a scaffold that sends nothing, or + * a request carries a parameter the new models answer 400 to. No network, no spend: + * the stub backend is $0 and the Anthropic backend gets a fake transport. + */ + +import { mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { EVAL_DATASET } from "./dataset"; +import { buildAnthropicRequest, invokeAgent } from "./agentInvoker"; +import { main } from "./runner"; +import type { EvalTask, RunConfig, RunTelemetry } from "./types"; + +const task: EvalTask = EVAL_DATASET.tasks.find((t) => t.tier === "tier1_deterministic")!; +const config = (agentMode: RunConfig["agentMode"], model = "candidate-model"): RunConfig => ({ + taskId: task.id, + agentMode, + model, + seed: 1, + timeout: 5_000, +}); + +function fakeMessagesTransport(responseBody: unknown, status = 200) { + const seen: Array<{ url: string; body: Record }> = []; + const fetchImpl = (async (url: string, init: { body: string }) => { + seen.push({ url: String(url), body: JSON.parse(init.body) }); + return new Response(JSON.stringify(responseBody), { status }); + }) as unknown as typeof fetch; + return { seen, fetchImpl }; +} + +describe("agentInvoker Anthropic backend", () => { + it("sends the named candidate with no sampling keys and tool_choice auto", async () => { + const { seen, fetchImpl } = fakeMessagesTransport({ + model: "candidate-model-echo", + stop_reason: "end_turn", + content: [{ type: "text", text: "done" }], + usage: { input_tokens: 100, output_tokens: 50 }, + }); + const telemetry = await invokeAgent(config("mcp_core"), task, { + backend: "anthropic", + apiKey: "test-key-not-real", + model: "candidate-model", + fetchImpl, + }); + expect(seen).toHaveLength(1); + const { url, body } = seen[0]; + expect(url).toBe("https://api.anthropic.com/v1/messages"); + expect(body.model).toBe("candidate-model"); + for (const key of ["temperature", "top_p", "top_k"]) expect(body, `${key} must be absent`).not.toHaveProperty(key); + expect(body.max_tokens).toBe(8192); + expect(body.tool_choice).toEqual({ type: "auto" }); + expect(body.tools.length).toBeGreaterThan(0); + for (const tool of body.tools) expect(tool.input_schema).toMatchObject({ type: "object" }); + expect(body.messages).toEqual([{ role: "user", content: task.description }]); + // the model string the provider returned is recorded next to the requested one + expect(telemetry.echoedModel).toBe("candidate-model-echo"); + expect(telemetry.config.model).toBe("candidate-model"); + expect(telemetry.error).toBeUndefined(); + expect(telemetry.scorecard.efficiency.tokenCount).toBe(150); + // a completed call is not a passed task: acceptance criteria are not graded here + expect(telemetry.scorecard.correctness.taskSuccessRate).toBe(0); + }); + + it("bare mode offers no tools and therefore no tool_choice", () => { + const body = buildAnthropicRequest(config("bare"), task, "candidate-model"); + expect(body).not.toHaveProperty("tools"); + expect(body).not.toHaveProperty("tool_choice"); + }); + + it("reads blocks by type: text after a thinking block, tool_use recorded as a tool call", async () => { + const { fetchImpl } = fakeMessagesTransport({ + model: "m", + content: [ + { type: "thinking", thinking: "…", signature: "s" }, + { type: "text", text: "plan" }, + { type: "tool_use", id: "t1", name: "convex_audit_schema", input: { path: "schema.ts" } }, + ], + usage: { input_tokens: 1, output_tokens: 1 }, + }); + const telemetry = await invokeAgent(config("mcp_lite"), task, { + backend: "anthropic", + apiKey: "k", + model: "candidate-model", + fetchImpl, + }); + expect(telemetry.toolCalls.map((c) => c.toolName)).toEqual(["convex_audit_schema"]); + expect(telemetry.outputHash).toMatch(/^[0-9a-f]{64}$/); + }); + + it("reports an HTTP failure and a refusal as errors, never as a clean run", async () => { + const failed = await invokeAgent(config("bare"), task, { + backend: "anthropic", + apiKey: "k", + model: "candidate-model", + fetchImpl: fakeMessagesTransport({ error: { type: "invalid_request_error" } }, 400).fetchImpl, + }); + expect(failed.error).toMatch(/HTTP 400/); + expect(failed.echoedModel).toBeUndefined(); + + const refused = await invokeAgent(config("bare"), task, { + backend: "anthropic", + apiKey: "k", + model: "candidate-model", + fetchImpl: fakeMessagesTransport({ model: "m", stop_reason: "refusal", content: [] }).fetchImpl, + }); + expect(refused.error).toMatch(/refusal/); + }); + + it("refuses to run without a key", async () => { + await expect(invokeAgent(config("bare"), task, { backend: "anthropic", model: "m" })).rejects.toThrow(/ANTHROPIC_API_KEY/); + }); + + it("stays a no-op stub by default", async () => { + const telemetry = await invokeAgent(config("bare"), task, { backend: "stub" }); + expect(telemetry.error).toMatch(/^STUB/); + }); +}); + +describe("runner --tier / --seeds", () => { + let dir: string; + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "eval-harness-")); + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "error").mockImplementation(() => {}); + }); + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + }); + + const readAll = () => + readdirSync(dir) + .filter((f) => f.endsWith(".json")) + .map((f) => ({ file: f, telemetry: JSON.parse(readFileSync(join(dir, f), "utf8")) as RunTelemetry })); + + it("writes two telemetry files per tier1 task for --seeds 2, each naming EVAL_MODEL", async () => { + vi.stubEnv("EVAL_BACKEND", "stub"); + vi.stubEnv("EVAL_MODEL", "candidate-under-test"); + const tier1 = EVAL_DATASET.tasks.filter((t) => t.tier.startsWith("tier1")); + expect(tier1.length).toBeGreaterThan(1); + + expect(await main(["--tier", "tier1", "--seeds", "2"], dir)).toBe(0); + + const rows = readAll(); + expect(rows).toHaveLength(tier1.length * 2); + for (const t of tier1) { + expect(rows.filter((r) => r.telemetry.config.taskId === t.id).map((r) => r.telemetry.config.seed).sort()).toEqual([1, 2]); + } + for (const { telemetry } of rows) { + expect(telemetry.config.model).toBe(process.env.EVAL_MODEL); + expect(telemetry.config).not.toHaveProperty("modelVersion"); + } + }); + + it("keeps the singular --seed for one-off runs", async () => { + vi.stubEnv("EVAL_BACKEND", "stub"); + vi.stubEnv("EVAL_MODEL", "candidate-under-test"); + expect(await main(["--task", task.id, "--mode", "bare", "--seed", "3"], dir)).toBe(0); + const rows = readAll(); + expect(rows.map((r) => r.file)).toEqual([`${task.id}_bare_s3.json`]); + }); + + it("requires EVAL_MODEL for a real backend, so a receipt always names its candidate", async () => { + vi.stubEnv("EVAL_BACKEND", "anthropic"); + vi.stubEnv("EVAL_MODEL", ""); + expect(await main(["--tier", "tier1", "--seeds", "1"], dir)).toBe(1); + expect(readAll()).toHaveLength(0); + }); + + it("rejects an empty tier selection and a bad --seeds", async () => { + vi.stubEnv("EVAL_BACKEND", "stub"); + expect(await main(["--tier", "tier9", "--seeds", "1"], dir)).toBe(1); + expect(await main(["--tier", "tier1", "--seeds", "0"], dir)).toBe(1); + expect(readAll()).toHaveLength(0); + }); +}); diff --git a/scripts/eval-harness/runner.ts b/scripts/eval-harness/runner.ts index add207e61..e4fe21315 100644 --- a/scripts/eval-harness/runner.ts +++ b/scripts/eval-harness/runner.ts @@ -5,15 +5,21 @@ * Usage: * npx tsx scripts/eval-harness/runner.ts --task t1_add_validator_returns --mode bare * npx tsx scripts/eval-harness/runner.ts --task t1_add_validator_returns --mode mcp_core - * npx tsx scripts/eval-harness/runner.ts --all --seeds 3 + * npx tsx scripts/eval-harness/runner.ts --tier tier1 --seeds 3 * npx tsx scripts/eval-harness/runner.ts --compare t1_add_validator_returns * - * Outputs: scripts/eval-harness/results/__.json + * The backend comes from EVAL_BACKEND (default "stub", $0). A real backend + * requires EVAL_MODEL, so every run names its candidate: + * EVAL_BACKEND=anthropic EVAL_MODEL= npx tsx scripts/eval-harness/runner.ts --tier tier1 --seeds 3 + * + * Outputs: scripts/eval-harness/results/__s.json */ -import { writeFileSync, mkdirSync, existsSync, readFileSync, readdirSync } from "node:fs"; +import { writeFileSync, mkdirSync, readFileSync, readdirSync } from "node:fs"; import { join } from "node:path"; +import { pathToFileURL } from "node:url"; import { EVAL_DATASET, getTaskById } from "./dataset.js"; +import { detectBackend, invokeAgent } from "./agentInvoker.js"; import type { RunConfig, RunTelemetry, @@ -25,8 +31,8 @@ import type { const RESULTS_DIR = join(import.meta.dirname || __dirname, "results"); -function ensureResultsDir() { - mkdirSync(RESULTS_DIR, { recursive: true }); +function ensureResultsDir(resultsDir: string) { + mkdirSync(resultsDir, { recursive: true }); } // ── Scorecard Computation ─────────────────────────────────────────── @@ -65,65 +71,24 @@ function computeCompositeScore(scorecard: Scorecard): number { ); } -// ── Stub runner (replace with real agent invocation) ───────────────── - -function createStubTelemetry(config: RunConfig, task: EvalTask): RunTelemetry { - const now = new Date().toISOString(); - const isMcp = config.agentMode !== "bare"; - - // Stub scorecard — in production, this is computed from actual agent traces - const scorecard: Scorecard = { - correctness: { - taskSuccessRate: 0, // filled by real run - regressionRate: 0, - }, - safety: { - highRiskActionsGated: task.riskTier === "low" ? 1 : 0, - issuesCaughtPreMerge: 0, - }, - efficiency: { - wallClockMs: 0, - toolCallCount: 0, - tokenCount: 0, - retryThrashRate: 0, - }, - compounding: { - knowledgeReuseRate: isMcp ? 0 : 0, - evalCasesBanked: 0, - }, - }; - - return { - runId: `run_${config.taskId}_${config.agentMode}_s${config.seed}_${Date.now()}`, - config, - startedAt: now, - completedAt: now, - scorecard, - toolCalls: [], - verificationCycles: [], - outputHash: "", - error: "STUB: Replace with real agent invocation", - }; -} - // ── Comparison Engine ─────────────────────────────────────────────── -function loadResults(taskId: string): RunTelemetry[] { - ensureResultsDir(); - const files = readdirSync(RESULTS_DIR).filter( +function loadResults(taskId: string, resultsDir: string): RunTelemetry[] { + ensureResultsDir(resultsDir); + const files = readdirSync(resultsDir).filter( (f) => f.startsWith(taskId) && f.endsWith(".json") ); return files.map((f) => { - const content = readFileSync(join(RESULTS_DIR, f), "utf-8"); + const content = readFileSync(join(resultsDir, f), "utf-8"); return JSON.parse(content) as RunTelemetry; }); } -function generateComparison(taskId: string): ComparisonReport | null { +function generateComparison(taskId: string, resultsDir: string): ComparisonReport | null { const task = getTaskById(taskId); if (!task) return null; - const allResults = loadResults(taskId); + const allResults = loadResults(taskId, resultsDir); if (allResults.length === 0) return null; const modes = [...new Set(allResults.map((r) => r.config.agentMode))]; @@ -193,16 +158,40 @@ function generateComparison(taskId: string): ComparisonReport | null { // ── CLI ───────────────────────────────────────────────────────────── -function main() { - const args = process.argv.slice(2); - ensureResultsDir(); +async function runOne( + task: EvalTask, + mode: AgentMode, + seed: number, + model: string, + resultsDir: string, +): Promise { + const config: RunConfig = { + taskId: task.id, + agentMode: mode, + model, + seed, + timeout: 300_000, + }; + + console.log(`Running: ${task.name} (${mode}, seed=${seed}, model=${model})`); + const telemetry = await invokeAgent(config, task); + + const outPath = join(resultsDir, `${task.id}_${mode}_s${seed}.json`); + writeFileSync(outPath, JSON.stringify(telemetry, null, 2)); + console.log(`Result written to: ${outPath}`); + console.log(`Composite score: ${computeCompositeScore(telemetry.scorecard).toFixed(3)}`); +} + +/** Returns the process exit code. `resultsDir` is a parameter so tests can use a temp dir. */ +export async function main(args: string[], resultsDir: string = RESULTS_DIR): Promise { + ensureResultsDir(resultsDir); if (args.includes("--list")) { console.log("Available eval tasks:"); for (const task of EVAL_DATASET.tasks) { console.log(` ${task.id} [${task.tier}] ${task.name}`); } - return; + return 0; } if (args.includes("--compare")) { @@ -210,58 +199,93 @@ function main() { const taskId = args[taskIdIdx]; if (!taskId) { console.error("Usage: --compare "); - process.exit(1); + return 1; } - const report = generateComparison(taskId); + const report = generateComparison(taskId, resultsDir); if (!report) { console.error(`No results found for task ${taskId}`); - process.exit(1); + return 1; } console.log(JSON.stringify(report, null, 2)); - return; + return 0; } - // Single task run const taskIdx = args.indexOf("--task"); + const tierIdx = args.indexOf("--tier"); const modeIdx = args.indexOf("--mode"); const seedIdx = args.indexOf("--seed"); + const seedsIdx = args.indexOf("--seeds"); - if (taskIdx === -1) { + if (taskIdx === -1 && tierIdx === -1) { console.log("Usage:"); console.log(" --list List all tasks"); console.log(" --task --mode Run a single task"); + console.log(" --tier --seeds Run every task whose tier starts with , seeds 1..N"); console.log(" --compare Compare results for a task"); console.log(""); console.log("Modes: bare, mcp_lite, mcp_core, mcp_full"); - return; + console.log("Backend: EVAL_BACKEND=stub|anthropic|openai (default stub); EVAL_MODEL is required for a real backend"); + return 0; } - const taskId = args[taskIdx + 1]; - const mode = (args[modeIdx + 1] || "bare") as AgentMode; - const seed = seedIdx !== -1 ? parseInt(args[seedIdx + 1]) : 1; - - const task = getTaskById(taskId); - if (!task) { - console.error(`Unknown task: ${taskId}`); - process.exit(1); + // The candidate id comes from the environment, never from this file, so a + // receipt can only name the model that was actually requested. + const backend = detectBackend(); + if (backend.backend !== "stub" && !process.env.EVAL_MODEL) { + console.error(`EVAL_MODEL is required when EVAL_BACKEND=${backend.backend}`); + return 1; + } + const model = backend.model ?? "stub-model"; + const mode = (modeIdx !== -1 ? args[modeIdx + 1] : "bare") as AgentMode; + + let tasks: EvalTask[]; + if (tierIdx !== -1) { + const prefix = args[tierIdx + 1]; + if (!prefix) { + console.error("Usage: --tier "); + return 1; + } + tasks = EVAL_DATASET.tasks.filter((t) => t.tier.startsWith(prefix)); + if (tasks.length === 0) { + console.error(`No tasks with a tier starting with "${prefix}"`); + return 1; + } + } else { + const task = getTaskById(args[taskIdx + 1]); + if (!task) { + console.error(`Unknown task: ${args[taskIdx + 1]}`); + return 1; + } + tasks = [task]; } - const config: RunConfig = { - taskId, - agentMode: mode, - model: "claude-sonnet-4-6", - modelVersion: "4.6", - seed, - timeout: 300_000, - }; - - console.log(`Running: ${task.name} (${mode}, seed=${seed})`); - const telemetry = createStubTelemetry(config, task); + let seeds: number[]; + if (seedsIdx !== -1) { + const count = parseInt(args[seedsIdx + 1], 10); + if (!Number.isInteger(count) || count < 1) { + console.error("Usage: --seeds with N >= 1"); + return 1; + } + seeds = Array.from({ length: count }, (_, i) => i + 1); + } else { + seeds = [seedIdx !== -1 ? parseInt(args[seedIdx + 1], 10) : 1]; + } - const outPath = join(RESULTS_DIR, `${taskId}_${mode}_s${seed}.json`); - writeFileSync(outPath, JSON.stringify(telemetry, null, 2)); - console.log(`Result written to: ${outPath}`); - console.log(`Composite score: ${computeCompositeScore(telemetry.scorecard).toFixed(3)}`); + for (const task of tasks) { + for (const seed of seeds) { + await runOne(task, mode, seed, model, resultsDir); + } + } + return 0; } -main(); +// Run only as a script, so tests can import `main` with their own results directory. +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main(process.argv.slice(2)).then( + (code) => process.exit(code), + (err) => { + console.error(err); + process.exit(1); + }, + ); +} diff --git a/scripts/eval-harness/types.ts b/scripts/eval-harness/types.ts index 520558a8f..90684adc1 100644 --- a/scripts/eval-harness/types.ts +++ b/scripts/eval-harness/types.ts @@ -65,8 +65,7 @@ export type AgentMode = "bare" | "mcp_lite" | "mcp_core" | "mcp_full"; export interface RunConfig { taskId: string; agentMode: AgentMode; - model: string; - modelVersion: string; + model: string; // candidate id: EVAL_MODEL (detectBackend().model), never hard-coded seed: number; // trial seed (run 3-5 per config) toolsetPreset?: string; // lite/core/full for MCP modes timeout: number; // max ms per run @@ -84,6 +83,8 @@ export interface RunTelemetry { verificationCycles: VerificationCycleRecord[]; /** Raw agent output for audit */ outputHash: string; + /** The `model` string the provider returned (not the one requested) - SPEC-v9 A.0.3. */ + echoedModel?: string; /** Error if run failed */ error?: string; } diff --git a/scripts/run-comprehensive-eval.ts b/scripts/run-comprehensive-eval.ts index 6d1c86ec6..c7daa4927 100644 --- a/scripts/run-comprehensive-eval.ts +++ b/scripts/run-comprehensive-eval.ts @@ -21,8 +21,10 @@ import dotenv from "dotenv"; import { mkdirSync, writeFileSync, appendFileSync, readFileSync } from "node:fs"; import { spawnSync } from "node:child_process"; import { join } from "node:path"; +import { pathToFileURL } from "node:url"; import { ConvexHttpClient } from "convex/browser"; import { api } from "../backend/convex/_generated/api"; +import { resolveModelAlias } from "../backend/convex/domains/agents/mcp_tools/models/modelResolver"; // ═══════════════════════════════════════════════════════════════════════════ // TYPES @@ -63,6 +65,10 @@ interface EpisodeResult { responseText?: string; responsePreview?: string; error?: string; + /** What the repo's resolver chose for `model` (returned by runPersonaEpisodeEval). */ + resolvedModelId?: string; + /** True when this script's own retry switched to getEvalFallbackModel(). */ + fallbackUsed?: boolean; } type ScenarioSuite = "core" | "next" | "stress"; @@ -239,6 +245,21 @@ function percentile(arr: number[], p: number): number { return sorted[Math.min(idx, sorted.length - 1)]; } +/** `--models` values the resolver does not know. Each one must abort the run (SPEC-v9 A.0.10). */ +export function unresolvedModels(models: readonly string[]): string[] { + return models.filter((m) => resolveModelAlias(m) === null); +} + +/** + * Rows that cannot count as evidence for the requested candidate: the resolver chose + * a different model, or this script's retry switched models (SPEC-v9 A.0.3). + */ +export function identityViolations(results: readonly EpisodeResult[]): EpisodeResult[] { + return results.filter( + (r) => r.fallbackUsed === true || (r.resolvedModelId !== undefined && r.resolvedModelId !== r.model), + ); +} + function getEvalFallbackModel(model: string): string | null { switch (model) { case "kimi-k2.6": @@ -289,6 +310,7 @@ async function runSingleEpisode( try { const maxAttempts = 2; let run: any = null; + let resolvedModelId: string | undefined; let activeModel = model; let fallbackActivated = false; @@ -310,6 +332,7 @@ async function runSingleEpisode( }); run = result?.runs?.[0] ?? null; + resolvedModelId = typeof result?.resolvedModelId === "string" ? result.resolvedModelId : undefined; const responseForRetry = String(run?.responseText ?? run?.responsePreview ?? "").trim(); const failureReasons = Array.isArray(run?.failureReasons) ? run.failureReasons.map((reason: unknown) => String(reason)) : []; const missingDebrief = failureReasons.some((reason) => reason.includes("Missing [DEBRIEF_V1_JSON] block")); @@ -376,6 +399,8 @@ async function runSingleEpisode( responseText: responseForJudge, responsePreview: run?.responsePreview?.slice(0, 500), error: run ? undefined : "No run returned by personaEpisodeEval", + resolvedModelId, + fallbackUsed: fallbackActivated, }; } catch (err) { const elapsed = Date.now() - startTime; @@ -548,7 +573,7 @@ async function main() { "deepseek-r1", "deepseek-v3.2", "qwen3-235b", - "minimax-m2.1", + // minimax-m2.1 was removed: it is not an approved id, so it used to resolve to the default. ]; let modelsToTest: string[]; @@ -561,6 +586,13 @@ async function main() { modelsToTest = ["kimi-k2.6"]; } + // Fail closed: an unknown --models value used to resolve silently to the default + // model, so a "candidate" run could measure a different model. + const unknown = unresolvedModels(modelsToTest); + if (unknown.length > 0) { + throw new Error(`Unknown --models value(s): ${unknown.join(", ")} (not an approved model or alias)`); + } + // Select scenarios based on suite let allScenarios; if (suite === "full" || suite === "all") { @@ -788,10 +820,23 @@ async function main() { writeFileSync(metricsPath, JSON.stringify(metrics, null, 2) + "\n", "utf8"); console.log(` ${metricsPath} (success metrics)`); } + + const violations = identityViolations(results); + if (violations.length > 0) { + console.error( + `\nERROR: ${violations.length} row(s) are not evidence for the requested model ` + + `(resolved to another id, or the retry switched models): ` + + violations.map((r) => `${r.model}/${r.scenarioId}->${r.resolvedModelId ?? "?"}${r.fallbackUsed ? " (fallback)" : ""}`).join(", "), + ); + process.exitCode = 1; + } } -main().catch((err) => { - const msg = err instanceof Error ? (err.stack || err.message || String(err)) : String(err); - process.stderr.write(`ERROR: ${msg}\n`); - process.exit(1); -}); +// Run only as a script, so the pure helpers above can be imported by tests. +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((err) => { + const msg = err instanceof Error ? (err.stack || err.message || String(err)) : String(err); + process.stderr.write(`ERROR: ${msg}\n`); + process.exit(1); + }); +} diff --git a/scripts/tool-gen/analyze-gap.mjs b/scripts/tool-gen/analyze-gap.mjs index e91f33366..6fc27e9bb 100644 --- a/scripts/tool-gen/analyze-gap.mjs +++ b/scripts/tool-gen/analyze-gap.mjs @@ -13,6 +13,7 @@ import { readFileSync, writeFileSync, appendFileSync, readdirSync } from "node:fs"; import { join } from "node:path"; +import { pathToFileURL } from "node:url"; const GAP_SPEC_PATH = "/tmp/gap-spec.json"; const SUMMARY_PATH = "/tmp/gap-analysis-summary.txt"; @@ -42,7 +43,26 @@ function buildToolCatalog() { return catalog; } -async function callClaude(systemPrompt, userPrompt) { +/** The Messages API body. No sampling parameters: Claude 4.7+/5.x answer 400 to them. */ +export function buildClaudeRequest(systemPrompt, userPrompt) { + return { + model: "claude-haiku-4-5-20251001", + max_tokens: 2048, + system: systemPrompt, + messages: [{ role: "user", content: userPrompt }], + }; +} + +/** Concatenated text blocks. content[0] is a thinking block when adaptive thinking is on. */ +export function claudeText(data) { + const blocks = Array.isArray(data?.content) ? data.content : []; + return blocks + .filter((b) => b?.type === "text" && typeof b.text === "string") + .map((b) => b.text) + .join(""); +} + +export async function callClaude(systemPrompt, userPrompt) { const apiKey = process.env.ANTHROPIC_API_KEY; if (!apiKey) throw new Error("Missing ANTHROPIC_API_KEY"); @@ -53,12 +73,7 @@ async function callClaude(systemPrompt, userPrompt) { "anthropic-version": "2023-06-01", "content-type": "application/json", }, - body: JSON.stringify({ - model: "claude-haiku-4-5-20251001", - max_tokens: 2048, - system: systemPrompt, - messages: [{ role: "user", content: userPrompt }], - }), + body: JSON.stringify(buildClaudeRequest(systemPrompt, userPrompt)), }); if (!res.ok) { @@ -66,8 +81,7 @@ async function callClaude(systemPrompt, userPrompt) { throw new Error(`Claude API ${res.status}: ${text}`); } - const data = await res.json(); - return data.content[0].text; + return claudeText(await res.json()); } async function main() { @@ -170,7 +184,10 @@ Identify ONE tool gap. Return JSON: setOutput("gap_spec_path", GAP_SPEC_PATH); } -main().catch((err) => { - console.error(err); - process.exit(1); -}); +// Run only as a script, so the request builder can be imported by tests. +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((err) => { + console.error(err); + process.exit(1); + }); +} diff --git a/scripts/tool-gen/generate-tool.mjs b/scripts/tool-gen/generate-tool.mjs index 63b3a7599..89e8f3fe2 100644 --- a/scripts/tool-gen/generate-tool.mjs +++ b/scripts/tool-gen/generate-tool.mjs @@ -75,7 +75,11 @@ async function callClaude(systemPrompt, userPrompt) { } const data = await res.json(); - return data.content[0].text; + // Read blocks by type: content[0] is a thinking block when adaptive thinking is on. + return (Array.isArray(data.content) ? data.content : []) + .filter((b) => b?.type === "text" && typeof b.text === "string") + .map((b) => b.text) + .join(""); } async function main() { diff --git a/scripts/ui/judgeScratchnodeChatVideo.mjs b/scripts/ui/judgeScratchnodeChatVideo.mjs index 718aa3c3c..0901e3d84 100644 --- a/scripts/ui/judgeScratchnodeChatVideo.mjs +++ b/scripts/ui/judgeScratchnodeChatVideo.mjs @@ -73,7 +73,7 @@ Return ONLY JSON: const result = await ai.models.generateContent({ model: MODEL, contents: [{ role: "user", parts: [{ fileData: { fileUri: file.uri, mimeType: "video/webm" } }, { text: PROMPT }] }], - config: { responseMimeType: "application/json", temperature: 0.2, maxOutputTokens: 3500, thinkingConfig: { thinkingBudget: 0 } }, + config: { responseMimeType: "application/json", temperature: 0.2, maxOutputTokens: 3500, thinkingConfig: { thinkingLevel: "low" } }, }); const text = (result.text || "").trim(); let parsed; diff --git a/shared/llm/anthropicWireIds.test.ts b/shared/llm/anthropicWireIds.test.ts new file mode 100644 index 000000000..031765625 --- /dev/null +++ b/shared/llm/anthropicWireIds.test.ts @@ -0,0 +1,123 @@ +// @vitest-environment node +/** + * The ~14 generateWithProvider copies (dealFlow.ts, morningDigest.ts, ...) call + * `getModelWithFailover(resolveModelAlias(x))` and pass `.model` straight to + * `anthropic.messages.create({ model })`. Before this change that string was the + * dotted catalog alias ("claude-sonnet-4.6"), which api.anthropic.com does not + * accept, so every anthropic branch 404'd. The fix lives once, in the catalog + * (SPEC-v9 A.0.5: an explicit table, not string surgery), so these tests drive + * the function every copy routes through. + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { + ANTHROPIC_WIRE_IDS, + anthropicText, + anthropicWireId, + getModelWithFailover, + llmModelCatalog, + resolveModelAlias, +} from "./modelCatalog"; +import { MODEL_SPECS } from "../../backend/convex/domains/agents/mcp_tools/models/modelResolver"; + +describe("ANTHROPIC_WIRE_IDS agrees with the resolver's MODEL_SPECS", () => { + const anthropicSpecs = Object.values(MODEL_SPECS).filter((spec) => spec.provider === "anthropic"); + + it("covers every anthropic spec with the same wire id", () => { + expect(anthropicSpecs.length).toBeGreaterThan(0); + for (const spec of anthropicSpecs) { + expect(ANTHROPIC_WIRE_IDS[spec.alias], `${spec.alias} missing from ANTHROPIC_WIRE_IDS`).toBe(spec.sdkId); + } + expect(Object.keys(ANTHROPIC_WIRE_IDS).sort()).toEqual(anthropicSpecs.map((s) => s.alias).sort()); + }); + + it("never contains a dotted wire id", () => { + for (const wire of Object.values(ANTHROPIC_WIRE_IDS)) expect(wire).not.toContain("."); + }); +}); + +describe("anthropicWireId", () => { + it("maps catalog aliases and short aliases to wire ids", () => { + expect(anthropicWireId("claude-sonnet-4.6")).toBe("claude-sonnet-4-6"); + expect(anthropicWireId("claude-haiku-4.5")).toBe("claude-haiku-4-5"); + expect(anthropicWireId("claude-opus-4.7")).toBe("claude-opus-4-7"); + expect(anthropicWireId("claude")).toBe("claude-sonnet-4-6"); + expect(anthropicWireId("Haiku")).toBe("claude-haiku-4-5"); + }); + + it("returns a string that already is a wire id unchanged, and null for anything else", () => { + expect(anthropicWireId("claude-sonnet-4-6")).toBe("claude-sonnet-4-6"); + expect(anthropicWireId("claude-haiku-4-5-20251001")).toBeNull(); + expect(anthropicWireId("gpt-5.4")).toBeNull(); + expect(anthropicWireId("")).toBeNull(); + }); +}); + +describe("getModelWithFailover sends wire ids to Anthropic", () => { + beforeEach(() => { + vi.spyOn(console, "warn").mockImplementation(() => {}); + vi.spyOn(console, "error").mockImplementation(() => {}); + }); + afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + }); + + it("every anthropic model in the catalog resolves to a wire id with no dot", () => { + vi.stubEnv("ANTHROPIC_API_KEY", "sk-ant-test-key-not-real"); + const aliases = new Set(Object.values(llmModelCatalog.anthropic).flat()); + expect(aliases.size).toBeGreaterThan(0); + for (const alias of aliases) { + const { model, provider } = getModelWithFailover(resolveModelAlias(alias)); + expect(provider).toBe("anthropic"); + expect(model, alias).toBe(ANTHROPIC_WIRE_IDS[alias]); + expect(model).not.toContain("."); + } + }); + + it("converts an equivalent chosen by provider failover too", () => { + // OpenAI and Gemini keys absent: gpt-5.4 fails over to its Anthropic equivalent. + vi.stubEnv("OPENAI_API_KEY", ""); + vi.stubEnv("GEMINI_API_KEY", ""); + vi.stubEnv("OPENROUTER_API_KEY", ""); + vi.stubEnv("ANTHROPIC_API_KEY", "sk-ant-test-key-not-real"); + expect(getModelWithFailover("gpt-5.4")).toMatchObject({ + model: "claude-sonnet-4-6", + provider: "anthropic", + isFallback: true, + }); + }); + + it("passes an operator-supplied claude string through untouched (env override precedence)", () => { + vi.stubEnv("ANTHROPIC_API_KEY", "sk-ant-test-key-not-real"); + expect(getModelWithFailover("claude-haiku-4-5-20251001").model).toBe("claude-haiku-4-5-20251001"); + }); + + it("leaves non-Anthropic providers alone", () => { + vi.stubEnv("GEMINI_API_KEY", "gemini-test-key-not-real"); + expect(getModelWithFailover("gemini-3-flash-preview")).toMatchObject({ + model: "gemini-3-flash-preview", + provider: "gemini", + }); + }); +}); + +describe("anthropicText reads content blocks by type", () => { + it("skips a leading thinking block (content[0] is not the answer)", () => { + expect( + anthropicText([ + { type: "thinking" }, + { type: "text", text: "hello " }, + { type: "tool_use" }, + { type: "text", text: "world" }, + ]), + ).toBe("hello world"); + }); + + it("returns an empty string when there is no text", () => { + expect(anthropicText([{ type: "thinking" }])).toBe(""); + expect(anthropicText(undefined)).toBe(""); + expect(anthropicText(null)).toBe(""); + }); +}); diff --git a/shared/llm/modelCatalog.ts b/shared/llm/modelCatalog.ts index 9ed6e8a35..13f25df6a 100644 --- a/shared/llm/modelCatalog.ts +++ b/shared/llm/modelCatalog.ts @@ -128,6 +128,8 @@ export const modelPricing: Record = { "glm-4.5-air-free": { inputPer1M: 0.00, outputPer1M: 0.00, contextWindow: 131072 }, "deepseek-chimera-free": { inputPer1M: 0.00, outputPer1M: 0.00, contextWindow: 163840 }, "venice-dolphin-free": { inputPer1M: 0.00, outputPer1M: 0.00, contextWindow: 32768 }, + // FREE_LANE_MODEL (modelResolver.ts); wire id nvidia/nemotron-3-super-120b-a12b:free, 262,144-token context (SPEC-v9 A.2.10 PR 1) + "nemotron-3-super-free": { inputPer1M: 0.00, outputPer1M: 0.00, contextWindow: 262144 }, }; // ═══════════════════════════════════════════════════════════════════════════ @@ -562,6 +564,54 @@ export function resolveModelAlias(modelInput: string): string { return modelAliases[normalized] ?? modelInput; } +// ----------------------------------------------------------------------------- +// ANTHROPIC WIRE IDS - explicit table, never string surgery +// ----------------------------------------------------------------------------- + +/** + * Catalog alias (dotted, UI-facing) -> the id api.anthropic.com accepts. + * The dotted alias is not a valid wire id, so every caller that hands a catalog + * model to Anthropic must go through `anthropicWireId`. Kept in sync with the + * `provider: "anthropic"` rows of `MODEL_SPECS` in modelResolver.ts by + * `anthropicWireIds.test.ts` (Convex cannot import this table from there, and + * this file cannot import from backend/convex). + */ +export const ANTHROPIC_WIRE_IDS: Readonly> = { + "claude-opus-4.7": "claude-opus-4-7", + "claude-sonnet-4.6": "claude-sonnet-4-6", + "claude-haiku-4.5": "claude-haiku-4-5", + "claude-opus-4.1": "claude-opus-4-1-20250805", + "claude-opus-4": "claude-opus-4-20250514", + "claude-sonnet-4": "claude-sonnet-4-20250514", + "claude-haiku-3.5": "claude-3-5-haiku-20241022", +}; + +/** + * The Anthropic wire id for a catalog alias (or short alias such as "claude"). + * A string that already is a wire id is returned unchanged. Returns null when + * the input is neither, so callers decide whether to fail or pass it through. + */ +export function anthropicWireId(modelInput: string): string | null { + const resolved = resolveModelAlias(modelInput.trim()); + const fromAlias = ANTHROPIC_WIRE_IDS[resolved]; + if (fromAlias) return fromAlias; + return Object.values(ANTHROPIC_WIRE_IDS).includes(resolved) ? resolved : null; +} + +/** + * Concatenated text of an Anthropic Messages response. Reads blocks by `type`: + * `content[0]` is a thinking block whenever adaptive thinking is on (default + * on Claude Haiku 5.5), so index 0 is not "the answer". + */ +export function anthropicText( + content: ReadonlyArray<{ type: string; text?: string }> | null | undefined, +): string { + if (!Array.isArray(content)) return ""; + return content + .map((block) => (block?.type === "text" && typeof block.text === "string" ? block.text : "")) + .join(""); +} + // ═══════════════════════════════════════════════════════════════════════════ // PROVIDER INTEGRATION STATUS // ═══════════════════════════════════════════════════════════════════════════ @@ -807,9 +857,21 @@ export function getEquivalentModel(modelName: string, targetProvider: LlmProvide return defaults[targetProvider]; } +/** + * The string to put in a provider request. Callers of `getModelWithFailover` + * send `model` straight to the provider SDK, so for Anthropic it must be the + * wire id, not the dotted catalog alias (api.anthropic.com 404s on + * "claude-sonnet-4.6"). An unrecognised string is passed through unchanged: + * it may already be an operator-supplied wire id. + */ +function toProviderWireId(model: string, provider: LlmProvider): string { + return provider === "anthropic" ? (anthropicWireId(model) ?? model) : model; +} + /** * Get a working model with failover. * Tries the preferred model's provider first, then falls back to alternatives. + * The returned `model` is the provider wire id (see `toProviderWireId`). */ export function getModelWithFailover(preferredModel: string): { model: string; @@ -824,7 +886,7 @@ export function getModelWithFailover(preferredModel: string): { // Check if preferred provider is configured if (isProviderConfigured(provider)) { - return { model: preferredModel, provider, isFallback: false }; + return { model: toProviderWireId(preferredModel, provider), provider, isFallback: false }; } // Try fallback providers @@ -833,7 +895,11 @@ export function getModelWithFailover(preferredModel: string): { if (isProviderConfigured(fallbackProvider)) { const equivalentModel = getEquivalentModel(preferredModel, fallbackProvider); console.warn(`[getModelWithFailover] ${provider} not configured, falling back to ${fallbackProvider}: ${equivalentModel}`); - return { model: equivalentModel, provider: fallbackProvider, isFallback: true }; + return { + model: toProviderWireId(equivalentModel, fallbackProvider), + provider: fallbackProvider, + isFallback: true, + }; } } diff --git a/shared/search/triSearch.test.ts b/shared/search/triSearch.test.ts index 2f10014fd..670e47fd1 100644 --- a/shared/search/triSearch.test.ts +++ b/shared/search/triSearch.test.ts @@ -181,3 +181,27 @@ describe("condenseQuery — multi-turn query rewrite", () => { expect(promptText).not.toContain("turn 0"); // beyond the last 3 → excluded }); }); + +describe("rerankWithGemini - request shape (Gemini 3.x)", () => { + // SPEC-v9 s1.3: Gemini 3.x replaced thinkingBudget with thinkingLevel (`minimal` unsupported). + // thinkingBudget:0 is how this reranker used to keep a thinking model from spending its + // 256-token cap on a reasoning preamble; thinkingLevel "low" is the replacement. + it("sends thinkingLevel low and no thinkingBudget", async () => { + const bodies: any[] = []; + const spyFetch = (async (_url: string, init: { body: string }) => { + bodies.push(JSON.parse(init.body)); + return { + ok: true, + status: 200, + json: async () => ({ candidates: [{ content: { parts: [{ text: "[1,0]" }] } }] }), + } as unknown as Response; + }) as unknown as typeof fetch; + const fused = rrfFuse([leg("linkup", ["a", "b"])]); + const r = await rerankWithGemini("entity funding", fused, { topN: 2, apiKey: "test", fetchImpl: spyFetch }); + expect(r.rerankStatus).toBe("ok"); + expect(bodies).toHaveLength(1); + const thinking = bodies[0].generationConfig.thinkingConfig; + expect(thinking).toEqual({ thinkingLevel: "low" }); + expect(thinking).not.toHaveProperty("thinkingBudget"); + }); +}); diff --git a/shared/search/triSearch.ts b/shared/search/triSearch.ts index 875ef5de8..9af73b190 100644 --- a/shared/search/triSearch.ts +++ b/shared/search/triSearch.ts @@ -52,9 +52,11 @@ export interface RerankResult { export const RRF_K = 60; export const MAX_RERANK_CANDIDATES = 12; // NOTE: must be a real model id the key can reach (verified via GET /v1beta/models). -// gemini-3.5-flash is a THINKING model — it MUST be paired with thinkingConfig.thinkingBudget:0 +// gemini-3.5-flash is a THINKING model — it MUST be paired with a low thinking level // below, else it spends the token budget on a reasoning preamble and never emits the bare index -// array (verified: thinking-on tok=96 → no array; thinking-0 tok=256 → "[0,1,2]" in ~700ms). +// array (verified with the since-removed thinkingBudget:0: thinking-on tok=96 → no array; +// thinking-0 tok=256 → "[0,1,2]" in ~700ms). Gemini 3.x replaced thinkingBudget with +// thinkingConfig.thinkingLevel (`minimal` is unsupported), so the request now sends "low". const RERANK_MODEL = process.env.TRI_SEARCH_RERANK_MODEL || "gemini-3.5-flash"; const RERANK_TIMEOUT_MS = Number(process.env.TRI_SEARCH_RERANK_TIMEOUT_MS) || 6000; const MAX_TITLE = 120; @@ -137,9 +139,9 @@ export async function rerankWithGemini( headers: { "Content-Type": "application/json" }, body: JSON.stringify({ contents: [{ parts: [{ text: prompt }] }], - // thinkingBudget:0 forces structured output on thinking models (3.x/2.5) so they - // emit the bare index array instead of a reasoning preamble; 256 tokens leaves room. - generationConfig: { temperature: 0, maxOutputTokens: 256, thinkingConfig: { thinkingBudget: 0 } }, + // thinkingLevel "low" keeps thinking models (3.x) from spending the budget on a + // reasoning preamble so they emit the bare index array; 256 tokens leaves room. + generationConfig: { temperature: 0, maxOutputTokens: 256, thinkingConfig: { thinkingLevel: "low" } }, }), signal: AbortSignal.timeout(opts.timeoutMs ?? RERANK_TIMEOUT_MS), }, diff --git a/workers/node/pipeline/extendedThinkingRunner.test.ts b/workers/node/pipeline/extendedThinkingRunner.test.ts index 8f00489fa..111f4ed7d 100644 --- a/workers/node/pipeline/extendedThinkingRunner.test.ts +++ b/workers/node/pipeline/extendedThinkingRunner.test.ts @@ -14,10 +14,13 @@ import { describe, it, expect } from "vitest"; import { buildCheckpointPrompt, + buildCheckpointRequestBody, parseCheckpointResponse, shouldContinue, EXTENDED_PROMPT_VERSION, MAX_CHECKPOINTS, + MAX_OUTPUT_TOKENS, + MAX_THINKING_BUDGET_TOKENS, type CheckpointPromptInput, } from "./extendedThinkingRunner"; @@ -265,3 +268,42 @@ describe("shouldContinue — chain termination rules", () => { expect(v.continue).toBe(false); }); }); + +describe("buildCheckpointRequestBody - Anthropic request shape (SPEC-v9 A.0.6)", () => { + const prompt = { system: "sys", user: "usr" }; + + it("sends the hyphenated wire id for a stored dotted alias", () => { + // run.modelName defaults to the catalog alias "claude-sonnet-4.6"; api.anthropic.com + // rejects that string, so the body must carry the resolver's wire id. + const body = buildCheckpointRequestBody({ modelName: "claude-sonnet-4.6", prompt, remainingThinkingTokens: 40_000 }); + expect(body.model).toBe("claude-sonnet-4-6"); + }); + + it("uses adaptive thinking with output_config.effort, never budget_tokens", () => { + const body = buildCheckpointRequestBody({ modelName: "claude-sonnet-4.6", prompt, remainingThinkingTokens: 40_000 }) as any; + expect(body.thinking).toEqual({ type: "adaptive" }); + expect(body.thinking).not.toHaveProperty("budget_tokens"); + expect(body.output_config).toEqual({ effort: "medium" }); + for (const key of ["temperature", "top_p", "top_k"]) expect(body).not.toHaveProperty(key); + expect(body.messages).toEqual([{ role: "user", content: "usr" }]); + expect(body.system).toBe("sys"); + }); + + it("keeps the thinking budget a bound: max_tokens = answer cap + min(allowance, remaining)", () => { + const at = (remaining: number) => + (buildCheckpointRequestBody({ modelName: "claude-sonnet-4.6", prompt, remainingThinkingTokens: remaining }) as any).max_tokens; + expect(at(40_000)).toBe(MAX_OUTPUT_TOKENS + MAX_THINKING_BUDGET_TOKENS); + expect(at(2_500)).toBe(MAX_OUTPUT_TOKENS + 2_500); + expect(at(0)).toBe(MAX_OUTPUT_TOKENS); + expect(at(-50)).toBe(MAX_OUTPUT_TOKENS); + }); + + it("accepts an operator-supplied wire id and refuses a name with no Anthropic id", () => { + expect(buildCheckpointRequestBody({ modelName: "claude-sonnet-4-6", prompt, remainingThinkingTokens: 1 }).model).toBe( + "claude-sonnet-4-6", + ); + expect(() => buildCheckpointRequestBody({ modelName: "kimi-k2.6", prompt, remainingThinkingTokens: 1 })).toThrow( + /no Anthropic wire id/, + ); + }); +}); diff --git a/workers/node/pipeline/extendedThinkingRunner.ts b/workers/node/pipeline/extendedThinkingRunner.ts index 3d75e75bf..27d302146 100644 --- a/workers/node/pipeline/extendedThinkingRunner.ts +++ b/workers/node/pipeline/extendedThinkingRunner.ts @@ -30,6 +30,8 @@ * prompt drift vs model drift. */ +import { anthropicWireId } from "../../../shared/llm/modelCatalog.js"; + export const EXTENDED_PROMPT_VERSION = "ext-think-v1"; /** Hard safety caps — Convex actions have wall-clock limits. */ @@ -89,6 +91,45 @@ export type ParseOutcome = | { ok: true; result: CheckpointOutput } | { ok: false; error: string }; +/* -------------------------------------------------------------------------- + * Request construction + * ------------------------------------------------------------------------ */ + +/** + * The Messages API body for ONE checkpoint. + * - `model` is the Anthropic wire id from the explicit table in + * shared/llm/modelCatalog.ts; the run's stored `modelName` may be a dotted + * catalog alias ("claude-sonnet-4.6"), which api.anthropic.com rejects. + * A name with no Anthropic wire id throws instead of being sent. + * - Thinking is adaptive with `output_config.effort`: manual + * `{type:"enabled", budget_tokens}` is deprecated on Sonnet/Opus 4.6 and + * 400s on Claude 5.x. Thinking tokens count toward `max_tokens`, so the + * per-checkpoint remaining budget raises the cap above the answer size, + * which keeps the run's thinking budget a real bound. + */ +export function buildCheckpointRequestBody(args: { + modelName: string; + prompt: Pick; + remainingThinkingTokens: number; +}): Record { + const model = anthropicWireId(args.modelName); + if (!model) { + throw new Error(`no Anthropic wire id for model "${args.modelName}"`); + } + const thinkingAllowance = Math.max( + 0, + Math.min(MAX_THINKING_BUDGET_TOKENS, args.remainingThinkingTokens), + ); + return { + model, + max_tokens: MAX_OUTPUT_TOKENS + thinkingAllowance, + thinking: { type: "adaptive" }, + output_config: { effort: "medium" }, + system: args.prompt.system, + messages: [{ role: "user", content: args.prompt.user }], + }; +} + /* -------------------------------------------------------------------------- * Prompt construction * ------------------------------------------------------------------------ */ From 7096f29f6dccc7da4ba72b6aa797323c4300bf71 Mon Sep 17 00:00:00 2001 From: Homen Shum Date: Wed, 14 Oct 2026 12:53:49 -0700 Subject: [PATCH 2/8] fix(models): keep free requests free, fail closed at the entry points, make the eval honest (PR 1 review round 1) Fixes the defects from the round 1 review of PR #662. Free requests stay free - FREE_LANE_MODEL heads the free segment of MODEL_PRIORITY_ORDER (resolver and web mirror). executeWithModelFallback, getFallbackChain and getLanguageModelWithFallback throw for a start model outside the order instead of restarting at kimi-k2.6, which dispatched a paid model for an auto-free run and reported isFree=false. piRuntime prices the fallback from the model that answered instead of a hardcoded 0. - normalizeNodeBenchRuntimeModel, the first resolver on the FastAgent entry points, follows resolveOrFailClosed: unknown throws, free-looking goes to the free lane, a known free model keeps its cost class in every profile. runtimeRouting stops swallowing an unknown requested id and never appends paid candidates to a free primary. continueThread rethrows UnknownModelError. getModelWithFailover no longer turns an unknown id into gpt-5.4-mini or fails a free model over to a paid provider. The free tier of modelRouter no longer appends the cheap list. Eval honesty - runner.ts exits 1 when a real-backend run has an error (stub smoke stays 0). - agentInvoker records tool_use blocks as requests, not successful calls; an unfinished tool_use turn or a response without a model is an error. - run-comprehensive-eval requires a canonical --models id, treats a row with no resolved identity as a violation, and keeps identity on error rows. - Extended-thinking runs are charged from usage.output_tokens minus the answer because the thinking text is omitted by default on Sonnet 5.x / Opus 4.7+; the max_tokens claim is corrected (it bounds thinking plus answer). Tests (A.0.6): request-shape and boundary tests through the real callers for modelRouter, benchmark, mcp-local, generate-tool, advanceRun, the 13 provider copies, continueThread and runtimeRouting; the web auto-free literal is pinned to the backend route resolver; the Gemini rerank test checks the model in the URL. All added to the CI runtime-smoke list. generate-tool.mjs now exports its own buildClaudeRequest. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/ci.yml | 7 + .../shared/llm/approvedModels.drift.test.ts | 11 ++ apps/web/src/shared/llm/approvedModels.ts | 6 + .../agentChatActions.failClosed.test.ts | 63 ++++++++ .../convex/domains/agents/agentChatActions.ts | 4 +- .../convex/domains/agents/fastAgentChat.ts | 3 +- .../mcp_tools/models/modelResolver.test.ts | 123 ++++++++++++++ .../agents/mcp_tools/models/modelResolver.ts | 53 +++++-- .../domains/agents/runtimeRouting.test.ts | 57 ++++++- .../convex/domains/agents/runtimeRouting.ts | 19 ++- .../models/modelRouter.requestShape.test.ts | 86 ++++++++++ .../convex/domains/ai/models/modelRouter.ts | 4 +- backend/convex/domains/ai/morningDigest.ts | 3 +- .../domains/knowledge/entityInsights.ts | 3 +- backend/convex/domains/pipelines/piRuntime.ts | 10 +- .../extendedThinking.requestShape.test.ts | 114 +++++++++++++ .../domains/product/extendedThinking.ts | 12 +- .../research/dailyBriefPersonalOverlay.ts | 3 +- .../domains/research/dailyBriefWorker.ts | 3 +- backend/convex/domains/research/dealFlow.ts | 3 +- .../domains/research/modelComparison.ts | 3 +- .../convex/domains/research/paperDetails.ts | 3 +- backend/convex/domains/research/repoScout.ts | 3 +- .../convex/domains/research/stackImpact.ts | 3 +- .../domains/research/strategyMetrics.ts | 3 +- .../fusion/benchmark.requestShape.test.ts | 78 +++++++++ .../convex/domains/social/postDedupAction.ts | 3 +- .../verification/claimVerificationAction.ts | 3 +- .../providerCopies.requestShape.test.ts | 150 ++++++++++++++++++ .../__tests__/anthropicRequestShape.test.ts | 68 ++++++++ .../__tests__/comprehensiveEvalModels.test.ts | 69 ++++++-- scripts/__tests__/toolGenRequestShape.test.ts | 47 ++++++ scripts/eval-harness/agentInvoker.ts | 21 ++- scripts/eval-harness/evalHarness.test.ts | 74 ++++++++- scripts/eval-harness/runner.ts | 15 +- scripts/eval-harness/types.ts | 8 + scripts/run-comprehensive-eval.ts | 34 ++-- scripts/tool-gen/generate-tool.mjs | 41 +++-- shared/llm/anthropicWireIds.test.ts | 47 +++++- shared/llm/modelCatalog.ts | 28 +++- shared/search/triSearch.test.ts | 7 +- .../pipeline/extendedThinkingRunner.test.ts | 40 ++++- .../node/pipeline/extendedThinkingRunner.ts | 24 ++- 43 files changed, 1250 insertions(+), 109 deletions(-) create mode 100644 backend/convex/domains/agents/agentChatActions.failClosed.test.ts create mode 100644 backend/convex/domains/ai/models/modelRouter.requestShape.test.ts create mode 100644 backend/convex/domains/product/extendedThinking.requestShape.test.ts create mode 100644 backend/convex/domains/search/fusion/benchmark.requestShape.test.ts create mode 100644 backend/convex/providerCopies.requestShape.test.ts create mode 100644 packages/mcp-local/src/__tests__/anthropicRequestShape.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43d87559a..a786a129b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -109,6 +109,13 @@ jobs: scripts/__tests__/toolGenRequestShape.test.ts scripts/__tests__/comprehensiveEvalModels.test.ts apps/web/src/shared/llm/approvedModels.drift.test.ts + backend/convex/domains/ai/models/modelRouter.requestShape.test.ts + backend/convex/domains/search/fusion/benchmark.requestShape.test.ts + backend/convex/domains/product/extendedThinking.requestShape.test.ts + backend/convex/providerCopies.requestShape.test.ts + backend/convex/domains/agents/runtimeRouting.test.ts + backend/convex/domains/agents/agentChatActions.failClosed.test.ts + packages/mcp-local/src/__tests__/anthropicRequestShape.test.ts apps/web/src/features/agents/components/FastAgentPanel/__tests__/RunState.test.tsx backend/convex/domains/agents/autonomy/autonomy.integration.test.ts backend/convex/domains/agents/autonomy/autonomyTasteBench.integration.test.ts diff --git a/apps/web/src/shared/llm/approvedModels.drift.test.ts b/apps/web/src/shared/llm/approvedModels.drift.test.ts index a9ca129ad..46a7b448f 100644 --- a/apps/web/src/shared/llm/approvedModels.drift.test.ts +++ b/apps/web/src/shared/llm/approvedModels.drift.test.ts @@ -10,6 +10,7 @@ import { describe, expect, it } from "vitest"; import * as web from "./approvedModels"; +import { PIPELINE_MODEL_OPTIONS } from "./pipelineModelRoutes"; import * as backend from "../../../../../backend/convex/domains/agents/mcp_tools/models/modelResolver"; describe("web approvedModels mirrors modelResolver", () => { @@ -49,4 +50,14 @@ describe("web approvedModels mirrors modelResolver", () => { } } }); + + it("pins every selector option, including the auto-free literal, to the backend route resolver", () => { + // pipelineModelRoutes.ts types its resolved ids by hand. With getBestFreeModel reverted to + // laguna, the web tests still passed and only modelResolver.test.ts failed; this closes that gap. + for (const option of PIPELINE_MODEL_OPTIONS) { + const resolved = backend.resolvePipelineModelSelection(option.value); + expect(resolved.resolvedModelId, `${option.value} resolvedModelId`).toBe(option.resolvedModelId); + expect(resolved.isFreeRoute, `${option.value} isFree`).toBe(option.isFree); + } + }); }); diff --git a/apps/web/src/shared/llm/approvedModels.ts b/apps/web/src/shared/llm/approvedModels.ts index fd7361788..5a96beee2 100644 --- a/apps/web/src/shared/llm/approvedModels.ts +++ b/apps/web/src/shared/llm/approvedModels.ts @@ -502,6 +502,7 @@ export const MODEL_PRIORITY_ORDER: ApprovedModel[] = [ "claude-opus-4.7", "claude-haiku-4.5", "gpt-5.4-nano", + "nemotron-3-super-free", "laguna-s-2.1-free", "laguna-xs-2.1-free", "qwen3-coder-free", @@ -569,6 +570,11 @@ export function normalizeNodeBenchRuntimeModel( return fallback; } + // A free model keeps its cost class in every profile (mirrors modelResolver.ts). + if (MODEL_UI_INFO[candidate].isFree) { + return candidate; + } + if (profile === "advisor" && NODEBENCH_ADVISOR_ALLOWED.has(candidate)) { return candidate; } diff --git a/backend/convex/domains/agents/agentChatActions.failClosed.test.ts b/backend/convex/domains/agents/agentChatActions.failClosed.test.ts new file mode 100644 index 000000000..6adb1af02 --- /dev/null +++ b/backend/convex/domains/agents/agentChatActions.failClosed.test.ts @@ -0,0 +1,63 @@ +// @vitest-environment node +/** + * agentChatActions.continueThread resumes a conversation on the model stored on its thread + * (SPEC-v9 A.0.10; PR #662 review round 1). Persona: a user returning to a thread whose stored + * model name no longer resolves (a retired or mistyped alias). The turn must stop with + * UnknownModelError. It used to catch the error, log a warning and carry on with DEFAULT_MODEL + * (kimi-k2.6), so the user paid for a model they never chose; logging a fallback is not a control. + * A transient failure to read the thread is a different thing and still uses the default. + * The agent is faked; no network, no spend. + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const createCoordinatorAgent = vi.hoisted(() => vi.fn()); +vi.mock("./core/coordinatorAgent", () => ({ createCoordinatorAgent })); +vi.mock("@convex-dev/auth/server", () => ({ getAuthUserId: vi.fn(async () => "user1") })); + +import { continueThread } from "./agentChatActions"; +import { UnknownModelError } from "./mcp_tools/models"; + +function ctxWith(streamThreadModel: string | Error) { + return { + runQuery: vi.fn(async (_ref: unknown, args: Record) => { + if ("agentThreadId" in args) { + if (streamThreadModel instanceof Error) throw streamThreadModel; + return { model: streamThreadModel }; + } + return { agentThreadId: "agent-thread-1" }; + }), + }; +} + +describe("agentChatActions.continueThread fails closed on a stored model", () => { + beforeEach(() => { + vi.spyOn(console, "warn").mockImplementation(() => {}); + createCoordinatorAgent.mockReset(); + const result = { consumeStream: vi.fn(async () => undefined) }; + createCoordinatorAgent.mockReturnValue({ + saveMessage: vi.fn(async () => ({ messageId: "m1" })), + streamText: vi.fn(async () => result), + }); + }); + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("an unknown stored model stops the turn and starts no agent", async () => { + await expect( + (continueThread as any)._handler(ctxWith("gpt-9-ultra"), { threadId: "t1", message: "hi" }), + ).rejects.toBeInstanceOf(UnknownModelError); + expect(createCoordinatorAgent).not.toHaveBeenCalled(); + }); + + it("a free-looking stored model continues on the free lane, not a paid default", async () => { + await (continueThread as any)._handler(ctxWith("retired-model-free"), { threadId: "t1", message: "hi" }); + expect(createCoordinatorAgent).toHaveBeenCalledWith("nemotron-3-super-free"); + }); + + it("a transient failure to read the thread still falls back to the default model", async () => { + await (continueThread as any)._handler(ctxWith(new Error("convex timeout")), { threadId: "t1", message: "hi" }); + expect(createCoordinatorAgent).toHaveBeenCalledWith("kimi-k2.6"); + }); +}); diff --git a/backend/convex/domains/agents/agentChatActions.ts b/backend/convex/domains/agents/agentChatActions.ts index 0668fd0f3..edde5dc09 100644 --- a/backend/convex/domains/agents/agentChatActions.ts +++ b/backend/convex/domains/agents/agentChatActions.ts @@ -11,7 +11,7 @@ import { api, internal } from "../../_generated/api"; import type { Id } from "../../_generated/dataModel"; import { getAuthUserId } from "@convex-dev/auth/server"; import { createCoordinatorAgent } from "./core/coordinatorAgent"; -import { DEFAULT_MODEL, normalizeModelInput } from "./mcp_tools/models"; +import { DEFAULT_MODEL, normalizeModelInput, UnknownModelError } from "./mcp_tools/models"; // Agent-mode chat should use the same CoordinatorAgent wiring as streaming mode. const createChatAgent = (model: string) => createCoordinatorAgent(model); @@ -133,6 +133,8 @@ export const continueThread = (action as any)({ modelName = normalizeModelInput(streamThreadByAgentId.model); } } catch (err) { + // An unknown stored model must stop the turn, not continue on DEFAULT_MODEL (SPEC-v9 A.0.10). + if (err instanceof UnknownModelError) throw err; console.warn("[agentChatActions.continueThread] Failed to resolve thread model, using default.", err); } diff --git a/backend/convex/domains/agents/fastAgentChat.ts b/backend/convex/domains/agents/fastAgentChat.ts index abe45cbbc..75cd57227 100644 --- a/backend/convex/domains/agents/fastAgentChat.ts +++ b/backend/convex/domains/agents/fastAgentChat.ts @@ -283,7 +283,8 @@ async function handleDocumentEdit( * Handle chat/question requests * Uses direct LLM call for quick responses */ -async function handleChatResponse( +// Exported for providerCopies.requestShape.test.ts (SPEC-v9 A.0.6); not a Convex function. +export async function handleChatResponse( ctx: ActionCtx, args: { message: string; diff --git a/backend/convex/domains/agents/mcp_tools/models/modelResolver.test.ts b/backend/convex/domains/agents/mcp_tools/models/modelResolver.test.ts index 143a3e173..76fc986a2 100644 --- a/backend/convex/domains/agents/mcp_tools/models/modelResolver.test.ts +++ b/backend/convex/domains/agents/mcp_tools/models/modelResolver.test.ts @@ -5,14 +5,19 @@ import { DEFAULT_MODEL, FREE_LANE_MODEL, MODEL_SPECS, + MODEL_PRIORITY_ORDER, UnknownModelError, assertKnownPipelineModel, + executeWithModelFallback, + getFallbackChain, getLanguageModel, + getLanguageModelWithFallback, getLanguageModelSafe, getPaidFallbackCount, isApprovedModel, isFreeModel, normalizeModelInput, + normalizeNodeBenchRuntimeModel, resolveModelAlias, resolvePipelineModelSelection, } from "./modelResolver"; @@ -170,3 +175,121 @@ describe("unknown model ids fail closed", () => { expect(() => assertKnownPipelineModel("x:free")).not.toThrow(); }); }); + +// --------------------------------------------------------------------------- +// A free request never becomes a paid one (SPEC-v9 A.0.10; PR #662 review round 1) +// --------------------------------------------------------------------------- + +describe("fallback helpers keep a free request on free models", () => { + beforeEach(() => { + vi.stubEnv("OPENROUTER_API_KEY", "test-key-not-real"); + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "warn").mockImplementation(() => {}); + }); + afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + }); + + it("every free id in MODEL_PRIORITY_ORDER sits after every paid id, so a chain from a free id is all free", () => { + const firstFree = MODEL_PRIORITY_ORDER.findIndex((id) => isFreeModel(id)); + expect(firstFree).toBeGreaterThan(0); + expect(MODEL_PRIORITY_ORDER.slice(firstFree).every((id) => isFreeModel(id))).toBe(true); + // FREE_LANE_MODEL heads the free segment: it is listed, and nothing paid follows it. + expect(MODEL_PRIORITY_ORDER[firstFree]).toBe(FREE_LANE_MODEL); + }); + + it("executeWithModelFallback from FREE_LANE_MODEL never reaches a paid model, even when every free model fails", async () => { + // Before: FREE_LANE_MODEL was not in the order, so an unlisted start model restarted at + // index 0 (kimi-k2.6, $0.75/$3.5 per M tokens) and reported isFree=false. + const tried: string[] = []; + const first = await executeWithModelFallback(async (_m, id) => { + tried.push(id); + return id; + }, { startModel: FREE_LANE_MODEL }); + expect(first.modelUsed).toBe(FREE_LANE_MODEL); + expect(first.isFree).toBe(true); + + const triedAll: string[] = []; + await expect( + executeWithModelFallback( + async (_m, id) => { + triedAll.push(id); + throw new Error("provider down"); + }, + { startModel: FREE_LANE_MODEL, maxRetries: 0 }, + ), + ).rejects.toThrow(/All models in fallback chain failed/); + expect(triedAll[0]).toBe(FREE_LANE_MODEL); + expect(triedAll.length).toBeGreaterThan(1); + expect(triedAll.every((id) => isFreeModel(id as never))).toBe(true); + }); + + it("starting on a legacy free id (laguna) stays free too", async () => { + const r = await executeWithModelFallback(async (_m, id) => id, { startModel: "laguna-s-2.1-free" }); + expect(r.modelUsed).toBe("laguna-s-2.1-free"); + expect(r.isFree).toBe(true); + }); + + it("getFallbackChain and getLanguageModelWithFallback from FREE_LANE_MODEL are all free", () => { + const chain = getFallbackChain(FREE_LANE_MODEL); + expect(chain[0]).toBe(FREE_LANE_MODEL); + expect(chain.every((id) => isFreeModel(id))).toBe(true); + const wf = getLanguageModelWithFallback(FREE_LANE_MODEL); + expect(wf.modelId).toBe(FREE_LANE_MODEL); + expect(wf.isFree).toBe(true); + expect(wf.fallbackChain.every((id) => isFreeModel(id))).toBe(true); + }); + + it("an approved model that is not in the priority order throws instead of restarting at the paid head", async () => { + const unlisted = APPROVED_MODELS.find((id) => !MODEL_PRIORITY_ORDER.includes(id) && isFreeModel(id)); + expect(unlisted, "expected an approved free model outside the priority order").toBeDefined(); + await expect(executeWithModelFallback(async () => "x", { startModel: unlisted! })).rejects.toThrow(/not in MODEL_PRIORITY_ORDER/); + expect(() => getFallbackChain(unlisted!)).toThrow(/not in MODEL_PRIORITY_ORDER/); + expect(() => getLanguageModelWithFallback(unlisted!)).toThrow(/not in MODEL_PRIORITY_ORDER/); + }); +}); + +describe("normalizeNodeBenchRuntimeModel is the first resolver and fails closed", () => { + beforeEach(() => { + vi.spyOn(console, "warn").mockImplementation(() => {}); + }); + afterEach(() => { + vi.restoreAllMocks(); + }); + + const PROFILES = ["advisor", "executor", "background"] as const; + + it("throws for an unknown non-free id in every profile (was: kimi / gemini, all paid)", () => { + for (const profile of PROFILES) { + expect(() => normalizeNodeBenchRuntimeModel("gpt-9-ultra", profile), profile).toThrow(UnknownModelError); + } + }); + + it("keeps a free-looking unknown id on the free lane in every profile", () => { + for (const profile of PROFILES) { + for (const input of ["x:free", "some-model-free", "auto-free"]) { + expect(normalizeNodeBenchRuntimeModel(input, profile), `${profile}/${input}`).toBe(FREE_LANE_MODEL); + } + } + }); + + it("keeps a known free model free in every profile (FREE_LANE_MODEL, qwen3-coder-free, laguna)", () => { + for (const profile of PROFILES) { + for (const id of [FREE_LANE_MODEL, "qwen3-coder-free", "laguna-s-2.1-free"] as const) { + expect(normalizeNodeBenchRuntimeModel(id, profile), `${profile}/${id}`).toBe(id); + } + } + }); + + it("empty input takes the profile default, and a known paid model outside the profile takes it too", () => { + expect(normalizeNodeBenchRuntimeModel(undefined, "advisor")).toBe("kimi-k2.6"); + expect(normalizeNodeBenchRuntimeModel("", "executor")).toBe("gemini-3.1-flash-lite-preview"); + expect(normalizeNodeBenchRuntimeModel(null, "background")).toBe("gemini-3.1-pro-preview"); + expect(normalizeNodeBenchRuntimeModel("claude-opus-4.7", "executor")).toBe("gemini-3.1-flash-lite-preview"); + }); + + it("allowPaidFallback turns an unknown id into the profile-checked DEFAULT_MODEL", () => { + expect(normalizeNodeBenchRuntimeModel("gpt-9-ultra", "advisor", { allowPaidFallback: true })).toBe(DEFAULT_MODEL); + }); +}); diff --git a/backend/convex/domains/agents/mcp_tools/models/modelResolver.ts b/backend/convex/domains/agents/mcp_tools/models/modelResolver.ts index 14f5c370a..2fc6ced0b 100644 --- a/backend/convex/domains/agents/mcp_tools/models/modelResolver.ts +++ b/backend/convex/domains/agents/mcp_tools/models/modelResolver.ts @@ -186,8 +186,10 @@ export const MODEL_PRIORITY_ORDER: ApprovedModel[] = [ "claude-opus-4.7", "claude-haiku-4.5", "gpt-5.4-nano", - // EXPERIMENTAL / FREE LANES - "laguna-s-2.1-free", // Current primary free route + // EXPERIMENTAL / FREE LANES. Every id from here on is free, so a chain that + // starts on a free model never reaches a paid one (modelResolver.test.ts pins this). + "nemotron-3-super-free", // FREE_LANE_MODEL: head of the free segment + "laguna-s-2.1-free", // Previous primary free route (expires 2026-10-31) "laguna-xs-2.1-free", // Current explicit free fallback "qwen3-coder-free", // Legacy free alias retained for compatibility "step-3.5-flash-free", // 196B MoE, reasoning + tools @@ -244,11 +246,18 @@ const NODEBENCH_EXECUTOR_ALLOWED = new Set([ const NODEBENCH_BACKGROUND_ALLOWED = new Set(NODEBENCH_BACKGROUND_MODELS); +/** + * First resolver on the FastAgent public entry points. Empty input takes the + * profile default. Anything else goes through `resolveOrFailClosed` (A.0.10): a + * free-looking unknown id becomes FREE_LANE_MODEL, any other unknown id throws + * `UnknownModelError`. A free model keeps its cost class in every profile; a + * known paid model the profile does not allow takes the profile default. + */ export function normalizeNodeBenchRuntimeModel( input: string | undefined | null, profile: NodeBenchRuntimeProfile = "advisor", + opts?: UnknownModelOptions, ): ApprovedModel { - const resolved = input ? resolveModelAlias(input) : null; const fallback = profile === "advisor" ? NODEBENCH_ADVISOR_MODEL @@ -256,10 +265,15 @@ export function normalizeNodeBenchRuntimeModel( ? NODEBENCH_BACKGROUND_MODELS[0] : NODEBENCH_EXECUTOR_MODELS[0]; - if (!resolved) { + if (!input) { return fallback; } + const resolved = resolveOrFailClosed(input, opts, "normalizeNodeBenchRuntimeModel"); + + if (isFreeModel(resolved)) { + return resolved; + } if (profile === "advisor" && NODEBENCH_ADVISOR_ALLOWED.has(resolved)) { return resolved; } @@ -1229,6 +1243,19 @@ export function getFreeModels(): ApprovedModel[] { return APPROVED_MODELS.filter((model) => isFreeModel(model)); } +/** + * Index of `model` in MODEL_PRIORITY_ORDER. A start model that is not in the + * order throws: restarting it at index 0 (a paid model) would turn a free + * request into a paid one (SPEC-v9 A.0.10). + */ +function priorityIndexOrThrow(model: ApprovedModel, caller: string): number { + const index = MODEL_PRIORITY_ORDER.indexOf(model); + if (index === -1) { + throw new Error(`[modelResolver] ${caller}: "${model}" is not in MODEL_PRIORITY_ORDER; refusing to restart at the head of the paid chain`); + } + return index; +} + /** * Get the next model in the priority fallback chain * Returns null if no more models available @@ -1253,12 +1280,13 @@ export function getLanguageModelWithFallback(preferredModel?: ApprovedModel): { } { const modelId = preferredModel ?? DEFAULT_MODEL; const spec = MODEL_SPECS[modelId]; + const index = priorityIndexOrThrow(modelId, "getLanguageModelWithFallback"); return { model: buildLanguageModel(spec), modelId, isFree: isFreeModel(modelId), - fallbackChain: MODEL_PRIORITY_ORDER.slice(MODEL_PRIORITY_ORDER.indexOf(modelId) + 1), + fallbackChain: MODEL_PRIORITY_ORDER.slice(index + 1), }; } @@ -1284,13 +1312,8 @@ export async function executeWithModelFallback( let fallbacksUsed = 0; let lastError: Error | null = null; - // Get the starting index in priority order - let modelIndex = MODEL_PRIORITY_ORDER.indexOf(currentModel); - if (modelIndex === -1) { - // Model not in priority order, start from beginning - modelIndex = 0; - currentModel = MODEL_PRIORITY_ORDER[0]; - } + // Get the starting index in priority order; an unlisted start model throws. + let modelIndex = priorityIndexOrThrow(currentModel, "executeWithModelFallback"); while (modelIndex < MODEL_PRIORITY_ORDER.length) { currentModel = MODEL_PRIORITY_ORDER[modelIndex]; @@ -1367,9 +1390,5 @@ export function getBestFreeModel(): ApprovedModel { * Get the fallback chain starting from a given model */ export function getFallbackChain(startModel: ApprovedModel = DEFAULT_MODEL): ApprovedModel[] { - const startIndex = MODEL_PRIORITY_ORDER.indexOf(startModel); - if (startIndex === -1) { - return [...MODEL_PRIORITY_ORDER]; - } - return MODEL_PRIORITY_ORDER.slice(startIndex); + return MODEL_PRIORITY_ORDER.slice(priorityIndexOrThrow(startModel, "getFallbackChain")); } diff --git a/backend/convex/domains/agents/runtimeRouting.test.ts b/backend/convex/domains/agents/runtimeRouting.test.ts index bee58b25f..cf1c736b4 100644 --- a/backend/convex/domains/agents/runtimeRouting.test.ts +++ b/backend/convex/domains/agents/runtimeRouting.test.ts @@ -1,5 +1,6 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { chooseNodeBenchRuntimeRoute } from "./runtimeRouting"; +import { FREE_LANE_MODEL, UnknownModelError, isFreeModel } from "./mcp_tools/models/modelResolver"; import type { UltraLongChatWorkingSet } from "../../../../shared/ultraLongChatContext"; function buildWorkingSet( @@ -201,3 +202,57 @@ describe("chooseNodeBenchRuntimeRoute", () => { expect(route.fallbackModels).toContain("kimi-k2.6"); }); }); + +// A request for the free lane, or for a model that does not exist, must not become a paid +// request (SPEC-v9 A.0.10). Persona: a user who picked the free model in the selector. +describe("chooseNodeBenchRuntimeRoute keeps cost class and fails closed", () => { + const paidAvailable = [ + "kimi-k2.6", + "gemini-3.1-pro-preview", + "gemini-3-flash-preview", + "gemini-3.1-flash-lite-preview", + "gpt-5.4", + "gpt-5.4-mini", + "minimax-m2.7", + ] as never; + const ask = (requestedModel: string, prompt: string, availableModels: never | null = paidAvailable) => + chooseNodeBenchRuntimeRoute({ prompt, requestedModel, useCoordinator: true, availableModels, workingSet: buildWorkingSet() }); + + it("an explicit free request stays on the free lane in the advisor, executor and background routes", () => { + const advisor = ask(FREE_LANE_MODEL, "Plan and orchestrate a deep research run on Stripe."); + expect(advisor.model).toBe(FREE_LANE_MODEL); + expect(advisor.fallbackModels.every((id) => isFreeModel(id))).toBe(true); + + const executor = ask(FREE_LANE_MODEL, "Summarize this company."); + expect(executor.model).toBe(FREE_LANE_MODEL); + expect(executor.fallbackModels.every((id) => isFreeModel(id))).toBe(true); + + const background = chooseNodeBenchRuntimeRoute({ + prompt: "Continue.", + requestedModel: FREE_LANE_MODEL, + availableModels: paidAvailable, + workingSet: buildWorkingSet({ contextRotRisk: "high" }), + }); + expect(background.model).toBe(FREE_LANE_MODEL); + expect(background.fallbackModels.every((id) => isFreeModel(id))).toBe(true); + }); + + it("a free primary that is not in the available list does not fall over to a paid model", () => { + // availableModels contains only paid ids: the old chooser would pick the first paid candidate. + const route = ask(FREE_LANE_MODEL, "Plan and orchestrate a deep research run on Stripe."); + expect(isFreeModel(route.model)).toBe(true); + }); + + it("an unknown requested id throws on every route instead of taking a paid default", () => { + for (const prompt of [ + "Plan and orchestrate a deep research run on Stripe.", // advisor + "Summarize this company.", // executor (reaches resolveRequestedExecutorModel) + ]) { + expect(() => ask("gpt-9-ultra", prompt), prompt).toThrow(UnknownModelError); + } + }); + + it("a free-looking unknown id takes the free lane on the executor route", () => { + expect(ask("some-model-free", "Summarize this company.").model).toBe(FREE_LANE_MODEL); + }); +}); diff --git a/backend/convex/domains/agents/runtimeRouting.ts b/backend/convex/domains/agents/runtimeRouting.ts index df42f309f..2896ef2e8 100644 --- a/backend/convex/domains/agents/runtimeRouting.ts +++ b/backend/convex/domains/agents/runtimeRouting.ts @@ -3,6 +3,7 @@ import { NODEBENCH_ADVISOR_MODEL, NODEBENCH_BACKGROUND_MODELS, NODEBENCH_EXECUTOR_MODELS, + isFreeModel, normalizeNodeBenchRuntimeModel, resolveModelAlias, type ApprovedModel, @@ -81,7 +82,11 @@ function chooseAvailableRouteModels(args: { candidates: ApprovedModel[]; availableModels?: ApprovedModel[] | null; }): { model: ApprovedModel; fallbackModels: ApprovedModel[] } { - const ordered = [args.primary, ...buildFallbacks(args.primary, args.candidates)]; + // A free primary never falls back to a paid candidate (SPEC-v9 A.0.10). + const candidates = isFreeModel(args.primary) + ? args.candidates.filter(isFreeModel) + : args.candidates; + const ordered = [args.primary, ...buildFallbacks(args.primary, candidates)]; const availableOrdered = ordered.filter((model, index, array) => { if (array.indexOf(model) !== index) return false; return isRoutableModelAvailable(model, args.availableModels); @@ -94,7 +99,7 @@ function chooseAvailableRouteModels(args: { return { model: args.primary, - fallbackModels: buildFallbacks(args.primary, args.candidates), + fallbackModels: buildFallbacks(args.primary, candidates), }; } @@ -105,9 +110,13 @@ function resolveRequestedExecutorModel( return null; } - const resolved = resolveModelAlias(requestedModel); - if (!resolved) { - return null; + // An unknown id throws (or becomes the free lane when free-looking); it must + // not fall through to the paid executor default (SPEC-v9 A.0.10). + const resolved = + resolveModelAlias(requestedModel) ?? + normalizeNodeBenchRuntimeModel(requestedModel, "executor"); + if (isFreeModel(resolved)) { + return resolved; } if (!REQUESTABLE_EXECUTOR_MODELS.has(resolved)) { diff --git a/backend/convex/domains/ai/models/modelRouter.requestShape.test.ts b/backend/convex/domains/ai/models/modelRouter.requestShape.test.ts new file mode 100644 index 000000000..48f277ee3 --- /dev/null +++ b/backend/convex/domains/ai/models/modelRouter.requestShape.test.ts @@ -0,0 +1,86 @@ +// @vitest-environment node +/** + * modelRouter.route is the single inference entrypoint. Two things changed in + * SPEC-v9 A.2.10 PR 1 and need a test that goes through the real action, not a helper: + * 1. Claude 4.7+/5.x answer 400 to `temperature` and Gemini 3.x deprecates it, so the + * request carries it only for the providers that still accept it. + * 2. The free tier no longer appends paid candidates to its fallback list (A.0.10): + * a user who asked for free must not be billed because the free models were down. + * Persona: a background agent calling route() with tier "free", then pinned to a Claude + * model for a repro pack. generateText is the fake transport; no network, no spend. + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const generateText = vi.hoisted(() => vi.fn()); +const resolveChain = vi.hoisted(() => vi.fn()); + +vi.mock("ai", async (importOriginal) => ({ ...(await importOriginal()), generateText })); +vi.mock("./chainResolver", () => ({ resolveChain })); + +import { route } from "./modelRouter"; + +const baseArgs = { + taskCategory: "research", + systemPrompt: "sys", + messages: [{ role: "user" as const, content: "hi" }], +}; +const makeCtx = () => ({ + runQuery: vi.fn(async () => ({ allowed: true })), + runMutation: vi.fn(async () => null), +}); +const call = (ctx: ReturnType, args: Record) => + (route as any)._handler(ctx, { ...baseArgs, ...args }); + +describe("modelRouter.route request shape and cost class", () => { + beforeEach(() => { + for (const key of ["OPENROUTER_API_KEY", "ANTHROPIC_API_KEY", "OPENAI_API_KEY", "GEMINI_API_KEY"]) { + vi.stubEnv(key, "test-key-not-real-0000"); + } + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "warn").mockImplementation(() => {}); + generateText.mockReset(); + resolveChain.mockReset(); + generateText.mockResolvedValue({ text: "ok", usage: { inputTokens: 1, outputTokens: 1 } }); + }); + afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + }); + + it("omits temperature for Claude and Gemini and sends it for the providers that accept it", async () => { + const sent: Record = {}; + for (const pinnedModel of ["claude-sonnet-4.6", "gemini-3.1-pro-preview", "kimi-k2.6", "gpt-5.4"]) { + generateText.mockClear(); + await call(makeCtx(), { pinnedModel, temperature: 0.7 }); + expect(generateText).toHaveBeenCalledTimes(1); + sent[pinnedModel] = "temperature" in generateText.mock.calls[0][0]; + } + expect(sent).toEqual({ + "claude-sonnet-4.6": false, + "gemini-3.1-pro-preview": false, + "kimi-k2.6": true, + "gpt-5.4": true, + }); + // and the value is the caller's, not a constant + expect(generateText.mock.calls[0][0].temperature).toBe(0.7); + }); + + it("the free tier tries only free models when every free model fails: no paid candidate is appended", async () => { + // Empty registry chain forces the static TIER_MODELS safety net, where the paid push used to live. + resolveChain.mockReturnValue({ chain: [], reason: "no_matches_for_requirement", diagnostics: { primaryOutcome: "not_supplied", candidatePoolSize: 0 } }); + generateText.mockRejectedValue(Object.assign(new Error("provider down"), { status: 400 })); + + await expect(call(makeCtx(), { tier: "free" })).rejects.toThrow(/All 3 models failed/); + + expect(generateText).toHaveBeenCalledTimes(3); + const tried = generateText.mock.calls.map((c) => (c[0].model as { modelId: string }).modelId); + expect(tried.every((id) => id.endsWith(":free")), tried.join(", ")).toBe(true); + }); + + it("a cheap-tier outage still falls through the cheap list (the paid list is only for paid tiers)", async () => { + resolveChain.mockReturnValue({ chain: [], reason: "no_matches_for_requirement", diagnostics: { primaryOutcome: "not_supplied", candidatePoolSize: 0 } }); + generateText.mockRejectedValue(Object.assign(new Error("provider down"), { status: 400 })); + await expect(call(makeCtx(), { tier: "cheap" })).rejects.toThrow(/All 4 models failed/); + }); +}); diff --git a/backend/convex/domains/ai/models/modelRouter.ts b/backend/convex/domains/ai/models/modelRouter.ts index 5ae533607..0e85524db 100644 --- a/backend/convex/domains/ai/models/modelRouter.ts +++ b/backend/convex/domains/ai/models/modelRouter.ts @@ -405,10 +405,8 @@ export const route = internalAction({ console.warn( `[modelRouter] chain resolver returned empty (reason=${resolution.reason}); falling back to TIER_MODELS[${tier}]`, ); + // The free tier never appends paid candidates (SPEC-v9 A.0.10). candidates = [...(TIER_MODELS[tier] ?? TIER_MODELS.cheap)]; - if (tier === "free") { - candidates.push(...TIER_MODELS.cheap); - } } } diff --git a/backend/convex/domains/ai/morningDigest.ts b/backend/convex/domains/ai/morningDigest.ts index 55ad44209..6473d1e4e 100644 --- a/backend/convex/domains/ai/morningDigest.ts +++ b/backend/convex/domains/ai/morningDigest.ts @@ -12,7 +12,8 @@ import { getLlmModel, getProviderForModel, resolveModelAlias, getModelWithFailov // Import it via: api.domains.ai.morningDigestQueries.getDigestData // Multi-provider LLM helper -async function generateWithProvider( +// Exported for providerCopies.requestShape.test.ts (SPEC-v9 A.0.6); not a Convex function. +export async function generateWithProvider( modelInput: string, systemPrompt: string, userPrompt: string, diff --git a/backend/convex/domains/knowledge/entityInsights.ts b/backend/convex/domains/knowledge/entityInsights.ts index 0c459afd7..c1ab6c6f6 100644 --- a/backend/convex/domains/knowledge/entityInsights.ts +++ b/backend/convex/domains/knowledge/entityInsights.ts @@ -592,7 +592,8 @@ function generatePersonaHooks(entity: EnrichedEntityPayload): PersonaHooks { }; } -async function generateWithProvider( +// Exported for providerCopies.requestShape.test.ts (SPEC-v9 A.0.6); not a Convex function. +export async function generateWithProvider( modelInput: string, systemPrompt: string, userPrompt: string, diff --git a/backend/convex/domains/pipelines/piRuntime.ts b/backend/convex/domains/pipelines/piRuntime.ts index 27d1cc8f1..aa3a9db4a 100644 --- a/backend/convex/domains/pipelines/piRuntime.ts +++ b/backend/convex/domains/pipelines/piRuntime.ts @@ -412,6 +412,7 @@ export async function runPiOrAiSdkCompletion(args: PiRunArgs): Promise = {}) { + return { + _id: "run1", + entitySlug: "Anthropic", + goal: "Build a diligence brief.", + status: "running", + currentCheckpoint: 0, + totalCheckpoints: 8, + thinkingBudgetTokens: 40_000, + thinkingTokensUsed: 0, + modelName: "claude-sonnet-4.6", + ...over, + }; +} + +async function advance(run: Record, outputTokens: number) { + const requests: any[] = []; + vi.stubGlobal( + "fetch", + vi.fn(async (_url: string, init: { body: string }) => { + requests.push(JSON.parse(init.body)); + return new Response( + JSON.stringify({ + model: "claude-sonnet-4-6", + // display omitted: the thinking block carries no text + content: [{ type: "thinking", thinking: "", signature: "sig" }, { type: "text", text: answer }], + usage: { input_tokens: 500, output_tokens: outputTokens }, + }), + { status: 200 }, + ); + }), + ); + const mutations: any[] = []; + const ctx = { + runQuery: vi.fn(async () => ({ run, checkpoints: [] })), + runMutation: vi.fn(async (_ref: unknown, args: any) => { + mutations.push(args); + return null; + }), + scheduler: { runAfter: vi.fn(async () => null) }, + }; + const result = await (advanceRun as any)._handler(ctx, { runId: "run1" }); + return { result, requests, mutations }; +} + +describe("extendedThinking.advanceRun", () => { + beforeEach(() => { + vi.stubEnv("ANTHROPIC_API_KEY", "test-key-not-real"); + }); + afterEach(() => { + vi.unstubAllGlobals(); + vi.unstubAllEnvs(); + }); + + it("sends adaptive thinking, no sampling keys and the wire model id", async () => { + const { requests } = await advance(runRow(), 5_000); + expect(requests).toHaveLength(1); + const body = requests[0]; + expect(body.model).toBe("claude-sonnet-4-6"); + expect(body.thinking).toEqual({ type: "adaptive" }); + expect(body.thinking).not.toHaveProperty("budget_tokens"); + for (const key of ["temperature", "top_p", "top_k"]) expect(body, key).not.toHaveProperty(key); + expect(body.max_tokens).toBe(14_000); + }); + + it("charges thinking from usage.output_tokens even though the thinking text is omitted", async () => { + const { result, mutations } = await advance(runRow(), 5_000); + const expected = 5_000 - Math.floor(answer.length / 4); + const recorded = mutations.find((m) => "promptHash" in m); + expect(recorded.thinkingTokens).toBe(expected); + expect(expected).toBeGreaterThan(4_800); + // Before: the estimate came from the empty thinking text, so this was 0 and the budget never went down. + const status = mutations.find((m) => "thinkingTokensUsedDelta" in m); + expect(status.thinkingTokensUsedDelta).toBe(expected); + expect(result.status).toBe("scheduled_next"); + }); + + it("stops the run once the charged thinking reaches the budget", async () => { + const { result } = await advance(runRow({ thinkingBudgetTokens: 6_000, thinkingTokensUsed: 2_000 }), 5_000); + expect(result.status).toBe("completed"); + expect(result.reason).toMatch(/thinking token budget exhausted/); + }); + + it("with 2,500 thinking tokens left the cap is 4,000 + 2,500: the bound is on total output, not thinking alone", async () => { + const { requests } = await advance(runRow({ thinkingBudgetTokens: 40_000, thinkingTokensUsed: 37_500 }), 6_500); + expect(requests[0].max_tokens).toBe(6_500); + }); +}); diff --git a/backend/convex/domains/product/extendedThinking.ts b/backend/convex/domains/product/extendedThinking.ts index 9748393bf..811340b10 100644 --- a/backend/convex/domains/product/extendedThinking.ts +++ b/backend/convex/domains/product/extendedThinking.ts @@ -45,6 +45,7 @@ import { api, internal } from "../../_generated/api"; import { buildCheckpointPrompt, buildCheckpointRequestBody, + estimateThinkingTokens, parseCheckpointResponse, shouldContinue, MAX_CHECKPOINTS, @@ -378,13 +379,10 @@ export const advanceRun = internalAction({ const textBlock = envelope.content?.find((b) => b.type === "text"); rawText = textBlock?.text ?? null; outputTokens = envelope.usage?.output_tokens ?? 0; - // Anthropic returns total tokens; extended thinking tokens are - // included in output_tokens for billing. We approximate - // thinkingTokens from the length of the 'thinking' block. - const thinkingBlock = envelope.content?.find((b) => b.type === "thinking"); - thinkingTokens = thinkingBlock?.thinking - ? Math.ceil(thinkingBlock.thinking.length / 4) - : 0; + // output_tokens is thinking + answer with no split, and the thinking + // block's text is omitted by default on Sonnet 5.x / Opus 4.7+, so its + // length cannot be used. Charge output_tokens minus the answer. + thinkingTokens = estimateThinkingTokens(outputTokens, rawText ?? ""); if (!rawText) errorMessage = "no text block in anthropic response"; } catch (err) { errorMessage = `anthropic envelope parse: ${ diff --git a/backend/convex/domains/research/dailyBriefPersonalOverlay.ts b/backend/convex/domains/research/dailyBriefPersonalOverlay.ts index aaa060a54..8df91f3ca 100644 --- a/backend/convex/domains/research/dailyBriefPersonalOverlay.ts +++ b/backend/convex/domains/research/dailyBriefPersonalOverlay.ts @@ -23,7 +23,8 @@ import { anthropicText, } from "../../../../shared/llm/modelCatalog"; -async function generateWithProvider( +// Exported for providerCopies.requestShape.test.ts (SPEC-v9 A.0.6); not a Convex function. +export async function generateWithProvider( modelInput: string, systemPrompt: string, userPrompt: string, diff --git a/backend/convex/domains/research/dailyBriefWorker.ts b/backend/convex/domains/research/dailyBriefWorker.ts index bf074131c..1d6d2c322 100644 --- a/backend/convex/domains/research/dailyBriefWorker.ts +++ b/backend/convex/domains/research/dailyBriefWorker.ts @@ -22,7 +22,8 @@ import { anthropicText, } from "../../../../shared/llm/modelCatalog"; -async function generateWithProvider( +// Exported for providerCopies.requestShape.test.ts (SPEC-v9 A.0.6); not a Convex function. +export async function generateWithProvider( modelInput: string, systemPrompt: string, userPrompt: string, diff --git a/backend/convex/domains/research/dealFlow.ts b/backend/convex/domains/research/dealFlow.ts index bf92905b3..7c85c2801 100644 --- a/backend/convex/domains/research/dealFlow.ts +++ b/backend/convex/domains/research/dealFlow.ts @@ -43,7 +43,8 @@ type DealInput = { regulatory?: { fdaStatus?: string; patents?: string[]; papers?: string[] }; }; -async function generateWithProvider( +// Exported for providerCopies.requestShape.test.ts (SPEC-v9 A.0.6); not a Convex function. +export async function generateWithProvider( modelInput: string, systemPrompt: string, userPrompt: string, diff --git a/backend/convex/domains/research/modelComparison.ts b/backend/convex/domains/research/modelComparison.ts index 750a4af3a..61f1fca8c 100644 --- a/backend/convex/domains/research/modelComparison.ts +++ b/backend/convex/domains/research/modelComparison.ts @@ -17,7 +17,8 @@ import { const CACHE_TTL_MS = 12 * 60 * 60 * 1000; -async function generateWithProvider( +// Exported for providerCopies.requestShape.test.ts (SPEC-v9 A.0.6); not a Convex function. +export async function generateWithProvider( modelInput: string, systemPrompt: string, userPrompt: string, diff --git a/backend/convex/domains/research/paperDetails.ts b/backend/convex/domains/research/paperDetails.ts index 600ae3e73..5d6f4e068 100644 --- a/backend/convex/domains/research/paperDetails.ts +++ b/backend/convex/domains/research/paperDetails.ts @@ -18,7 +18,8 @@ import { linkupSearch } from "../../tools/media/linkupSearch"; const CACHE_TTL_MS = 24 * 60 * 60 * 1000; -async function generateWithProvider( +// Exported for providerCopies.requestShape.test.ts (SPEC-v9 A.0.6); not a Convex function. +export async function generateWithProvider( modelInput: string, systemPrompt: string, userPrompt: string, diff --git a/backend/convex/domains/research/repoScout.ts b/backend/convex/domains/research/repoScout.ts index 999508098..f066f8e47 100644 --- a/backend/convex/domains/research/repoScout.ts +++ b/backend/convex/domains/research/repoScout.ts @@ -30,7 +30,8 @@ type MoatAnalysis = { moatRisks?: string[]; }; -async function generateWithProvider( +// Exported for providerCopies.requestShape.test.ts (SPEC-v9 A.0.6); not a Convex function. +export async function generateWithProvider( modelInput: string, systemPrompt: string, userPrompt: string, diff --git a/backend/convex/domains/research/stackImpact.ts b/backend/convex/domains/research/stackImpact.ts index 70c45233b..9b25148bf 100644 --- a/backend/convex/domains/research/stackImpact.ts +++ b/backend/convex/domains/research/stackImpact.ts @@ -29,7 +29,8 @@ type ReaderContentResult = { const CACHE_TTL_MS = 12 * 60 * 60 * 1000; -async function generateWithProvider( +// Exported for providerCopies.requestShape.test.ts (SPEC-v9 A.0.6); not a Convex function. +export async function generateWithProvider( modelInput: string, systemPrompt: string, userPrompt: string, diff --git a/backend/convex/domains/research/strategyMetrics.ts b/backend/convex/domains/research/strategyMetrics.ts index 640bf2b83..15b39b475 100644 --- a/backend/convex/domains/research/strategyMetrics.ts +++ b/backend/convex/domains/research/strategyMetrics.ts @@ -17,7 +17,8 @@ import { const CACHE_TTL_MS = 12 * 60 * 60 * 1000; -async function generateWithProvider( +// Exported for providerCopies.requestShape.test.ts (SPEC-v9 A.0.6); not a Convex function. +export async function generateWithProvider( modelInput: string, systemPrompt: string, userPrompt: string, diff --git a/backend/convex/domains/search/fusion/benchmark.requestShape.test.ts b/backend/convex/domains/search/fusion/benchmark.requestShape.test.ts new file mode 100644 index 000000000..17351781c --- /dev/null +++ b/backend/convex/domains/search/fusion/benchmark.requestShape.test.ts @@ -0,0 +1,78 @@ +// @vitest-environment node +/** + * benchmark.evaluateSearch calls its judge over raw fetch (SPEC-v9 A.0.6, A.2.10 PR 1). + * Persona: an engineer running the search-quality benchmark with SEARCH_JUDGE_MODEL set + * to a Claude model after the migration. Claude 4.7+/5.x answer 400 to `temperature`, so + * the Anthropic body must not carry it; the OpenAI and Google bodies still pin + * temperature to 0 for determinism, so removing it only for Claude must not leak. + * The action runs through its real handler against a fake transport; no network, no spend. + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { evaluateSearch } from "./benchmark"; + +const searchPayload = { + payload: { mode: "balanced", sourcesQueried: [], results: [], totalTimeMs: 1, timing: {}, errors: [] }, +}; +const judgeJson = JSON.stringify({ scores: { relevance: 1 }, pass: false, reasoning: "r", issues: [], suggestions: [] }); + +function run(judgeEnv: string) { + vi.stubEnv("SEARCH_JUDGE_MODEL", judgeEnv); + const captured: Array<{ url: string; body: Record }> = []; + vi.stubGlobal( + "fetch", + vi.fn(async (url: string, init: { body: string }) => { + captured.push({ url: String(url), body: JSON.parse(init.body) }); + const body = String(url).includes("anthropic") + ? { content: [{ type: "thinking", thinking: "…" }, { type: "text", text: judgeJson }] } + : String(url).includes("openai") + ? { choices: [{ message: { content: judgeJson } }] } + : { candidates: [{ content: { parts: [{ text: judgeJson }] } }] }; + return new Response(JSON.stringify(body), { status: 200 }); + }), + ); + const ctx = { + runAction: vi.fn(async () => searchPayload), + runMutation: vi.fn(async () => null), + }; + return (evaluateSearch as any)._handler(ctx, { query: "q" }).then((result: any) => ({ result, captured })); +} + +describe("benchmark.evaluateSearch judge request shape", () => { + beforeEach(() => { + vi.stubEnv("ANTHROPIC_API_KEY", "test-key-not-real"); + vi.stubEnv("OPENAI_API_KEY", "test-key-not-real"); + vi.stubEnv("GOOGLE_GENERATIVE_AI_API_KEY", "test-key-not-real"); + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "warn").mockImplementation(() => {}); + }); + afterEach(() => { + vi.unstubAllEnvs(); + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + }); + + it("Claude judge: model is the wire id, no sampling keys, answer read past a thinking block", async () => { + const { result, captured } = await run("claude-sonnet-4.6"); + expect(captured).toHaveLength(1); + expect(captured[0].url).toBe("https://api.anthropic.com/v1/messages"); + expect(captured[0].body.model).toBe("claude-sonnet-4-6"); + for (const key of ["temperature", "top_p", "top_k"]) expect(captured[0].body, key).not.toHaveProperty(key); + expect(captured[0].body.max_tokens).toBeGreaterThanOrEqual(1024); + expect(result.judgeModel).toBe("claude-sonnet-4.6"); + expect(result.judgeResult.reasoning).toBe("r"); + }); + + it("OpenAI judge keeps temperature pinned to 0", async () => { + const { captured } = await run("gpt-5.4"); + expect(captured[0].url).toBe("https://api.openai.com/v1/chat/completions"); + expect(captured[0].body.temperature).toBe(0); + }); + + it("Gemini judge keeps temperature pinned to 0 in generationConfig, with the model in the URL", async () => { + const { captured } = await run("gemini-3.1-pro-preview"); + expect(captured[0].url).toContain("/models/gemini-3.1-pro-preview:generateContent"); + expect(captured[0].body.generationConfig.temperature).toBe(0); + }); +}); diff --git a/backend/convex/domains/social/postDedupAction.ts b/backend/convex/domains/social/postDedupAction.ts index e73306eb3..6da438cfd 100644 --- a/backend/convex/domains/social/postDedupAction.ts +++ b/backend/convex/domains/social/postDedupAction.ts @@ -54,7 +54,8 @@ interface DedupJudgmentResult { /** * Call LLM to judge dedup verdict. */ -async function callDedupJudge(prompt: string): Promise { +// Exported for providerCopies.requestShape.test.ts (SPEC-v9 A.0.6); not a Convex function. +export async function callDedupJudge(prompt: string): Promise { const { model: modelName, provider } = getModelWithFailover( resolveModelAlias(getLlmModel("judge")) ); diff --git a/backend/convex/domains/verification/claimVerificationAction.ts b/backend/convex/domains/verification/claimVerificationAction.ts index f4199e523..6515eeced 100644 --- a/backend/convex/domains/verification/claimVerificationAction.ts +++ b/backend/convex/domains/verification/claimVerificationAction.ts @@ -143,7 +143,8 @@ async function fetchSourceContent( /** * Call LLM-as-a-judge to verify claim against source */ -async function judgeClaimAgainstSource( +// Exported for providerCopies.requestShape.test.ts (SPEC-v9 A.0.6); not a Convex function. +export async function judgeClaimAgainstSource( claimText: string, sourceContent: string ): Promise { diff --git a/backend/convex/providerCopies.requestShape.test.ts b/backend/convex/providerCopies.requestShape.test.ts new file mode 100644 index 000000000..324ef9d59 --- /dev/null +++ b/backend/convex/providerCopies.requestShape.test.ts @@ -0,0 +1,150 @@ +// @vitest-environment node +/** + * The ~16 hand-copied multi-provider helpers (dealFlow, morningDigest, repoScout, ...) each carry + * their own Anthropic branch. SPEC-v9 A.0.6 and A.2.10 PR 1 changed every copy the same way: send + * the hyphenated wire id from the catalog's explicit table (api.anthropic.com answers 404 to the + * dotted alias "claude-sonnet-4.6") and read the answer from the text block (content[0] is a + * thinking block whenever adaptive thinking is on). A test of the shared helper alone cannot + * catch one copy being reverted, so this drives each copy's real function against a fake SDK + * transport and asserts the outgoing request and the returned text. + * + * Persona: an operator whose catalog default for these tasks is a Claude model; a reply that + * starts with a thinking block must still yield the answer, and the request must name a model + * Anthropic accepts, with no sampling keys. No network, no spend. + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const create = vi.hoisted(() => vi.fn()); + +vi.mock("@anthropic-ai/sdk", () => ({ + default: class FakeAnthropic { + messages = { create }; + }, +})); +// Every copy asks the catalog for its task model; put a Claude alias there so each copy takes its Anthropic branch. +vi.mock("../../shared/llm/modelCatalog", async (importOriginal) => ({ + ...(await importOriginal()), + getLlmModel: () => "claude-sonnet-4.6", +})); +// entityInsights tries pi-ai first and only then its legacy provider branch. +vi.mock("./domains/pipelines/piRuntime", () => ({ + runPiOrAiSdkCompletion: vi.fn(async () => { + throw new Error("pi-ai unavailable in this test"); + }), +})); +vi.mock("@convex-dev/auth/server", () => ({ getAuthUserId: vi.fn(async () => "user1") })); + +import { generateWithProvider as morningDigest } from "./domains/ai/morningDigest"; +import { generateWithProvider as entityInsights } from "./domains/knowledge/entityInsights"; +import { generateWithProvider as dailyBriefPersonalOverlay } from "./domains/research/dailyBriefPersonalOverlay"; +import { generateWithProvider as dailyBriefWorker } from "./domains/research/dailyBriefWorker"; +import { generateWithProvider as dealFlow } from "./domains/research/dealFlow"; +import { generateWithProvider as modelComparison } from "./domains/research/modelComparison"; +import { generateWithProvider as paperDetails } from "./domains/research/paperDetails"; +import { generateWithProvider as repoScout } from "./domains/research/repoScout"; +import { generateWithProvider as stackImpact } from "./domains/research/stackImpact"; +import { generateWithProvider as strategyMetrics } from "./domains/research/strategyMetrics"; +import { callDedupJudge } from "./domains/social/postDedupAction"; +import { judgeClaimAgainstSource } from "./domains/verification/claimVerificationAction"; +import { handleChatResponse } from "./domains/agents/fastAgentChat"; +import { generateResponse } from "./domains/ai/ai"; +import { generateForDocument } from "./tags_actions"; + +const ANSWER = "the answer"; +const thinkingThen = (text: string) => ({ + content: [ + { type: "thinking", thinking: "…", signature: "sig" }, + { type: "text", text }, + ], + usage: { input_tokens: 3, output_tokens: 4 }, +}); + +function expectAnthropicRequest(maxTokens?: number) { + expect(create).toHaveBeenCalledTimes(1); + const body = create.mock.calls[0][0]; + expect(body.model).toBe("claude-sonnet-4-6"); // the wire id, never "claude-sonnet-4.6" + for (const key of ["temperature", "top_p", "top_k"]) expect(body, key).not.toHaveProperty(key); + expect(body.messages.at(-1).role).toBe("user"); // no prefill + if (maxTokens !== undefined) expect(body.max_tokens).toBe(maxTokens); + return body; +} + +describe("Anthropic branch of every multi-provider copy", () => { + beforeEach(() => { + vi.stubEnv("ANTHROPIC_API_KEY", "sk-ant-test-key-not-real"); + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "warn").mockImplementation(() => {}); + vi.spyOn(console, "error").mockImplementation(() => {}); + create.mockReset(); + create.mockResolvedValue(thinkingThen(ANSWER)); + }); + afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + }); + + const copies: Array<[string, (model: string, system: string, user: string, max: number) => Promise]> = [ + ["morningDigest", morningDigest], + ["entityInsights", entityInsights], + ["dailyBriefPersonalOverlay", dailyBriefPersonalOverlay], + ["dailyBriefWorker", dailyBriefWorker], + ["dealFlow", dealFlow], + ["modelComparison", modelComparison], + ["paperDetails", paperDetails], + ["repoScout", repoScout], + ["stackImpact", stackImpact], + ["strategyMetrics", strategyMetrics], + ]; + + it.each(copies)("%s: wire id, no sampling keys, answer read past the thinking block", async (_name, generateWithProvider) => { + const text = await generateWithProvider("claude-sonnet-4.6", "sys", "usr", 321); + const body = expectAnthropicRequest(321); + expect(body.system).toBe("sys"); + expect(body.messages).toEqual([{ role: "user", content: "usr" }]); + expect(text).toBe(ANSWER); + }); + + it("postDedupAction.callDedupJudge reads the verdict past the thinking block", async () => { + create.mockResolvedValue(thinkingThen(JSON.stringify({ verdict: "NEW", confidence: 0.9, reasoning: "r", keyDifferences: [] }))); + const result = await callDedupJudge("compare these"); + expectAnthropicRequest(300); + // A reverted content[0] read gives "Empty response", which the judge turns into INCONCLUSIVE. + expect(result.verdict).toBe("NEW"); + }); + + it("claimVerificationAction.judgeClaimAgainstSource reads the verdict past the thinking block", async () => { + create.mockResolvedValue(thinkingThen(JSON.stringify({ verdict: "supported", confidence: 0.8, explanation: "e" }))); + const result = await judgeClaimAgainstSource("claim", "source"); + expectAnthropicRequest(200); + expect(result.verdict).toBe("supported"); + }); + + it("fastAgentChat.handleChatResponse returns the text block", async () => { + const ctx = { runQuery: vi.fn(async () => null), runMutation: vi.fn(async () => null) }; + const text = await handleChatResponse(ctx as never, { message: "hello", userId: "u1" as never, model: "anthropic", fastMode: false }); + expectAnthropicRequest(2000); + expect(text).toBe(ANSWER); + }); + + it("ai.generateResponse returns the text block", async () => { + const ctx = { + runQuery: vi.fn(async () => []), + runMutation: vi.fn(async () => null), + }; + const result = await (generateResponse as any)._handler(ctx, { userMessage: "hello" }); + expectAnthropicRequest(2000); + expect(result.message).toBe(ANSWER); + }); + + it("tags_actions.generateForDocument parses tags from the text block", async () => { + create.mockResolvedValue(thinkingThen(JSON.stringify([{ name: "React", kind: "entity", importance: 0.9 }]))); + const ctx = { + runQuery: vi.fn(async () => "A document about the React web framework and its ecosystem of tooling."), + runMutation: vi.fn(async (_ref: unknown, args: any) => args?.tags ?? null), + }; + const result = await (generateForDocument as any)._handler(ctx, { documentId: "doc1" }); + expectAnthropicRequest(500); + expect(result).toMatchObject({ success: true, tagsGenerated: 1 }); + }); +}); diff --git a/packages/mcp-local/src/__tests__/anthropicRequestShape.test.ts b/packages/mcp-local/src/__tests__/anthropicRequestShape.test.ts new file mode 100644 index 000000000..b6fe5a736 --- /dev/null +++ b/packages/mcp-local/src/__tests__/anthropicRequestShape.test.ts @@ -0,0 +1,68 @@ +/** + * llmTools.call_llm and researchWritingTools.polish_academic_text reach Anthropic through + * the SDK (SPEC-v9 A.0.6, A.2.10 PR 1). Persona: a user of the published MCP server who + * has only ANTHROPIC_API_KEY set and asks for a low-temperature polish. Claude 4.7+/5.x + * answer 400 to `temperature`, so the Messages call must not carry it even though the + * caller supplied one, and the answer must be read from the text block (content[0] is a + * thinking block when adaptive thinking is on). The SDK is the fake transport; no + * network, no spend. The tools are driven through their real handlers. + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const create = vi.hoisted(() => vi.fn()); +vi.mock("@anthropic-ai/sdk", () => ({ + default: class FakeAnthropic { + messages = { create }; + }, +})); + +import { llmTools } from "../tools/llmTools.js"; +import { researchWritingTools } from "../tools/researchWritingTools.js"; + +const thinkingThenText = (text: string) => ({ + content: [ + { type: "thinking", thinking: "…", signature: "sig" }, + { type: "text", text }, + ], + usage: { input_tokens: 3, output_tokens: 4 }, +}); + +function handler(tools: Array<{ name: string; handler: (a: any) => Promise }>, name: string) { + const tool = tools.find((t) => t.name === name); + if (!tool) throw new Error(`tool ${name} not found`); + return tool.handler; +} + +describe("mcp-local Anthropic provider request shape", () => { + beforeEach(() => { + for (const key of ["GEMINI_API_KEY", "GOOGLE_AI_API_KEY", "OPENAI_API_KEY"]) vi.stubEnv(key, ""); + vi.stubEnv("ANTHROPIC_API_KEY", "test-key-not-real"); + create.mockReset(); + create.mockResolvedValue(thinkingThenText("the answer")); + }); + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it("call_llm: no temperature although the caller passed one; text read past a thinking block", async () => { + const result = await handler(llmTools as never, "call_llm")({ prompt: "hi", temperature: 0.9, maxTokens: 777 }); + expect(create).toHaveBeenCalledTimes(1); + const body = create.mock.calls[0][0]; + expect(body.model).toBe("claude-haiku-4-5-20251001"); // PR 1 moves no ids + for (const key of ["temperature", "top_p", "top_k"]) expect(body, key).not.toHaveProperty(key); + expect(body.max_tokens).toBe(777); + expect(body.messages).toEqual([{ role: "user", content: "hi" }]); + expect(result.response).toBe("the answer"); + expect(result.provider).toBe("anthropic"); + }); + + it("polish_academic_text: no temperature although the tool passes 0.2; text read past a thinking block", async () => { + const result = await handler(researchWritingTools as never, "polish_academic_text")({ text: "We propose a method." }); + expect(create).toHaveBeenCalledTimes(1); + const body = create.mock.calls[0][0]; + for (const key of ["temperature", "top_p", "top_k"]) expect(body, key).not.toHaveProperty(key); + expect(body.max_tokens).toBe(4096); + expect(result.polishedText).toBe("the answer"); + }); +}); diff --git a/scripts/__tests__/comprehensiveEvalModels.test.ts b/scripts/__tests__/comprehensiveEvalModels.test.ts index cbd8cb001..15f338e2b 100644 --- a/scripts/__tests__/comprehensiveEvalModels.test.ts +++ b/scripts/__tests__/comprehensiveEvalModels.test.ts @@ -1,24 +1,32 @@ // @vitest-environment node /** * run-comprehensive-eval.ts is how a model candidate gets measured - * (`--models --suite full --judge ...`, SPEC-v9 A.2.10). Two ways it used + * (`--models --suite full --judge ...`, SPEC-v9 A.2.10). Ways it used * to mislead: a `--models` value the resolver did not know was silently served by the - * default model (so "candidate X" measured kimi-k2.6), and its own retry could switch - * models mid-row. These helpers are what makes the run abort and the row disqualify. + * default model (so "candidate X" measured kimi-k2.6), a legacy alias such as + * gpt-4o-mini was accepted and then every row was disqualified only after the paid + * run finished, its own retry could switch models mid-row, and a row from a Convex + * deployment built before the identity fields existed passed with no identity at all. + * These helpers make the run abort before spending and the row disqualify. * Pure functions; no Convex, no network, no spend. */ import { describe, expect, it } from "vitest"; -import { identityViolations, unresolvedModels } from "../run-comprehensive-eval"; +import { identityViolations, nonCanonicalModels, runSingleEpisode } from "../run-comprehensive-eval"; -describe("unresolvedModels (--models must resolve or the run aborts)", () => { - it("accepts approved ids and legacy aliases", () => { - expect(unresolvedModels(["kimi-k2.6", "claude-sonnet-4.6", "gpt-4o-mini", "nemotron-3-super-free"])).toEqual([]); +describe("nonCanonicalModels (--models must be an approved id exactly, or the run aborts before spending)", () => { + it("accepts approved ids", () => { + expect(nonCanonicalModels(["kimi-k2.6", "claude-sonnet-4.6", "gpt-5.4-mini", "nemotron-3-super-free"])).toEqual([]); }); it("flags ids the resolver does not know, including the removed minimax-m2.1", () => { - expect(unresolvedModels(["kimi-k2.6", "minimax-m2.1", "claude-opus-9"])).toEqual(["minimax-m2.1", "claude-opus-9"]); + expect(nonCanonicalModels(["kimi-k2.6", "minimax-m2.1", "claude-opus-9"])).toEqual(["minimax-m2.1", "claude-opus-9"]); + }); + + it("flags a legacy alias, which resolves to a different id and would disqualify every row after the paid run", () => { + // gpt-4o-mini resolves to gpt-5.4-mini; the previous check (resolveModelAlias(m) === null) accepted it. + expect(nonCanonicalModels(["gpt-4o-mini", "GPT-5.4-MINI"])).toEqual(["gpt-4o-mini", "GPT-5.4-MINI"]); }); }); @@ -31,12 +39,53 @@ describe("identityViolations (a row is evidence only for the model it requested) }); it("disqualifies a row whose resolved id differs from the request", () => { - const bad = row({ resolvedModelId: "kimi-k2.6" }); - expect(identityViolations([row({ resolvedModelId: "claude-sonnet-4.6" }), bad])).toEqual([bad]); + const bad = row({ resolvedModelId: "kimi-k2.6", fallbackUsed: false }); + expect(identityViolations([row({ resolvedModelId: "claude-sonnet-4.6", fallbackUsed: false }), bad])).toEqual([bad]); }); it("disqualifies a row whose own retry switched models", () => { const bad = row({ resolvedModelId: "claude-sonnet-4.6", fallbackUsed: true }); expect(identityViolations([bad])).toEqual([bad]); }); + + it("disqualifies a passing row with no resolved identity (a deployment built before the identity fields)", () => { + // Before: a row without resolvedModelId counted as clean, so the run exited 0 with no per-row identity. + const old = row({}); + const empty = row({ resolvedModelId: "", fallbackUsed: false }); + expect(identityViolations([old, empty])).toEqual([old, empty]); + }); + + it("disqualifies a row with an identity but no explicit fallback status", () => { + const bad = row({ resolvedModelId: "claude-sonnet-4.6" }); + expect(identityViolations([bad])).toEqual([bad]); + }); +}); + +describe("runSingleEpisode keeps identity on an error row", () => { + const scenario = { id: "s1", name: "s", query: "q", expectedPersona: "p", evalSuite: "core" } as never; + + it("records resolvedModelId and fallbackUsed when the retry throws after the first attempt answered", async () => { + let calls = 0; + const client = { + action: async () => { + calls += 1; + if (calls === 1) return { resolvedModelId: "kimi-k2.6", runs: [{ ok: false, responseText: "", failureReasons: [] }] }; + throw new Error("boom"); + }, + } as never; + const row = await runSingleEpisode(client, "secret", "kimi-k2.6", scenario, "core", false, "kimi-k2.6"); + expect(row.error).toBe("boom"); + // The first attempt's identity and the retry's model switch survive the exception. + expect(row.resolvedModelId).toBe("kimi-k2.6"); + expect(row.fallbackUsed).toBe(true); + expect(identityViolations([row])).toEqual([row]); + }); + + it("reports an explicit fallbackUsed=false even when the very first call throws", async () => { + const client = { action: async () => { throw new Error("down"); } } as never; + const row = await runSingleEpisode(client, "secret", "kimi-k2.6", scenario, "core", false, "kimi-k2.6"); + expect(row.fallbackUsed).toBe(false); + expect(row.resolvedModelId).toBeUndefined(); + expect(identityViolations([row])).toEqual([row]); + }); }); diff --git a/scripts/__tests__/toolGenRequestShape.test.ts b/scripts/__tests__/toolGenRequestShape.test.ts index 15bd3a402..18a280259 100644 --- a/scripts/__tests__/toolGenRequestShape.test.ts +++ b/scripts/__tests__/toolGenRequestShape.test.ts @@ -12,6 +12,8 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; // @ts-expect-error - plain .mjs script without types import { buildClaudeRequest, callClaude, claudeText } from "../tool-gen/analyze-gap.mjs"; +// @ts-expect-error - plain .mjs script without types +import * as generateTool from "../tool-gen/generate-tool.mjs"; describe("analyze-gap.mjs Anthropic request", () => { beforeEach(() => { @@ -65,3 +67,48 @@ describe("analyze-gap.mjs Anthropic request", () => { // Importing the module at the top of this file is itself the guard test: an unguarded // main() would read /tmp/daily-brief.json and call process.exit(1) inside the worker. }); + +// generate-tool.mjs is the second raw-fetch Anthropic call of the daily job. It builds its +// own body (a longer output cap and a different model than analyze-gap), so it needs its +// own request-shape test rather than leaning on analyze-gap's. +describe("generate-tool.mjs Anthropic request", () => { + beforeEach(() => { + vi.stubEnv("ANTHROPIC_API_KEY", "test-key-not-real"); + }); + afterEach(() => { + vi.unstubAllGlobals(); + vi.unstubAllEnvs(); + }); + + it("builds a body with no sampling keys, no prefill and room to think", () => { + const body = generateTool.buildClaudeRequest("system text", "user text"); + // PR 1 moves no ids; PR 2 changes this literal to claude-sonnet-5-5 with its own update. + expect(body.model).toBe("claude-sonnet-4-6"); + for (const key of ["temperature", "top_p", "top_k"]) expect(body).not.toHaveProperty(key); + expect(body.max_tokens).toBe(4096); + expect(body.messages).toEqual([{ role: "user", content: "user text" }]); + }); + + it("sends exactly that body and returns the text block, not content[0]", async () => { + const seen: Array<{ url: string; body: any }> = []; + vi.stubGlobal( + "fetch", + vi.fn(async (url: string, init: { body: string }) => { + seen.push({ url, body: JSON.parse(init.body) }); + return new Response( + JSON.stringify({ content: [{ type: "thinking", thinking: "x", signature: "s" }, { type: "text", text: '{"toolObjectCode":"{}"}' }] }), + { status: 200 }, + ); + }), + ); + expect(await generateTool.callClaude("sys", "usr")).toBe('{"toolObjectCode":"{}"}'); + expect(seen).toHaveLength(1); + expect(seen[0].url).toBe("https://api.anthropic.com/v1/messages"); + expect(seen[0].body).toEqual(generateTool.buildClaudeRequest("sys", "usr")); + }); + + it("surfaces an HTTP failure instead of returning an empty answer", async () => { + vi.stubGlobal("fetch", vi.fn(async () => new Response("bad request", { status: 400 }))); + await expect(generateTool.callClaude("sys", "usr")).rejects.toThrow(/Claude API 400/); + }); +}); diff --git a/scripts/eval-harness/agentInvoker.ts b/scripts/eval-harness/agentInvoker.ts index 43e0fe8bb..0b5ef3876 100644 --- a/scripts/eval-harness/agentInvoker.ts +++ b/scripts/eval-harness/agentInvoker.ts @@ -14,7 +14,7 @@ */ import { createHash } from "node:crypto"; -import type { RunConfig, RunTelemetry, Scorecard, ToolCallRecord, EvalTask } from "./types.js"; +import type { RunConfig, RunTelemetry, Scorecard, ToolCallRecord, ToolRequestRecord, EvalTask } from "./types.js"; export type InvokerBackend = "stub" | "anthropic" | "openai"; @@ -130,7 +130,10 @@ async function invokeAnthropic( const startedAt = new Date().toISOString(); const startMs = Date.now(); + // No tool runs in this harness and no result is sent back, so tool_use blocks are + // only requests: they are recorded apart from `toolCalls` and earn no score. const toolCalls: ToolCallRecord[] = []; + const requestedTools: ToolRequestRecord[] = []; let error: string | undefined; let outputHash = ""; let tokenCount = 0; @@ -169,21 +172,22 @@ async function invokeAnthropic( .join(""); for (const block of blocks) { if (block.type !== "tool_use" || !block.name) continue; - toolCalls.push({ + requestedTools.push({ toolName: block.name, - durationMs: 0, - success: true, inputHash: sha256(JSON.stringify(block.input ?? null)), - outputHash: "", - followedQuickRef: null, - timestamp: new Date().toISOString(), }); } outputHash = sha256(text); tokenCount = (data.usage?.input_tokens ?? 0) + (data.usage?.output_tokens ?? 0); // The model string the provider returned, not the one requested (SPEC-v9 A.0.3). echoedModel = data.model; - if (data.stop_reason === "refusal") error = "Anthropic stop_reason: refusal"; + const problems: string[] = []; + if (data.stop_reason === "refusal") problems.push("Anthropic stop_reason: refusal"); + if (data.stop_reason === "tool_use") { + problems.push(`Anthropic stop_reason: tool_use; ${requestedTools.length} tool(s) requested, none executed (unfinished turn)`); + } + if (!echoedModel) problems.push("Anthropic response carries no model string, so the run has no identity evidence"); + if (problems.length > 0) error = problems.join("; "); } } catch (err) { error = err instanceof Error ? (err.name === "AbortError" ? "request timeout" : err.message) : String(err); @@ -220,6 +224,7 @@ async function invokeAnthropic( completedAt, scorecard, toolCalls, + requestedTools, verificationCycles: [], outputHash, echoedModel, diff --git a/scripts/eval-harness/evalHarness.test.ts b/scripts/eval-harness/evalHarness.test.ts index 27006e7ef..f96ffa6de 100644 --- a/scripts/eval-harness/evalHarness.test.ts +++ b/scripts/eval-harness/evalHarness.test.ts @@ -79,26 +79,55 @@ describe("agentInvoker Anthropic backend", () => { expect(body).not.toHaveProperty("tool_choice"); }); - it("reads blocks by type: text after a thinking block, tool_use recorded as a tool call", async () => { + it("reads blocks by type: text after a thinking block; a tool_use is a request, not an executed call", async () => { const { fetchImpl } = fakeMessagesTransport({ model: "m", + stop_reason: "end_turn", content: [ { type: "thinking", thinking: "…", signature: "s" }, { type: "text", text: "plan" }, - { type: "tool_use", id: "t1", name: "convex_audit_schema", input: { path: "schema.ts" } }, + { type: "tool_use", id: "t1", name: "convex_search_gotchas", input: { path: "schema.ts" } }, ], usage: { input_tokens: 1, output_tokens: 1 }, }); - const telemetry = await invokeAgent(config("mcp_lite"), task, { + const telemetry = await invokeAgent(config("mcp_core"), task, { backend: "anthropic", apiKey: "k", model: "candidate-model", fetchImpl, }); - expect(telemetry.toolCalls.map((c) => c.toolName)).toEqual(["convex_audit_schema"]); + // No tool runs in this harness: the request is recorded apart and earns no score. + // (Before: it was recorded as a successful ToolCallRecord and set knowledgeReuseRate to 1.) + expect(telemetry.requestedTools?.map((c) => c.toolName)).toEqual(["convex_search_gotchas"]); + expect(telemetry.toolCalls).toEqual([]); + expect(telemetry.scorecard.efficiency.toolCallCount).toBe(0); + expect(telemetry.scorecard.compounding.knowledgeReuseRate).toBe(0); expect(telemetry.outputHash).toMatch(/^[0-9a-f]{64}$/); }); + it("marks an unfinished tool_use turn as an error", async () => { + const { fetchImpl } = fakeMessagesTransport({ + model: "m", + stop_reason: "tool_use", + content: [{ type: "tool_use", id: "t1", name: "convex_audit_schema", input: {} }], + usage: { input_tokens: 1, output_tokens: 1 }, + }); + const telemetry = await invokeAgent(config("mcp_lite"), task, { backend: "anthropic", apiKey: "k", model: "candidate-model", fetchImpl }); + expect(telemetry.error).toMatch(/tool_use.*none executed/); + expect(telemetry.toolCalls).toEqual([]); + }); + + it("marks a response that echoes no model as an error: it is not identity evidence", async () => { + const { fetchImpl } = fakeMessagesTransport({ + stop_reason: "end_turn", + content: [{ type: "text", text: "done" }], + usage: { input_tokens: 1, output_tokens: 1 }, + }); + const telemetry = await invokeAgent(config("bare"), task, { backend: "anthropic", apiKey: "k", model: "candidate-model", fetchImpl }); + expect(telemetry.error).toMatch(/no model string/); + expect(telemetry.echoedModel).toBeUndefined(); + }); + it("reports an HTTP failure and a refusal as errors, never as a clean run", async () => { const failed = await invokeAgent(config("bare"), task, { backend: "anthropic", @@ -138,6 +167,7 @@ describe("runner --tier / --seeds", () => { afterEach(() => { rmSync(dir, { recursive: true, force: true }); vi.unstubAllEnvs(); + vi.unstubAllGlobals(); vi.restoreAllMocks(); }); @@ -180,6 +210,42 @@ describe("runner --tier / --seeds", () => { expect(readAll()).toHaveLength(0); }); + it("exits non-zero when a real backend answers HTTP 400 on every run, and still writes the error rows", async () => { + // Before: ten HTTP 400 responses wrote ten error rows and the runner exited 0, so the + // declared migrationEvalCommand could pass without a single successful provider call. + vi.stubEnv("EVAL_BACKEND", "anthropic"); + vi.stubEnv("EVAL_MODEL", "candidate-under-test"); + vi.stubEnv("ANTHROPIC_API_KEY", "test-key-not-real"); + const fetchSpy = vi.fn(async () => new Response(JSON.stringify({ error: { type: "invalid_request_error" } }), { status: 400 })); + vi.stubGlobal("fetch", fetchSpy); + const tier1 = EVAL_DATASET.tasks.filter((t) => t.tier.startsWith("tier1")); + + expect(await main(["--tier", "tier1", "--seeds", "1"], dir)).toBe(1); + + expect(fetchSpy).toHaveBeenCalledTimes(tier1.length); + const rows = readAll(); + expect(rows).toHaveLength(tier1.length); + for (const { telemetry } of rows) expect(telemetry.error).toMatch(/HTTP 400/); + vi.unstubAllGlobals(); + }); + + it("exits 0 when every real-backend run completes cleanly", async () => { + vi.stubEnv("EVAL_BACKEND", "anthropic"); + vi.stubEnv("EVAL_MODEL", "candidate-under-test"); + vi.stubEnv("ANTHROPIC_API_KEY", "test-key-not-real"); + vi.stubGlobal( + "fetch", + vi.fn(async () => + new Response( + JSON.stringify({ model: "candidate-under-test", stop_reason: "end_turn", content: [{ type: "text", text: "ok" }], usage: { input_tokens: 1, output_tokens: 1 } }), + { status: 200 }, + ), + ), + ); + expect(await main(["--task", task.id, "--mode", "bare", "--seed", "1"], dir)).toBe(0); + vi.unstubAllGlobals(); + }); + it("rejects an empty tier selection and a bad --seeds", async () => { vi.stubEnv("EVAL_BACKEND", "stub"); expect(await main(["--tier", "tier9", "--seeds", "1"], dir)).toBe(1); diff --git a/scripts/eval-harness/runner.ts b/scripts/eval-harness/runner.ts index e4fe21315..4308874b5 100644 --- a/scripts/eval-harness/runner.ts +++ b/scripts/eval-harness/runner.ts @@ -164,7 +164,7 @@ async function runOne( seed: number, model: string, resultsDir: string, -): Promise { +): Promise { const config: RunConfig = { taskId: task.id, agentMode: mode, @@ -180,6 +180,7 @@ async function runOne( writeFileSync(outPath, JSON.stringify(telemetry, null, 2)); console.log(`Result written to: ${outPath}`); console.log(`Composite score: ${computeCompositeScore(telemetry.scorecard).toFixed(3)}`); + return telemetry; } /** Returns the process exit code. `resultsDir` is a parameter so tests can use a temp dir. */ @@ -271,11 +272,21 @@ export async function main(args: string[], resultsDir: string = RESULTS_DIR): Pr seeds = [seedIdx !== -1 ? parseInt(args[seedIdx + 1], 10) : 1]; } + const failed: string[] = []; for (const task of tasks) { for (const seed of seeds) { - await runOne(task, mode, seed, model, resultsDir); + const telemetry = await runOne(task, mode, seed, model, resultsDir); + if (telemetry.error) failed.push(`${task.id} s${seed}: ${telemetry.error}`); } } + // A real backend exits non-zero when any run failed, so a migration eval cannot + // pass on error rows. The stub backend always reports its STUB marker; that smoke + // path stays exit 0. + if (backend.backend !== "stub" && failed.length > 0) { + console.error(`${failed.length} run(s) failed:`); + for (const line of failed) console.error(` ${line}`); + return 1; + } return 0; } diff --git a/scripts/eval-harness/types.ts b/scripts/eval-harness/types.ts index 90684adc1..f160bd891 100644 --- a/scripts/eval-harness/types.ts +++ b/scripts/eval-harness/types.ts @@ -79,7 +79,10 @@ export interface RunTelemetry { startedAt: string; completedAt: string; scorecard: Scorecard; + /** Tools that actually ran and returned a result. This harness runs none, so it stays empty. */ toolCalls: ToolCallRecord[]; + /** Tools the model asked for (tool_use blocks). A request is not an execution and earns no score. */ + requestedTools?: ToolRequestRecord[]; verificationCycles: VerificationCycleRecord[]; /** Raw agent output for audit */ outputHash: string; @@ -89,6 +92,11 @@ export interface RunTelemetry { error?: string; } +export interface ToolRequestRecord { + toolName: string; + inputHash: string; +} + export interface ToolCallRecord { toolName: string; durationMs: number; diff --git a/scripts/run-comprehensive-eval.ts b/scripts/run-comprehensive-eval.ts index c7daa4927..53a3a4a72 100644 --- a/scripts/run-comprehensive-eval.ts +++ b/scripts/run-comprehensive-eval.ts @@ -245,18 +245,25 @@ function percentile(arr: number[], p: number): number { return sorted[Math.min(idx, sorted.length - 1)]; } -/** `--models` values the resolver does not know. Each one must abort the run (SPEC-v9 A.0.10). */ -export function unresolvedModels(models: readonly string[]): string[] { - return models.filter((m) => resolveModelAlias(m) === null); +/** + * `--models` values that are not a canonical approved id: unknown ids, and legacy + * aliases such as `gpt-4o-mini` (which resolve to another model, so every row would be + * disqualified only after the paid run finished). Each one aborts the run before any + * request is made (SPEC-v9 A.0.10). + */ +export function nonCanonicalModels(models: readonly string[]): string[] { + return models.filter((m) => resolveModelAlias(m) !== m); } /** - * Rows that cannot count as evidence for the requested candidate: the resolver chose - * a different model, or this script's retry switched models (SPEC-v9 A.0.3). + * Rows that cannot count as evidence for the requested candidate (SPEC-v9 A.0.3): the + * resolver chose a different model, this script's retry switched models, or the row + * carries no identity at all (a deployment built before the identity fields existed, + * or a call that threw before it returned). */ export function identityViolations(results: readonly EpisodeResult[]): EpisodeResult[] { return results.filter( - (r) => r.fallbackUsed === true || (r.resolvedModelId !== undefined && r.resolvedModelId !== r.model), + (r) => !r.resolvedModelId || r.resolvedModelId !== r.model || r.fallbackUsed !== false, ); } @@ -290,7 +297,7 @@ function getEvalJudgeFallbackModel(model: string): string | null { // EVALUATION FUNCTIONS // ═══════════════════════════════════════════════════════════════════════════ -async function runSingleEpisode( +export async function runSingleEpisode( client: ConvexHttpClient, secret: string, model: string, @@ -307,12 +314,13 @@ async function runSingleEpisode( ? suite : "core"; + // Hoisted so an error row keeps the identity evidence it gathered before it threw. + let resolvedModelId: string | undefined; + let fallbackActivated = false; try { const maxAttempts = 2; let run: any = null; - let resolvedModelId: string | undefined; let activeModel = model; - let fallbackActivated = false; for (let attempt = 1; attempt <= maxAttempts; attempt++) { const scenarioInput = { @@ -412,6 +420,8 @@ async function runSingleEpisode( elapsed, failureReasons: [], error: err instanceof Error ? err.message : String(err), + resolvedModelId, + fallbackUsed: fallbackActivated, }; } } @@ -588,9 +598,9 @@ async function main() { // Fail closed: an unknown --models value used to resolve silently to the default // model, so a "candidate" run could measure a different model. - const unknown = unresolvedModels(modelsToTest); + const unknown = nonCanonicalModels(modelsToTest); if (unknown.length > 0) { - throw new Error(`Unknown --models value(s): ${unknown.join(", ")} (not an approved model or alias)`); + throw new Error(`Non-canonical --models value(s): ${unknown.join(", ")} (each must be an approved id exactly; aliases and unknown ids are refused)`); } // Select scenarios based on suite @@ -825,7 +835,7 @@ async function main() { if (violations.length > 0) { console.error( `\nERROR: ${violations.length} row(s) are not evidence for the requested model ` + - `(resolved to another id, or the retry switched models): ` + + `(resolved to another id, no resolved identity, or the retry switched models): ` + violations.map((r) => `${r.model}/${r.scenarioId}->${r.resolvedModelId ?? "?"}${r.fallbackUsed ? " (fallback)" : ""}`).join(", "), ); process.exitCode = 1; diff --git a/scripts/tool-gen/generate-tool.mjs b/scripts/tool-gen/generate-tool.mjs index 89e8f3fe2..779b26357 100644 --- a/scripts/tool-gen/generate-tool.mjs +++ b/scripts/tool-gen/generate-tool.mjs @@ -15,6 +15,8 @@ import { readFileSync, writeFileSync, appendFileSync, existsSync } from "node:fs"; import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { claudeText } from "./analyze-gap.mjs"; const MANIFEST_PATH = "/tmp/tool-manifest.json"; const MCP_ROOT = join(process.cwd(), "packages/mcp-local"); @@ -50,7 +52,20 @@ function setOutput(key, value) { console.log(`::set-output name=${key}::${value}`); } -async function callClaude(systemPrompt, userPrompt) { +/** + * The Messages API body. No sampling parameters: Claude 4.7+/5.x answer 400 to them. + * PR 1 moves no ids; PR 2 changes the model literal with its own request-shape update. + */ +export function buildClaudeRequest(systemPrompt, userPrompt) { + return { + model: "claude-sonnet-4-6", + max_tokens: 4096, + system: systemPrompt, + messages: [{ role: "user", content: userPrompt }], + }; +} + +export async function callClaude(systemPrompt, userPrompt) { const apiKey = process.env.ANTHROPIC_API_KEY; if (!apiKey) throw new Error("Missing ANTHROPIC_API_KEY"); @@ -61,12 +76,7 @@ async function callClaude(systemPrompt, userPrompt) { "anthropic-version": "2023-06-01", "content-type": "application/json", }, - body: JSON.stringify({ - model: "claude-sonnet-4-6", - max_tokens: 4096, - system: systemPrompt, - messages: [{ role: "user", content: userPrompt }], - }), + body: JSON.stringify(buildClaudeRequest(systemPrompt, userPrompt)), }); if (!res.ok) { @@ -74,12 +84,8 @@ async function callClaude(systemPrompt, userPrompt) { throw new Error(`Claude API ${res.status}: ${text}`); } - const data = await res.json(); // Read blocks by type: content[0] is a thinking block when adaptive thinking is on. - return (Array.isArray(data.content) ? data.content : []) - .filter((b) => b?.type === "text" && typeof b.text === "string") - .map((b) => b.text) - .join(""); + return claudeText(await res.json()); } async function main() { @@ -204,7 +210,10 @@ describe("Static: ${spec.toolName} tool", () => { setOutput("manifest_path", MANIFEST_PATH); } -main().catch((err) => { - console.error(err); - process.exit(1); -}); +// Run only as a script, so the request builder can be imported by tests. +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((err) => { + console.error(err); + process.exit(1); + }); +} diff --git a/shared/llm/anthropicWireIds.test.ts b/shared/llm/anthropicWireIds.test.ts index 031765625..7888a8941 100644 --- a/shared/llm/anthropicWireIds.test.ts +++ b/shared/llm/anthropicWireIds.test.ts @@ -15,11 +15,12 @@ import { ANTHROPIC_WIRE_IDS, anthropicText, anthropicWireId, + FREE_LANE_CATALOG_ID, getModelWithFailover, llmModelCatalog, resolveModelAlias, } from "./modelCatalog"; -import { MODEL_SPECS } from "../../backend/convex/domains/agents/mcp_tools/models/modelResolver"; +import { FREE_LANE_MODEL, MODEL_SPECS } from "../../backend/convex/domains/agents/mcp_tools/models/modelResolver"; describe("ANTHROPIC_WIRE_IDS agrees with the resolver's MODEL_SPECS", () => { const anthropicSpecs = Object.values(MODEL_SPECS).filter((spec) => spec.provider === "anthropic"); @@ -121,3 +122,47 @@ describe("anthropicText reads content blocks by type", () => { expect(anthropicText(null)).toBe(""); }); }); + +// An unknown id used to become gpt-5.4-mini on OpenAI, free-looking ones included +// (SPEC-v9 A.0.10; PR #662 review round 1). +describe("getModelWithFailover fails closed", () => { + beforeEach(() => { + vi.spyOn(console, "warn").mockImplementation(() => {}); + vi.spyOn(console, "error").mockImplementation(() => {}); + vi.stubEnv("OPENAI_API_KEY", "sk-test-key-not-real-0000"); + vi.stubEnv("ANTHROPIC_API_KEY", "sk-ant-test-key-not-real"); + vi.stubEnv("OPENROUTER_API_KEY", "sk-or-test-key-not-real"); + }); + afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + }); + + it("names the same free lane as the resolver", () => { + expect(FREE_LANE_CATALOG_ID).toBe(FREE_LANE_MODEL); + }); + + it("sends an unknown free-looking id to the free lane on OpenRouter, never to a paid OpenAI default", () => { + for (const id of ["retired-model-free", "someone/new-model:free"]) { + expect(getModelWithFailover(id), id).toEqual({ model: FREE_LANE_MODEL, provider: "openrouter", isFallback: true }); + } + }); + + it("every model the catalog itself names still resolves, so no default now throws", () => { + const named = new Set(Object.values(llmModelCatalog).flatMap((byTask) => Object.values(byTask).flat())); + expect(named.size).toBeGreaterThan(10); + for (const id of named) expect(() => getModelWithFailover(resolveModelAlias(id)), id).not.toThrow(); + }); + + it("throws for any other unknown id", () => { + for (const id of ["not-a-model", "freeloader-pro", "mistral-unknown"]) { + expect(() => getModelWithFailover(id), id).toThrow(/Unknown model/); + } + }); + + it("a configured free catalog model stays free, and an unconfigured provider never fails over to a paid one", () => { + expect(getModelWithFailover(FREE_LANE_MODEL)).toMatchObject({ provider: "openrouter", isFallback: false }); + vi.stubEnv("OPENROUTER_API_KEY", ""); + expect(() => getModelWithFailover(FREE_LANE_MODEL)).toThrow(/never fails over to a paid provider/); + }); +}); diff --git a/shared/llm/modelCatalog.ts b/shared/llm/modelCatalog.ts index 13f25df6a..4709807e9 100644 --- a/shared/llm/modelCatalog.ts +++ b/shared/llm/modelCatalog.ts @@ -868,8 +868,27 @@ function toProviderWireId(model: string, provider: LlmProvider): string { return provider === "anthropic" ? (anthropicWireId(model) ?? model) : model; } +/** + * The verified free lane as the catalog names it. Must equal `FREE_LANE_MODEL` + * in modelResolver.ts (this file cannot import from backend/convex); + * anthropicWireIds.test.ts pins the two together. + */ +export const FREE_LANE_CATALOG_ID = "nemotron-3-super-free"; + +/** Free-looking unknown ids: the OpenRouter `-free` / `:free` suffix. */ +function isFreeLookingModel(model: string): boolean { + return /(-|:)free$/.test(model.trim().toLowerCase()); +} + +function isFreeCatalogModel(model: string): boolean { + const price = modelPricing[model]; + return price != null && price.inputPer1M === 0 && price.outputPer1M === 0; +} + /** * Get a working model with failover. + * An unknown id fails closed (SPEC-v9 A.0.10): free-looking -> the free lane, + * anything else throws. A free model never fails over to a paid provider. * Tries the preferred model's provider first, then falls back to alternatives. * The returned `model` is the provider wire id (see `toProviderWireId`). */ @@ -881,7 +900,10 @@ export function getModelWithFailover(preferredModel: string): { const provider = getProviderForModel(preferredModel); if (!provider) { - return { model: DEFAULT_FALLBACK_MODEL, provider: "openai", isFallback: true }; + if (isFreeLookingModel(preferredModel)) { + return { model: FREE_LANE_CATALOG_ID, provider: "openrouter", isFallback: true }; + } + throw new Error(`[getModelWithFailover] Unknown model "${preferredModel}"`); } // Check if preferred provider is configured @@ -889,6 +911,10 @@ export function getModelWithFailover(preferredModel: string): { return { model: toProviderWireId(preferredModel, provider), provider, isFallback: false }; } + if (isFreeCatalogModel(preferredModel)) { + throw new Error(`[getModelWithFailover] Free model "${preferredModel}": ${provider} is not configured and a free request never fails over to a paid provider`); + } + // Try fallback providers const fallbacks = providerFallbackChain[provider]; for (const fallbackProvider of fallbacks) { diff --git a/shared/search/triSearch.test.ts b/shared/search/triSearch.test.ts index 670e47fd1..42855345d 100644 --- a/shared/search/triSearch.test.ts +++ b/shared/search/triSearch.test.ts @@ -188,7 +188,9 @@ describe("rerankWithGemini - request shape (Gemini 3.x)", () => { // 256-token cap on a reasoning preamble; thinkingLevel "low" is the replacement. it("sends thinkingLevel low and no thinkingBudget", async () => { const bodies: any[] = []; - const spyFetch = (async (_url: string, init: { body: string }) => { + const urls: string[] = []; + const spyFetch = (async (url: string, init: { body: string }) => { + urls.push(String(url)); bodies.push(JSON.parse(init.body)); return { ok: true, @@ -200,6 +202,9 @@ describe("rerankWithGemini - request shape (Gemini 3.x)", () => { const r = await rerankWithGemini("entity funding", fused, { topN: 2, apiKey: "test", fetchImpl: spyFetch }); expect(r.rerankStatus).toBe("ok"); expect(bodies).toHaveLength(1); + // The model rides in the URL, not the body: pin it so a changed default is a visible test change. + // PR 1 moves no ids; PR 2 changes it to gemini-3.8-flash with TRI_SEARCH_RERANK_MODEL precedence unchanged. + expect(urls[0]).toContain(`/models/${process.env.TRI_SEARCH_RERANK_MODEL || "gemini-3.5-flash"}:generateContent`); const thinking = bodies[0].generationConfig.thinkingConfig; expect(thinking).toEqual({ thinkingLevel: "low" }); expect(thinking).not.toHaveProperty("thinkingBudget"); diff --git a/workers/node/pipeline/extendedThinkingRunner.test.ts b/workers/node/pipeline/extendedThinkingRunner.test.ts index 111f4ed7d..32359c472 100644 --- a/workers/node/pipeline/extendedThinkingRunner.test.ts +++ b/workers/node/pipeline/extendedThinkingRunner.test.ts @@ -15,6 +15,7 @@ import { describe, it, expect } from "vitest"; import { buildCheckpointPrompt, buildCheckpointRequestBody, + estimateThinkingTokens, parseCheckpointResponse, shouldContinue, EXTENDED_PROMPT_VERSION, @@ -289,7 +290,10 @@ describe("buildCheckpointRequestBody - Anthropic request shape (SPEC-v9 A.0.6)", expect(body.system).toBe("sys"); }); - it("keeps the thinking budget a bound: max_tokens = answer cap + min(allowance, remaining)", () => { + it("caps total output per checkpoint: max_tokens = answer cap + min(allowance, remaining)", () => { + // This bounds thinking PLUS answer. Adaptive thinking has no separate thinking cap, so with 2,500 + // tokens left a checkpoint may still think for up to 6,500 (the 4,000 answer room is not reserved). + // The run-level bound is the next test. const at = (remaining: number) => (buildCheckpointRequestBody({ modelName: "claude-sonnet-4.6", prompt, remainingThinkingTokens: remaining }) as any).max_tokens; expect(at(40_000)).toBe(MAX_OUTPUT_TOKENS + MAX_THINKING_BUDGET_TOKENS); @@ -298,6 +302,27 @@ describe("buildCheckpointRequestBody - Anthropic request shape (SPEC-v9 A.0.6)", expect(at(-50)).toBe(MAX_OUTPUT_TOKENS); }); + it("a run spends at most thinkingBudgetTokens + MAX_OUTPUT_TOKENS on thinking when every checkpoint thinks to its cap", () => { + // Worst case: each checkpoint uses all of max_tokens, answers with 1 token of text, and the run + // charges thinking from usage exactly as extendedThinking.advanceRun does. The thinking block's text is + // omitted (Sonnet 5.x), so nothing but usage.output_tokens can drive the charge. + const budget = 15_000; + let charged = 0; + let actuallySpent = 0; + for (let checkpoint = 1; checkpoint <= MAX_CHECKPOINTS; checkpoint++) { + const maxTokens = (buildCheckpointRequestBody({ modelName: "claude-sonnet-4.6", prompt, remainingThinkingTokens: budget - charged }) as any).max_tokens; + const answer = "x".repeat(4); // 1 token + actuallySpent += maxTokens - 1; + charged += estimateThinkingTokens(maxTokens, answer); + const verdict = shouldContinue({ currentCheckpoint: checkpoint, totalCheckpoints: MAX_CHECKPOINTS, researchComplete: false, thinkingTokensUsed: charged, thinkingBudgetTokens: budget }); + if (!verdict.continue) break; + } + expect(charged).toBeGreaterThanOrEqual(budget); + expect(actuallySpent).toBeLessThanOrEqual(budget + MAX_OUTPUT_TOKENS); + // and the charge never undercounts what was spent + expect(charged).toBeGreaterThanOrEqual(actuallySpent); + }); + it("accepts an operator-supplied wire id and refuses a name with no Anthropic id", () => { expect(buildCheckpointRequestBody({ modelName: "claude-sonnet-4-6", prompt, remainingThinkingTokens: 1 }).model).toBe( "claude-sonnet-4-6", @@ -307,3 +332,16 @@ describe("buildCheckpointRequestBody - Anthropic request shape (SPEC-v9 A.0.6)", ); }); }); + +describe("estimateThinkingTokens", () => { + it("is output_tokens minus the answer's estimated tokens, never negative", () => { + expect(estimateThinkingTokens(5_000, "x".repeat(400))).toBe(4_900); + expect(estimateThinkingTokens(50, "x".repeat(4_000))).toBe(0); + expect(estimateThinkingTokens(0, "")).toBe(0); + }); + + it("does not depend on the thinking block text (it is omitted by default on Sonnet 5.x / Opus 4.7+)", () => { + // The function has no thinking-text input at all: a run that omits display still pays. + expect(estimateThinkingTokens(3_000, "answer")).toBeGreaterThan(2_900); + }); +}); diff --git a/workers/node/pipeline/extendedThinkingRunner.ts b/workers/node/pipeline/extendedThinkingRunner.ts index 27d302146..8b637bc75 100644 --- a/workers/node/pipeline/extendedThinkingRunner.ts +++ b/workers/node/pipeline/extendedThinkingRunner.ts @@ -103,9 +103,14 @@ export type ParseOutcome = * A name with no Anthropic wire id throws instead of being sent. * - Thinking is adaptive with `output_config.effort`: manual * `{type:"enabled", budget_tokens}` is deprecated on Sonnet/Opus 4.6 and - * 400s on Claude 5.x. Thinking tokens count toward `max_tokens`, so the - * per-checkpoint remaining budget raises the cap above the answer size, - * which keeps the run's thinking budget a real bound. + * 400s on Claude 5.x. Thinking tokens count toward `max_tokens`, and adaptive + * thinking has no separate thinking cap, so `max_tokens` bounds thinking PLUS + * answer, not thinking alone. What it guarantees: one checkpoint spends at most + * MAX_OUTPUT_TOKENS + min(MAX_THINKING_BUDGET_TOKENS, remaining) tokens in total, + * so a run, charged after every checkpoint by `estimateThinkingTokens` (from + * `usage.output_tokens`) and stopped by `shouldContinue`, spends at most + * `thinkingBudgetTokens + MAX_OUTPUT_TOKENS` on thinking. The budget is a + * checkpoint-granular bound with that one-answer overshoot, not an exact cap. */ export function buildCheckpointRequestBody(args: { modelName: string; @@ -130,6 +135,19 @@ export function buildCheckpointRequestBody(args: { }; } +/** + * Thinking tokens one checkpoint spent, from the provider's own usage. The Messages + * API reports `output_tokens` as thinking + answer with no split, and on Sonnet 5.x + * and Opus 4.7+ the thinking block's text is omitted by default, so its length says + * nothing. Subtracting the answer's estimated tokens (chars / 4, which under-counts + * JSON and therefore leans toward charging more thinking, never less) leaves the + * thinking charge. + */ +export function estimateThinkingTokens(outputTokens: number, answerText: string): number { + const answerTokens = Math.floor(answerText.length / 4); + return Math.max(0, Math.floor(outputTokens) - answerTokens); +} + /* -------------------------------------------------------------------------- * Prompt construction * ------------------------------------------------------------------------ */ From cabf937bcc135a14802bfb1b2e50d24912d366cf Mon Sep 17 00:00:00 2001 From: Homen Shum Date: Thu, 15 Oct 2026 05:29:51 -0700 Subject: [PATCH 3/8] fix(models): close the round 2 review findings (PR 1) Eval honesty: only stop_reason end_turn is a clean run (max_tokens, pause_turn and a missing stop_reason are errors), and the response read is bounded by content-length and by a capped stream instead of buffering the whole body. Cost class (A.0.10): the priority-index throw applies to unlisted FREE start models only, known paid ids outside MODEL_PRIORITY_ORDER keep main's behaviour; every executor branch in runtimeRouting honours an explicit free request; modelRouter caps tier "free" at free and prices the free lane at zero; getModelWithFailover returns the free lane's wire id, takes free route aliases, and no longer lets a gpt-/claude-/gemini- prefix make an invented id fail over to a paid provider; instagramClaimVerification fails instead of building a paid model. Request shape: condenseQuery in triSearch uses thinkingLevel low like the rerank call on the same model. Tests: boundary test now fills the launch window so it can see the order, request-shape tests assert the outgoing model string and top_p/top_k. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/ci.yml | 1 + .../mcp_tools/models/modelResolver.test.ts | 19 ++++++ .../agents/mcp_tools/models/modelResolver.ts | 19 +++--- .../domains/agents/runtimeRouting.test.ts | 43 ++++++++++++++ .../convex/domains/agents/runtimeRouting.ts | 58 ++++++++++--------- .../models/modelRouter.requestShape.test.ts | 42 +++++++++++++- .../convex/domains/ai/models/modelRouter.ts | 15 ++++- .../pipelines/pipelineModelBoundary.test.ts | 49 +++++++++++++--- .../instagramClaimVerification.test.ts | 36 ++++++++++++ .../instagramClaimVerification.ts | 6 +- .../__tests__/anthropicRequestShape.test.ts | 1 + scripts/eval-harness/agentInvoker.ts | 41 +++++++++++-- scripts/eval-harness/evalHarness.test.ts | 43 ++++++++++++++ shared/llm/anthropicWireIds.test.ts | 32 +++++++++- shared/llm/modelCatalog.ts | 54 +++++++++++++---- shared/search/triSearch.test.ts | 19 ++++++ shared/search/triSearch.ts | 5 +- 17 files changed, 415 insertions(+), 68 deletions(-) create mode 100644 backend/convex/domains/verification/instagramClaimVerification.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a786a129b..717334fc2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -114,6 +114,7 @@ jobs: backend/convex/domains/product/extendedThinking.requestShape.test.ts backend/convex/providerCopies.requestShape.test.ts backend/convex/domains/agents/runtimeRouting.test.ts + backend/convex/domains/verification/instagramClaimVerification.test.ts backend/convex/domains/agents/agentChatActions.failClosed.test.ts packages/mcp-local/src/__tests__/anthropicRequestShape.test.ts apps/web/src/features/agents/components/FastAgentPanel/__tests__/RunState.test.tsx diff --git a/backend/convex/domains/agents/mcp_tools/models/modelResolver.test.ts b/backend/convex/domains/agents/mcp_tools/models/modelResolver.test.ts index 76fc986a2..028693331 100644 --- a/backend/convex/domains/agents/mcp_tools/models/modelResolver.test.ts +++ b/backend/convex/domains/agents/mcp_tools/models/modelResolver.test.ts @@ -250,6 +250,25 @@ describe("fallback helpers keep a free request on free models", () => { }); }); +describe("an approved PAID model outside the priority order keeps main's behaviour", () => { + // Main restarts such a start model at the head of the order (kimi-k2.6) and lists the whole chain. + // Changing that is a caller-argument decision (PR body, C item c), not part of A.0.10. + const paidUnlisted = ["glm-4.7-flash", "deepseek-r1", "gemini-2.5-flash", "mistral-large", "claude-opus-4"]; + it.each(paidUnlisted)("%s does not throw and starts at the head", async (id) => { + // buildLanguageModel needs a key to construct the head model and the requested model. + vi.stubEnv("OPENROUTER_API_KEY", "test-key-not-real"); + vi.stubEnv("GEMINI_API_KEY", "test-key-not-real"); + expect(APPROVED_MODELS as readonly string[]).toContain(id); + expect(MODEL_PRIORITY_ORDER as readonly string[]).not.toContain(id); + expect(isFreeModel(id as never)).toBe(false); + const r = await executeWithModelFallback(async (_m, used) => used, { startModel: id as never }); + expect(r.modelUsed).toBe(MODEL_PRIORITY_ORDER[0]); + expect(getFallbackChain(id as never)).toEqual(MODEL_PRIORITY_ORDER); + expect(getLanguageModelWithFallback(id as never).fallbackChain).toEqual(MODEL_PRIORITY_ORDER); + vi.unstubAllEnvs(); + }); +}); + describe("normalizeNodeBenchRuntimeModel is the first resolver and fails closed", () => { beforeEach(() => { vi.spyOn(console, "warn").mockImplementation(() => {}); diff --git a/backend/convex/domains/agents/mcp_tools/models/modelResolver.ts b/backend/convex/domains/agents/mcp_tools/models/modelResolver.ts index 2fc6ced0b..5cb5ea6d0 100644 --- a/backend/convex/domains/agents/mcp_tools/models/modelResolver.ts +++ b/backend/convex/domains/agents/mcp_tools/models/modelResolver.ts @@ -1244,14 +1244,16 @@ export function getFreeModels(): ApprovedModel[] { } /** - * Index of `model` in MODEL_PRIORITY_ORDER. A start model that is not in the - * order throws: restarting it at index 0 (a paid model) would turn a free - * request into a paid one (SPEC-v9 A.0.10). + * Index of `model` in MODEL_PRIORITY_ORDER, or -1 for an approved model that is + * not listed (callers then keep main's behaviour: the chain starts at the head). + * A free start model that is not listed throws: restarting it at index 0 (a paid + * model) would turn a free request into a paid one (SPEC-v9 A.0.10). Known paid + * ids are untouched until the listed-order decision (PR body, C item c). */ function priorityIndexOrThrow(model: ApprovedModel, caller: string): number { const index = MODEL_PRIORITY_ORDER.indexOf(model); - if (index === -1) { - throw new Error(`[modelResolver] ${caller}: "${model}" is not in MODEL_PRIORITY_ORDER; refusing to restart at the head of the paid chain`); + if (index === -1 && isFreeModel(model)) { + throw new Error(`[modelResolver] ${caller}: free model "${model}" is not in MODEL_PRIORITY_ORDER; refusing to restart at the head of the paid chain`); } return index; } @@ -1312,8 +1314,9 @@ export async function executeWithModelFallback( let fallbacksUsed = 0; let lastError: Error | null = null; - // Get the starting index in priority order; an unlisted start model throws. - let modelIndex = priorityIndexOrThrow(currentModel, "executeWithModelFallback"); + // Starting index in priority order; an unlisted paid model starts at the head (as on main), + // an unlisted free model throws. + let modelIndex = Math.max(0, priorityIndexOrThrow(currentModel, "executeWithModelFallback")); while (modelIndex < MODEL_PRIORITY_ORDER.length) { currentModel = MODEL_PRIORITY_ORDER[modelIndex]; @@ -1390,5 +1393,5 @@ export function getBestFreeModel(): ApprovedModel { * Get the fallback chain starting from a given model */ export function getFallbackChain(startModel: ApprovedModel = DEFAULT_MODEL): ApprovedModel[] { - return MODEL_PRIORITY_ORDER.slice(priorityIndexOrThrow(startModel, "getFallbackChain")); + return MODEL_PRIORITY_ORDER.slice(Math.max(0, priorityIndexOrThrow(startModel, "getFallbackChain"))); } diff --git a/backend/convex/domains/agents/runtimeRouting.test.ts b/backend/convex/domains/agents/runtimeRouting.test.ts index cf1c736b4..0937283e3 100644 --- a/backend/convex/domains/agents/runtimeRouting.test.ts +++ b/backend/convex/domains/agents/runtimeRouting.test.ts @@ -237,6 +237,49 @@ describe("chooseNodeBenchRuntimeRoute keeps cost class and fails closed", () => expect(background.fallbackModels.every((id) => isFreeModel(id))).toBe(true); }); + it("an explicit free request stays free on every executor branch a prompt keyword can reach", () => { + // Persona: a user who picked the free model, then typed "patch this script" or "return JSON". + // Before: these branches never read requestedModel and sent the turn to a paid Gemini model. + const cases: Array<[string, string, Parameters[0]]> = [ + ["tool_heavy_executor", "Use the browser tool to patch this script", { prompt: "", workingSet: buildWorkingSet() }], + ["structured_executor", "Return JSON with these fields", { prompt: "", workingSet: buildWorkingSet() }], + [ + "context_rot_guard", + "Continue the analysis.", + { prompt: "", workingSet: buildWorkingSet({ contextRotRisk: "medium", activeAngles: ["entity_profile", "public_signals", "funding_intelligence"] }) }, + ], + [ + "large_context_executor", + "Continue the analysis.", + { prompt: "", workingSet: buildWorkingSet({ activeAngles: ["entity_profile", "public_signals"] }) }, + ], + ["anonymous_fast_executor", "Summarize this company.", { prompt: "", isAnonymous: true, workingSet: buildWorkingSet() }], + ]; + for (const [reason, prompt, base] of cases) { + const route = chooseNodeBenchRuntimeRoute({ + ...base, + prompt, + requestedModel: FREE_LANE_MODEL, + useCoordinator: false, + availableModels: paidAvailable, + }); + expect(route.reason, reason).toBe(reason); + expect(route.model, reason).toBe(FREE_LANE_MODEL); + expect(route.fallbackModels.every((id) => isFreeModel(id)), reason).toBe(true); + } + }); + + it("without a free request those branches keep their paid defaults", () => { + const route = chooseNodeBenchRuntimeRoute({ + prompt: "Use the browser tool to patch this script", + requestedModel: "gpt-5.4-mini", + availableModels: paidAvailable, + workingSet: buildWorkingSet(), + }); + expect(route.reason).toBe("tool_heavy_executor"); + expect(route.model).toBe("gemini-3-flash-preview"); + }); + it("a free primary that is not in the available list does not fall over to a paid model", () => { // availableModels contains only paid ids: the old chooser would pick the first paid candidate. const route = ask(FREE_LANE_MODEL, "Plan and orchestrate a deep research run on Stripe."); diff --git a/backend/convex/domains/agents/runtimeRouting.ts b/backend/convex/domains/agents/runtimeRouting.ts index 2896ef2e8..0276886b3 100644 --- a/backend/convex/domains/agents/runtimeRouting.ts +++ b/backend/convex/domains/agents/runtimeRouting.ts @@ -103,18 +103,35 @@ function chooseAvailableRouteModels(args: { }; } -function resolveRequestedExecutorModel( - requestedModel?: string | null, -): ApprovedModel | null { +function resolveRequestedModel(requestedModel?: string | null): ApprovedModel | null { if (!requestedModel) { return null; } - // An unknown id throws (or becomes the free lane when free-looking); it must // not fall through to the paid executor default (SPEC-v9 A.0.10). - const resolved = + return ( resolveModelAlias(requestedModel) ?? - normalizeNodeBenchRuntimeModel(requestedModel, "executor"); + normalizeNodeBenchRuntimeModel(requestedModel, "executor") + ); +} + +/** + * The model the caller explicitly asked for, but only when it is free. Every + * executor branch starts from this so a request for the free lane is never + * routed to a paid model by a prompt keyword (SPEC-v9 A.0.10). + */ +function resolveRequestedFreeModel(requestedModel?: string | null): ApprovedModel | null { + const resolved = resolveRequestedModel(requestedModel); + return resolved && isFreeModel(resolved) ? resolved : null; +} + +function resolveRequestedExecutorModel( + requestedModel?: string | null, +): ApprovedModel | null { + const resolved = resolveRequestedModel(requestedModel); + if (!resolved) { + return null; + } if (isFreeModel(resolved)) { return resolved; } @@ -136,6 +153,10 @@ export function chooseNodeBenchRuntimeRoute( const messagesCompacted = workingSet?.messagesCompacted ?? 0; const contextRotRisk = workingSet?.contextRotRisk ?? "low"; const hasOpenRouter = args.hasOpenRouter !== false; + const requestedFree = resolveRequestedFreeModel(args.requestedModel); + // Executor branches that pick a fixed default start from the requested free model when there is one. + const executorPrimary = (fixedDefault: ApprovedModel) => + requestedFree ?? normalizeNodeBenchRuntimeModel(fixedDefault, "executor"); const pickRouteModels = ( primary: ApprovedModel, candidates: ApprovedModel[], @@ -147,10 +168,7 @@ export function chooseNodeBenchRuntimeRoute( }); if (args.isAnonymous) { - const preferredModel = normalizeNodeBenchRuntimeModel( - "gemini-3.1-flash-lite-preview", - "executor", - ); + const preferredModel = executorPrimary("gemini-3.1-flash-lite-preview"); const { model, fallbackModels } = pickRouteModels(preferredModel, [ "gemini-3-flash-preview", ...(hasOpenRouter ? (["minimax-m2.7"] as ApprovedModel[]) : []), @@ -272,10 +290,7 @@ export function chooseNodeBenchRuntimeRoute( contextRotRisk === "medium" && (activeAngles.length >= 3 || (messagesCompacted >= 18 && activeAngles.length >= 2)) ) { - const preferredModel = normalizeNodeBenchRuntimeModel( - "gemini-3-flash-preview", - "executor", - ); + const preferredModel = executorPrimary("gemini-3-flash-preview"); const { model, fallbackModels } = pickRouteModels(preferredModel, [ "gemini-3.1-flash-lite-preview", ...(hasOpenRouter ? (["minimax-m2.7"] as ApprovedModel[]) : []), @@ -292,10 +307,7 @@ export function chooseNodeBenchRuntimeRoute( } if (toolHeavyExecutorSignal) { - const preferredModel = normalizeNodeBenchRuntimeModel( - "gemini-3-flash-preview", - "executor", - ); + const preferredModel = executorPrimary("gemini-3-flash-preview"); const { model, fallbackModels } = pickRouteModels(preferredModel, [ "gemini-3.1-flash-lite-preview", ...(hasOpenRouter ? (["minimax-m2.7"] as ApprovedModel[]) : []), @@ -312,10 +324,7 @@ export function chooseNodeBenchRuntimeRoute( } if (structuredExecutorSignal) { - const preferredModel = normalizeNodeBenchRuntimeModel( - "gemini-3.1-flash-lite-preview", - "executor", - ); + const preferredModel = executorPrimary("gemini-3.1-flash-lite-preview"); const { model, fallbackModels } = pickRouteModels(preferredModel, [ "gemini-3-flash-preview", ...(hasOpenRouter ? (["minimax-m2.7"] as ApprovedModel[]) : []), @@ -332,10 +341,7 @@ export function chooseNodeBenchRuntimeRoute( } if (activeAngles.length >= 2 || jitSlices.length >= 2) { - const preferredModel = normalizeNodeBenchRuntimeModel( - "gemini-3-flash-preview", - "executor", - ); + const preferredModel = executorPrimary("gemini-3-flash-preview"); const { model, fallbackModels } = pickRouteModels(preferredModel, [ "gemini-3.1-flash-lite-preview", ...(hasOpenRouter ? (["minimax-m2.7"] as ApprovedModel[]) : []), diff --git a/backend/convex/domains/ai/models/modelRouter.requestShape.test.ts b/backend/convex/domains/ai/models/modelRouter.requestShape.test.ts index 48f277ee3..c35ad0319 100644 --- a/backend/convex/domains/ai/models/modelRouter.requestShape.test.ts +++ b/backend/convex/domains/ai/models/modelRouter.requestShape.test.ts @@ -50,12 +50,23 @@ describe("modelRouter.route request shape and cost class", () => { it("omits temperature for Claude and Gemini and sends it for the providers that accept it", async () => { const sent: Record = {}; + const dispatched: Record = {}; for (const pinnedModel of ["claude-sonnet-4.6", "gemini-3.1-pro-preview", "kimi-k2.6", "gpt-5.4"]) { generateText.mockClear(); await call(makeCtx(), { pinnedModel, temperature: 0.7 }); expect(generateText).toHaveBeenCalledTimes(1); - sent[pinnedModel] = "temperature" in generateText.mock.calls[0][0]; + const request = generateText.mock.calls[0][0]; + sent[pinnedModel] = "temperature" in request; + dispatched[pinnedModel] = (request.model as { modelId: string }).modelId; + for (const key of ["top_p", "top_k", "topP", "topK"]) expect(request, `${pinnedModel} ${key}`).not.toHaveProperty(key); } + // The outgoing model string, so a wrong-model dispatch cannot pass on the temperature check alone. + expect(dispatched).toEqual({ + "claude-sonnet-4.6": "claude-sonnet-4-6", + "gemini-3.1-pro-preview": "gemini-3.1-pro-preview", + "kimi-k2.6": "moonshotai/kimi-k2.6", + "gpt-5.4": "gpt-5.4", + }); expect(sent).toEqual({ "claude-sonnet-4.6": false, "gemini-3.1-pro-preview": false, @@ -78,6 +89,35 @@ describe("modelRouter.route request shape and cost class", () => { expect(tried.every((id) => id.endsWith(":free")), tried.join(", ")).toBe(true); }); + it("a free request is capped at free even when the registry chain holds a paid model (requireVision)", async () => { + // Before: tier is a floor in the registry chain, so requireVision put glm-4.7-flash ($0.07/M) + // in the free chain; it was dispatched and recorded isFree=false, actualTier=cheap. + resolveChain.mockReturnValue({ chain: ["glm-4.7-flash", "nemotron-3-super-free"], reason: "ok", diagnostics: { primaryOutcome: "primary", candidatePoolSize: 2 } }); + const result = await call(makeCtx(), { tier: "free", requireVision: true }); + const tried = generateText.mock.calls.map((c) => (c[0].model as { modelId: string }).modelId); + expect(tried).toEqual(["nvidia/nemotron-3-super-120b-a12b:free"]); + expect(result.modelId).toBe("nemotron-3-super-free"); + expect(result.isFree).toBe(true); + expect(result.routing.actualTier).toBe("free"); + expect(result.costUsd).toBe(0); + + // No free model qualifies: the call fails, nothing is dispatched. + generateText.mockClear(); + resolveChain.mockReturnValue({ chain: ["glm-4.7-flash"], reason: "ok", diagnostics: { primaryOutcome: "primary", candidatePoolSize: 1 } }); + await expect(call(makeCtx(), { tier: "free", requireVision: true })).rejects.toThrow(/All 0 models failed/); + expect(generateText).not.toHaveBeenCalled(); + }); + + it("a pinned free-lane route is recorded as free, not as premium paid usage", async () => { + const ctx = makeCtx(); + const result = await call(ctx, { pinnedModel: "nemotron-3-super-free" }); + expect(result.isFree).toBe(true); + expect(result.costUsd).toBe(0); + expect(result.routing.actualTier).toBe("free"); + const logged = (ctx.runMutation.mock.calls[0] as unknown as [unknown, Record])[1]; + expect(logged).toMatchObject({ modelId: "nemotron-3-super-free", actualTier: "free", costUsd: 0 }); + }); + it("a cheap-tier outage still falls through the cheap list (the paid list is only for paid tiers)", async () => { resolveChain.mockReturnValue({ chain: [], reason: "no_matches_for_requirement", diagnostics: { primaryOutcome: "not_supplied", candidatePoolSize: 0 } }); generateText.mockRejectedValue(Object.assign(new Error("provider down"), { status: 400 })); diff --git a/backend/convex/domains/ai/models/modelRouter.ts b/backend/convex/domains/ai/models/modelRouter.ts index 0e85524db..9446c575c 100644 --- a/backend/convex/domains/ai/models/modelRouter.ts +++ b/backend/convex/domains/ai/models/modelRouter.ts @@ -16,7 +16,7 @@ import { v } from "convex/values"; import { internalAction } from "../../../_generated/server"; import { internal } from "../../../_generated/api"; import { generateText } from "ai"; -import { getLanguageModelSafe, getProviderForModel, resolveModelAlias } from "../../agents/mcp_tools/models"; +import { getLanguageModelSafe, getProviderForModel, isFreeModel, resolveModelAlias } from "../../agents/mcp_tools/models"; // B-PR4: capability-aware fallback chain. // resolveChain replaces the static `[...TIER_MODELS[tier]]` candidate list // with a tier-floor-enforced ordering that respects requested capabilities @@ -148,6 +148,9 @@ const MODEL_PRICING: Record = { "arcee-trinity-large-free": { input: 0, output: 0 }, "nemotron-3-nano-30b-free": { input: 0, output: 0 }, "llama-3.3-70b-free": { input: 0, output: 0 }, + // FREE_LANE_MODEL: without this row calculateCost bills it at the $1/$5 default and the audit + // row records a pinned free route as premium paid usage. Prices for the new paid ids land in PR 2. + "nemotron-3-super-free": { input: 0, output: 0 }, "glm-4.7-flash": { input: 0.07, output: 0.07 }, "gemini-3-flash-preview": { input: 0.50, output: 3.00 }, "claude-haiku-4.5": { input: 1.00, output: 5.00 }, @@ -408,6 +411,16 @@ export const route = internalAction({ // The free tier never appends paid candidates (SPEC-v9 A.0.10). candidates = [...(TIER_MODELS[tier] ?? TIER_MODELS.cheap)]; } + + // A free request is capped at free (SPEC-v9 A.0.10): the registry chain applies tier as a + // floor, so a capability requirement (for example requireVision) can put a paid model in it. + // If no free model qualifies the call fails below instead of billing. + if (tier === "free") { + candidates = candidates.filter((id) => { + const alias = resolveModelAlias(id); + return alias !== null && isFreeModel(alias); + }); + } } // ─── Try models in order with fallback ─────────────────────────── diff --git a/backend/convex/domains/pipelines/pipelineModelBoundary.test.ts b/backend/convex/domains/pipelines/pipelineModelBoundary.test.ts index 86258b241..5dd9abc98 100644 --- a/backend/convex/domains/pipelines/pipelineModelBoundary.test.ts +++ b/backend/convex/domains/pipelines/pipelineModelBoundary.test.ts @@ -15,6 +15,7 @@ import { describe, expect, it, vi } from "vitest"; import schema from "../../schema"; import { api } from "../../_generated/api"; import { UnknownModelError } from "../agents/mcp_tools/models/modelResolver"; +import { PIPELINE_SHORT_WINDOW_LIMIT } from "./pipelineAdmission"; import { parseModelKey } from "./piRuntime"; const DIR_SEGMENTS = ["domains", "pipelines"]; @@ -44,14 +45,38 @@ try { const pipelines = (api as any).domains.pipelines; +// The model check must run BEFORE admission. A Convex mutation rolls back its writes when it throws, +// so "the counter is unchanged afterwards" cannot tell the two orders apart. Filling the owner's launch +// window can: if admission ran first the error would be the limit error, not UnknownModelError. +async function fillLaunchWindow(t: any, ownerKey: string) { + const now = Date.now(); + await t.run(async (ctx: any) => + ctx.db.insert("pipelineLaunchAdmissions", { + ownerKey, + shortWindowStartedAt: now, + shortCount: PIPELINE_SHORT_WINDOW_LIMIT, + dayWindowStartedAt: now, + dayCount: PIPELINE_SHORT_WINDOW_LIMIT, + updatedAt: now, + }), + ); +} +const admissionCounts = (t: any) => + t.run(async (ctx: any) => (await ctx.db.query("pipelineLaunchAdmissions").collect()).map((r: any) => [r.ownerKey, r.shortCount])); + describe.skipIf(!convexTestAvailable)("startPipelineRun model boundary", () => { - it("rejects an unknown modelId and leaves pipelineLaunchAdmissions unchanged", async () => { + it("rejects an unknown modelId before admission: with the launch window full it still says UnknownModel", async () => { const t = convexTest(schema, convexModules); const userId = await t.run(async (ctx: any) => ctx.db.insert("users", { email: "boundary@example.com" })); const user = t.withIdentity({ subject: String(userId) }); - const countAdmissions = () => t.run(async (ctx: any) => (await ctx.db.query("pipelineLaunchAdmissions").collect()).length); + await fillLaunchWindow(t, `user:${String(userId)}`); + const before = await admissionCounts(t); + + // Control: a launch with a valid (omitted) model hits the full window, so the window really is full. + await expect( + user.mutation(pipelines.pipelineWorkflow.startPipelineRun, { pipelineKind: "research", spec: "Research a company" }), + ).rejects.toThrow(/Pipeline launch limit reached/); - const before = await countAdmissions(); await expect( user.mutation(pipelines.pipelineWorkflow.startPipelineRun, { pipelineKind: "research", @@ -59,13 +84,21 @@ describe.skipIf(!convexTestAvailable)("startPipelineRun model boundary", () => { modelId: "gpt-9-ultra", }), ).rejects.toThrow(/Unknown model "gpt-9-ultra"/); - expect(await countAdmissions()).toBe(before); + expect(await admissionCounts(t)).toEqual(before); }, 20_000); - it("rejects an unknown modelId on the trusted internal entry point too", async () => { + it("rejects an unknown modelId on the trusted internal entry point before admission too", async () => { const t = convexTest(schema, convexModules); - const countAdmissions = () => t.run(async (ctx: any) => (await ctx.db.query("pipelineLaunchAdmissions").collect()).length); - const before = await countAdmissions(); + await fillLaunchWindow(t, "user:cron-owner"); + const before = await admissionCounts(t); + await expect( + t.mutation(pipelines.pipelineWorkflow.startPipelineRunInternal, { + pipelineKind: "research", + spec: "Cron research", + ownerKey: "user:cron-owner", + }), + ).rejects.toThrow(/Pipeline launch limit reached/); // control: the window is full + await expect( t.mutation(pipelines.pipelineWorkflow.startPipelineRunInternal, { pipelineKind: "research", @@ -74,7 +107,7 @@ describe.skipIf(!convexTestAvailable)("startPipelineRun model boundary", () => { ownerKey: "user:cron-owner", }), ).rejects.toThrow(/Unknown model/); - expect(await countAdmissions()).toBe(before); + expect(await admissionCounts(t)).toEqual(before); }, 20_000); }); diff --git a/backend/convex/domains/verification/instagramClaimVerification.test.ts b/backend/convex/domains/verification/instagramClaimVerification.test.ts new file mode 100644 index 000000000..2115aa8b7 --- /dev/null +++ b/backend/convex/domains/verification/instagramClaimVerification.test.ts @@ -0,0 +1,36 @@ +// @vitest-environment node +/** + * verifyInstagramClaims is a free-lane job ("Cost: $0.00"). Persona: an operator whose deploy has + * no OPENROUTER_API_KEY. Before SPEC-v9 A.0.10 the action quietly built gemini-3-flash-preview + * (a paid model) and verified claims on it; now it fails visibly and nothing is dispatched. + * generateText is the fake transport; no network, no spend. + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const generateText = vi.hoisted(() => vi.fn()); +vi.mock("ai", async (importOriginal) => ({ ...(await importOriginal()), generateText })); + +import { verifyInstagramClaims } from "./instagramClaimVerification"; + +const claims = [{ claim: "The sky is blue", confidence: 0.9 }]; +const run = () => (verifyInstagramClaims as any)._handler({ runMutation: vi.fn(async () => null) }, { postId: "p1", claims }); + +describe("instagramClaimVerification free model selection", () => { + beforeEach(() => { + for (const key of ["OPENROUTER_API_KEY", "GEMINI_API_KEY", "GOOGLE_AI_API_KEY", "GOOGLE_GENERATIVE_AI_API_KEY"]) vi.stubEnv(key, ""); + vi.spyOn(console, "warn").mockImplementation(() => {}); + vi.spyOn(console, "log").mockImplementation(() => {}); + generateText.mockReset(); + }); + afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + }); + + it("with no free model buildable it throws and never builds or calls a paid model", async () => { + vi.stubEnv("GEMINI_API_KEY", "test-key-not-real-0000"); // a paid provider IS configured: it must not be used + await expect(run()).rejects.toThrow(/No free model available/); + expect(generateText).not.toHaveBeenCalled(); + }); +}); diff --git a/backend/convex/domains/verification/instagramClaimVerification.ts b/backend/convex/domains/verification/instagramClaimVerification.ts index 629c07e55..e5d62ad8d 100644 --- a/backend/convex/domains/verification/instagramClaimVerification.ts +++ b/backend/convex/domains/verification/instagramClaimVerification.ts @@ -154,9 +154,9 @@ async function getFreeModel() { console.warn(`[instagramClaimVerification] Model ${modelAlias} unavailable, trying next`); } } - // Fallback to default (may have cost) - console.warn("[instagramClaimVerification] No free models available, using default"); - return { model: getLanguageModelSafe("gemini-3-flash-preview"), alias: "gemini-3-flash-preview" }; + // A free-lane job never becomes a paid job (SPEC-v9 A.0.10): with no free model buildable + // (OpenRouter not configured) the action fails visibly instead of billing gemini-3-flash-preview. + throw new Error("[instagramClaimVerification] No free model available (is OPENROUTER_API_KEY set?); refusing to fall back to a paid model"); } // ═══════════════════════════════════════════════════════════════════════════ diff --git a/packages/mcp-local/src/__tests__/anthropicRequestShape.test.ts b/packages/mcp-local/src/__tests__/anthropicRequestShape.test.ts index b6fe5a736..1ec60834b 100644 --- a/packages/mcp-local/src/__tests__/anthropicRequestShape.test.ts +++ b/packages/mcp-local/src/__tests__/anthropicRequestShape.test.ts @@ -61,6 +61,7 @@ describe("mcp-local Anthropic provider request shape", () => { const result = await handler(researchWritingTools as never, "polish_academic_text")({ text: "We propose a method." }); expect(create).toHaveBeenCalledTimes(1); const body = create.mock.calls[0][0]; + expect(body.model).toBe("claude-haiku-4-5-20251001"); // PR 1 moves no ids for (const key of ["temperature", "top_p", "top_k"]) expect(body, key).not.toHaveProperty(key); expect(body.max_tokens).toBe(4096); expect(result.polishedText).toBe("the answer"); diff --git a/scripts/eval-harness/agentInvoker.ts b/scripts/eval-harness/agentInvoker.ts index 0b5ef3876..0460953ca 100644 --- a/scripts/eval-harness/agentInvoker.ts +++ b/scripts/eval-harness/agentInvoker.ts @@ -30,6 +30,33 @@ interface InvokerOptions { /** BOUND_READ: a Messages response larger than this is treated as a failed run. */ const MAX_RESPONSE_CHARS = 4_000_000; +/** + * BOUND_READ: never buffers more than MAX_RESPONSE_CHARS. A declared content-length over the cap is + * refused before any byte is read; otherwise the stream is cancelled the moment the cap is crossed. + * Returns null when the body is too large. + */ +async function readBounded(response: Response): Promise { + const declared = Number(response.headers.get("content-length")); + if (Number.isFinite(declared) && declared > MAX_RESPONSE_CHARS) { + await response.body?.cancel(); + return null; + } + if (!response.body) return ""; + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let text = ""; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + text += decoder.decode(value, { stream: true }); + if (text.length > MAX_RESPONSE_CHARS) { + await reader.cancel(); + return null; + } + } + return text + decoder.decode(); +} + function sha256(value: string): string { return createHash("sha256").update(value).digest("hex"); } @@ -152,11 +179,11 @@ async function invokeAnthropic( body: JSON.stringify(buildAnthropicRequest(config, task, options.model ?? config.model)), signal: controller.signal, }); - const raw = await response.text(); - if (!response.ok) { - error = `Anthropic HTTP ${response.status}: ${raw.slice(0, 240)}`; - } else if (raw.length > MAX_RESPONSE_CHARS) { + const raw = await readBounded(response); + if (raw === null) { error = `Anthropic response exceeded ${MAX_RESPONSE_CHARS} chars`; + } else if (!response.ok) { + error = `Anthropic HTTP ${response.status}: ${raw.slice(0, 240)}`; } else { const data = JSON.parse(raw) as { model?: string; @@ -182,9 +209,13 @@ async function invokeAnthropic( // The model string the provider returned, not the one requested (SPEC-v9 A.0.3). echoedModel = data.model; const problems: string[] = []; - if (data.stop_reason === "refusal") problems.push("Anthropic stop_reason: refusal"); + // Only end_turn is a finished answer. max_tokens (adaptive thinking counts toward the cap on + // Claude 5.x, so a cap sized for 4.5 truncates), pause_turn and a missing stop_reason are all + // unfinished turns and must not count as a clean run. if (data.stop_reason === "tool_use") { problems.push(`Anthropic stop_reason: tool_use; ${requestedTools.length} tool(s) requested, none executed (unfinished turn)`); + } else if (data.stop_reason !== "end_turn") { + problems.push(`Anthropic stop_reason: ${data.stop_reason ?? "missing"} (not end_turn, unfinished or truncated turn)`); } if (!echoedModel) problems.push("Anthropic response carries no model string, so the run has no identity evidence"); if (problems.length > 0) error = problems.join("; "); diff --git a/scripts/eval-harness/evalHarness.test.ts b/scripts/eval-harness/evalHarness.test.ts index f96ffa6de..f86147a50 100644 --- a/scripts/eval-harness/evalHarness.test.ts +++ b/scripts/eval-harness/evalHarness.test.ts @@ -147,6 +147,49 @@ describe("agentInvoker Anthropic backend", () => { expect(refused.error).toMatch(/refusal/); }); + it("counts only end_turn as a clean run: max_tokens, pause_turn and a missing stop_reason are errors", async () => { + for (const stop_reason of ["max_tokens", "pause_turn", undefined]) { + const telemetry = await invokeAgent(config("bare"), task, { + backend: "anthropic", + apiKey: "k", + model: "candidate-model", + fetchImpl: fakeMessagesTransport({ model: "m", stop_reason, content: [{ type: "text", text: "cut off" }] }).fetchImpl, + }); + expect(telemetry.error, `stop_reason ${String(stop_reason)}`).toMatch(/stop_reason: (max_tokens|pause_turn|missing)/); + } + }); + + it("bounds the read: an oversized stream is cancelled, not buffered", async () => { + let pulled = 0; + const chunk = new TextEncoder().encode("x".repeat(1_000_000)); + const stream = new ReadableStream({ + pull(controller) { + pulled += 1; + controller.enqueue(chunk); + if (pulled > 50) controller.close(); + }, + }); + const telemetry = await invokeAgent(config("bare"), task, { + backend: "anthropic", + apiKey: "k", + model: "candidate-model", + fetchImpl: (async () => new Response(stream, { status: 200 })) as unknown as typeof fetch, + }); + expect(telemetry.error).toMatch(/exceeded/); + expect(pulled).toBeLessThan(10); + }); + + it("refuses a declared content-length over the cap without reading the body", async () => { + const telemetry = await invokeAgent(config("bare"), task, { + backend: "anthropic", + apiKey: "k", + model: "candidate-model", + fetchImpl: (async () => + new Response("{}", { status: 200, headers: { "content-length": "999999999" } })) as unknown as typeof fetch, + }); + expect(telemetry.error).toMatch(/exceeded/); + }); + it("refuses to run without a key", async () => { await expect(invokeAgent(config("bare"), task, { backend: "anthropic", model: "m" })).rejects.toThrow(/ANTHROPIC_API_KEY/); }); diff --git a/shared/llm/anthropicWireIds.test.ts b/shared/llm/anthropicWireIds.test.ts index 7888a8941..9800f8aa4 100644 --- a/shared/llm/anthropicWireIds.test.ts +++ b/shared/llm/anthropicWireIds.test.ts @@ -16,6 +16,7 @@ import { anthropicText, anthropicWireId, FREE_LANE_CATALOG_ID, + FREE_LANE_WIRE_ID, getModelWithFailover, llmModelCatalog, resolveModelAlias, @@ -140,14 +141,39 @@ describe("getModelWithFailover fails closed", () => { it("names the same free lane as the resolver", () => { expect(FREE_LANE_CATALOG_ID).toBe(FREE_LANE_MODEL); + expect(FREE_LANE_WIRE_ID).toBe(MODEL_SPECS[FREE_LANE_MODEL].sdkId); }); - it("sends an unknown free-looking id to the free lane on OpenRouter, never to a paid OpenAI default", () => { + it("sends an unknown free-looking id to the free lane on OpenRouter as the WIRE id, never a paid default or the alias", () => { + // The returned `model` goes straight into the provider request, so the alias + // `nemotron-3-super-free` must never be what comes back (SPEC-v9 A.2.10 rule (b)). for (const id of ["retired-model-free", "someone/new-model:free"]) { - expect(getModelWithFailover(id), id).toEqual({ model: FREE_LANE_MODEL, provider: "openrouter", isFallback: true }); + expect(getModelWithFailover(id), id).toEqual({ model: FREE_LANE_WIRE_ID, provider: "openrouter", isFallback: true }); } }); + it("free route aliases take the free lane instead of throwing", () => { + for (const id of ["openrouter/free", "auto-free", "nodebench:auto-free", "kilo-auto/free", "free"]) { + expect(getModelWithFailover(id), id).toEqual({ model: FREE_LANE_WIRE_ID, provider: "openrouter", isFallback: true }); + } + }); + + it("a prefix-shaped invented id never fails over to a paid provider, even when only Gemini is configured", () => { + // Before: getProviderForModel inferred a provider from the prefix, so these missed the + // unknown-id branch and failed over to the paid gemini-3.1-flash-lite-preview. + vi.stubEnv("OPENAI_API_KEY", ""); + vi.stubEnv("ANTHROPIC_API_KEY", ""); + vi.stubEnv("OPENROUTER_API_KEY", ""); + vi.stubEnv("GEMINI_API_KEY", "gemini-test-key-not-real"); + for (const id of ["gpt-not-real", "claude-not-real"]) { + expect(() => getModelWithFailover(id), id).toThrow(/never fails over to a paid default/); + } + // free-looking wins over the prefix + expect(getModelWithFailover("gpt-not-real-free")).toEqual({ model: FREE_LANE_WIRE_ID, provider: "openrouter", isFallback: true }); + // a KNOWN catalog model still fails over as before + expect(getModelWithFailover("gpt-5.4").provider).toBe("gemini"); + }); + it("every model the catalog itself names still resolves, so no default now throws", () => { const named = new Set(Object.values(llmModelCatalog).flatMap((byTask) => Object.values(byTask).flat())); expect(named.size).toBeGreaterThan(10); @@ -161,7 +187,7 @@ describe("getModelWithFailover fails closed", () => { }); it("a configured free catalog model stays free, and an unconfigured provider never fails over to a paid one", () => { - expect(getModelWithFailover(FREE_LANE_MODEL)).toMatchObject({ provider: "openrouter", isFallback: false }); + expect(getModelWithFailover(FREE_LANE_MODEL)).toEqual({ model: FREE_LANE_WIRE_ID, provider: "openrouter", isFallback: false }); vi.stubEnv("OPENROUTER_API_KEY", ""); expect(() => getModelWithFailover(FREE_LANE_MODEL)).toThrow(/never fails over to a paid provider/); }); diff --git a/shared/llm/modelCatalog.ts b/shared/llm/modelCatalog.ts index 4709807e9..5bf62f71e 100644 --- a/shared/llm/modelCatalog.ts +++ b/shared/llm/modelCatalog.ts @@ -857,14 +857,24 @@ export function getEquivalentModel(modelName: string, targetProvider: LlmProvide return defaults[targetProvider]; } +/** + * The verified free lane's OpenRouter wire id. Must equal + * `MODEL_SPECS[FREE_LANE_MODEL].sdkId` in modelResolver.ts (this file cannot import from + * backend/convex); anthropicWireIds.test.ts pins the two together. SPEC-v9 A.2.10: the + * alias `nemotron-3-super-free` never reaches a provider. + */ +export const FREE_LANE_WIRE_ID = "nvidia/nemotron-3-super-120b-a12b:free"; + /** * The string to put in a provider request. Callers of `getModelWithFailover` * send `model` straight to the provider SDK, so for Anthropic it must be the * wire id, not the dotted catalog alias (api.anthropic.com 404s on - * "claude-sonnet-4.6"). An unrecognised string is passed through unchanged: - * it may already be an operator-supplied wire id. + * "claude-sonnet-4.6"), and the free-lane alias becomes its OpenRouter wire id. + * An unrecognised string is passed through unchanged: it may already be an + * operator-supplied wire id. */ function toProviderWireId(model: string, provider: LlmProvider): string { + if (provider === "openrouter" && model === FREE_LANE_CATALOG_ID) return FREE_LANE_WIRE_ID; return provider === "anthropic" ? (anthropicWireId(model) ?? model) : model; } @@ -875,9 +885,22 @@ function toProviderWireId(model: string, provider: LlmProvider): string { */ export const FREE_LANE_CATALOG_ID = "nemotron-3-super-free"; -/** Free-looking unknown ids: the OpenRouter `-free` / `:free` suffix. */ +/** The free route aliases; mirrors `routeAliasToModel` (routeTier "free") in modelResolver.ts. */ +const FREE_ROUTE_ALIASES = new Set(["free", "auto-free", "nodebench:auto-free", "kilo-auto/free", "openrouter/free"]); + +/** Free-looking unknown ids: the OpenRouter `-free` / `:free` suffix or a free route alias. */ function isFreeLookingModel(model: string): boolean { - return /(-|:)free$/.test(model.trim().toLowerCase()); + const normalized = model.trim().toLowerCase(); + return /(-|:)free$/.test(normalized) || FREE_ROUTE_ALIASES.has(normalized); +} + +/** + * A model this catalog knows by name. A provider prefix (`gpt-`, `claude-`, `gemini-`) is enough to + * route an operator-supplied wire id to its own provider (env overrides keep working), but it is not + * knowledge: an id that is only prefix-shaped never fails over to a different, paid provider. + */ +function isKnownCatalogModel(model: string): boolean { + return modelPricing[model] != null || modelEquivalents[model] != null || anthropicWireId(model) !== null; } function isFreeCatalogModel(model: string): boolean { @@ -887,8 +910,9 @@ function isFreeCatalogModel(model: string): boolean { /** * Get a working model with failover. - * An unknown id fails closed (SPEC-v9 A.0.10): free-looking -> the free lane, - * anything else throws. A free model never fails over to a paid provider. + * An unknown id fails closed (SPEC-v9 A.0.10): free-looking -> the free lane (wire id), + * anything else is passed only to the provider its prefix names when that provider is + * configured, and throws otherwise. A free model never fails over to a paid provider. * Tries the preferred model's provider first, then falls back to alternatives. * The returned `model` is the provider wire id (see `toProviderWireId`). */ @@ -897,20 +921,26 @@ export function getModelWithFailover(preferredModel: string): { provider: LlmProvider; isFallback: boolean; } { + const known = isKnownCatalogModel(preferredModel); const provider = getProviderForModel(preferredModel); - + + if (!known && isFreeLookingModel(preferredModel)) { + // The wire id, not the alias: the caller sends `model` to the provider as is. + return { model: FREE_LANE_WIRE_ID, provider: "openrouter", isFallback: true }; + } if (!provider) { - if (isFreeLookingModel(preferredModel)) { - return { model: FREE_LANE_CATALOG_ID, provider: "openrouter", isFallback: true }; - } throw new Error(`[getModelWithFailover] Unknown model "${preferredModel}"`); } - + // Check if preferred provider is configured if (isProviderConfigured(provider)) { return { model: toProviderWireId(preferredModel, provider), provider, isFallback: false }; } - + + if (!known) { + throw new Error(`[getModelWithFailover] Unknown model "${preferredModel}": ${provider} is not configured and an unrecognised id never fails over to a paid default`); + } + if (isFreeCatalogModel(preferredModel)) { throw new Error(`[getModelWithFailover] Free model "${preferredModel}": ${provider} is not configured and a free request never fails over to a paid provider`); } diff --git a/shared/search/triSearch.test.ts b/shared/search/triSearch.test.ts index 42855345d..91801df78 100644 --- a/shared/search/triSearch.test.ts +++ b/shared/search/triSearch.test.ts @@ -209,4 +209,23 @@ describe("rerankWithGemini - request shape (Gemini 3.x)", () => { expect(thinking).toEqual({ thinkingLevel: "low" }); expect(thinking).not.toHaveProperty("thinkingBudget"); }); + + it("condenseQuery sends the same thinking shape to the same model (thinkingBudget is removed on 3.6+)", async () => { + // Before: condenseQuery kept thinkingBudget:0 against RERANK_MODEL while the rerank call moved to + // thinkingLevel, so the rewrite would 400 and silently fall back to the raw question. + const bodies: any[] = []; + const urls: string[] = []; + const spyFetch = (async (url: string, init: { body: string }) => { + urls.push(String(url)); + bodies.push(JSON.parse(init.body)); + return { ok: true, status: 200, json: async () => ({ candidates: [{ content: { parts: [{ text: "rewritten query" }] } }] }) } as unknown as Response; + }) as unknown as typeof fetch; + const r = await condenseQuery("and the cost?", ["audit tool calls"], { apiKey: "test", fetchImpl: spyFetch }); + expect(r.status).toBe("rewritten"); + expect(urls[0]).toContain(`/models/${process.env.TRI_SEARCH_RERANK_MODEL || "gemini-3.5-flash"}:generateContent`); + const thinking = bodies[0].generationConfig.thinkingConfig; + expect(thinking).toEqual({ thinkingLevel: "low" }); + expect(thinking).not.toHaveProperty("thinkingBudget"); + expect(JSON.stringify(bodies[0])).not.toContain("thinkingBudget"); + }); }); diff --git a/shared/search/triSearch.ts b/shared/search/triSearch.ts index 9af73b190..fd213771b 100644 --- a/shared/search/triSearch.ts +++ b/shared/search/triSearch.ts @@ -264,7 +264,10 @@ export async function condenseQuery( headers: { "Content-Type": "application/json" }, body: JSON.stringify({ contents: [{ parts: [{ text: prompt }] }], - generationConfig: { temperature: 0, maxOutputTokens: 128, thinkingConfig: { thinkingBudget: 0 } }, + // Same Gemini 3.x model as the reranker: thinkingBudget is removed there (SPEC-v9 s1.3), so + // thinkingLevel "low" replaces it. Thinking counts toward the cap, so 256 (the rerank sizing) + // leaves room for a 400-char rewrite; a truncated reply would silently fall back to the raw question. + generationConfig: { temperature: 0, maxOutputTokens: 256, thinkingConfig: { thinkingLevel: "low" } }, }), signal: AbortSignal.timeout(opts.timeoutMs ?? RERANK_TIMEOUT_MS), }, From a9e0906794e2700514a6d12dbcbc759aca348295 Mon Sep 17 00:00:00 2001 From: homen Date: Thu, 15 Oct 2026 12:08:42 -0700 Subject: [PATCH 4/8] fix(models): close the round 3 review findings (PR 1) Cost class (A.0.10): - selectTierEligibleRuntimeModel no longer appends the paid tier-safe fallbacks or the billing gate's paid suggestion for a free primary (requireFree, default = primary is free); on the anonymous and free tiers a free-lane request now fails with the tier reason instead of being answered by gpt-5.4-nano / gpt-5.4-mini. The post-stream fallback passes requireFree for a free active model. - getModelWithFailover throws for any id that is neither in the catalog nor a known wire id, before provider availability is checked; dailyBriefWorker.runNextTask and dailyBriefPersonalOverlay.runNextPersonalTask validate model with normalizeModelInput. - executeWithModelFallback, getFallbackChain and getLanguageModelWithFallback resolve the start model through resolveOrFailClosed first (free-looking or the free-lane wire id -> FREE_LANE_MODEL, other unknown -> UnknownModelError) instead of a TypeError on pricing. - modelRouter.route resolves the candidate once and uses the resolved alias for dispatch, pricing, isFree, actualTier, cache and the audit row. Request shape: Gemini sampling (temperature) removed from both triSearch calls, the benchmark Gemini judge and judgeScratchnodeChatVideo.mjs; the benchmark test no longer pins temperature 0 for Gemini and every changed builder asserts the keys are absent. Tests: failing-before/passing-after for each defect; two new files added to the CI runtime-smoke list (47 files, 619 tests pass at this head). Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/ci.yml | 2 + .../agentRuntimeTierFallback.test.ts | 60 +++++++++++++++++++ .../domains/agents/fastAgentPanelStreaming.ts | 2 + .../mcp_tools/models/modelResolver.test.ts | 47 +++++++++++++++ .../agents/mcp_tools/models/modelResolver.ts | 9 ++- .../domains/agents/runtimeTierFallback.ts | 11 +++- .../models/modelRouter.requestShape.test.ts | 20 +++++++ .../convex/domains/ai/models/modelRouter.ts | 11 +++- .../research/dailyBriefModelBoundary.test.ts | 54 +++++++++++++++++ .../research/dailyBriefPersonalOverlay.ts | 5 +- .../domains/research/dailyBriefWorker.ts | 5 +- .../fusion/benchmark.requestShape.test.ts | 8 ++- .../convex/domains/search/fusion/benchmark.ts | 2 +- ...eScratchnodeChatVideo.requestShape.test.ts | 55 +++++++++++++++++ scripts/ui/judgeScratchnodeChatVideo.mjs | 2 +- shared/llm/anthropicWireIds.test.ts | 25 +++++++- shared/llm/modelCatalog.ts | 22 +++---- shared/search/triSearch.test.ts | 10 ++++ shared/search/triSearch.ts | 4 +- 19 files changed, 324 insertions(+), 30 deletions(-) create mode 100644 backend/convex/domains/research/dailyBriefModelBoundary.test.ts create mode 100644 scripts/__tests__/judgeScratchnodeChatVideo.requestShape.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 717334fc2..b3b32fa88 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -116,6 +116,8 @@ jobs: backend/convex/domains/agents/runtimeRouting.test.ts backend/convex/domains/verification/instagramClaimVerification.test.ts backend/convex/domains/agents/agentChatActions.failClosed.test.ts + backend/convex/domains/research/dailyBriefModelBoundary.test.ts + scripts/__tests__/judgeScratchnodeChatVideo.requestShape.test.ts packages/mcp-local/src/__tests__/anthropicRequestShape.test.ts apps/web/src/features/agents/components/FastAgentPanel/__tests__/RunState.test.tsx backend/convex/domains/agents/autonomy/autonomy.integration.test.ts diff --git a/backend/convex/__tests__/agentRuntimeTierFallback.test.ts b/backend/convex/__tests__/agentRuntimeTierFallback.test.ts index 00cef04b7..ec55f78dd 100644 --- a/backend/convex/__tests__/agentRuntimeTierFallback.test.ts +++ b/backend/convex/__tests__/agentRuntimeTierFallback.test.ts @@ -17,6 +17,13 @@ import { formatPublicAgentRunError, projectAgentRunPresentation, } from "../domains/agents/agentRunPresentation"; +import { + getBestModelForTier, + getProviderForModel, + isModelAllowedForTier, + type UserTier, +} from "../../../shared/llm/modelCatalog"; +import { FREE_LANE_MODEL } from "../domains/agents/mcp_tools/models/modelResolver"; import { setEffectiveModel } from "../domains/agents/orchestrator/queueProtocol"; import { checkRequestAllowed, @@ -176,6 +183,59 @@ describe("selectTierEligibleRuntimeModel", () => { }); }); + // A.0.10: the real tier policy (checkRequestAccess' own calls), not a stub. + const realTierCheck = (tier: UserTier) => async (model: string) => + isModelAllowedForTier(model, tier) + ? allowed() + : denied( + `Model "${model}" is not available on the ${tier} tier`, + getBestModelForTier("chat", tier, getProviderForModel(model) || "openai"), + ); + + it.each(["anonymous", "free"] as const)( + "never answers a free-lane request with a paid model on the %s tier", + async (tier) => { + const checkModel = vi.fn(realTierCheck(tier)); + const result = await selectTierEligibleRuntimeModel({ + primaryModel: FREE_LANE_MODEL, + fallbackModels: [], + checkModel, + }); + + expect(result.model).toBeNull(); + expect(checkModel.mock.calls.map(([m]) => m)).toEqual([FREE_LANE_MODEL]); + expect(getTierSelectionFailureReason(result)).toContain( + `not available on the ${tier} tier`, + ); + }, + ); + + it("still serves the free lane where the tier allows it", async () => { + const result = await selectTierEligibleRuntimeModel({ + primaryModel: FREE_LANE_MODEL, + checkModel: realTierCheck("pro"), + }); + expect(result.model).toBe(FREE_LANE_MODEL); + }); + + it("keeps a failed free run free when choosing a post-stream fallback", async () => { + const result = await selectTierEligibleRuntimeModel({ + primaryModel: "gpt-5.4-mini", + excludedModels: [FREE_LANE_MODEL], + requireFree: true, + checkModel: realTierCheck("pro"), + }); + expect(result.model).toBeNull(); + }); + + it("preserves the paid fallbacks for a paid primary", async () => { + const result = await selectTierEligibleRuntimeModel({ + primaryModel: "kimi-k2.6", + checkModel: realTierCheck("anonymous"), + }); + expect(result.model).toBe("gpt-5.4-nano"); + }); + it("skips already-attempted models when choosing a tier-eligible fallback", async () => { const checkModel = vi.fn(async () => allowed()); const result = await selectTierEligibleRuntimeModel({ diff --git a/backend/convex/domains/agents/fastAgentPanelStreaming.ts b/backend/convex/domains/agents/fastAgentPanelStreaming.ts index e2521aaf1..a289b7737 100644 --- a/backend/convex/domains/agents/fastAgentPanelStreaming.ts +++ b/backend/convex/domains/agents/fastAgentPanelStreaming.ts @@ -445,6 +445,7 @@ import { getLanguageModelSafe, normalizeModelInput, normalizeNodeBenchRuntimeModel, + isFreeModel, DEFAULT_MODEL, isModelAvailable, type ApprovedModel, @@ -4806,6 +4807,7 @@ export const streamAsync = internalAction({ ...route.fallbackModels, ], excludedModels: [activeModel, ...attemptedModels], + requireFree: isFreeModel(activeModel), isModelAvailable, checkModel: checkRuntimeModelAccess, }); diff --git a/backend/convex/domains/agents/mcp_tools/models/modelResolver.test.ts b/backend/convex/domains/agents/mcp_tools/models/modelResolver.test.ts index 028693331..683826aa5 100644 --- a/backend/convex/domains/agents/mcp_tools/models/modelResolver.test.ts +++ b/backend/convex/domains/agents/mcp_tools/models/modelResolver.test.ts @@ -269,6 +269,53 @@ describe("an approved PAID model outside the priority order keeps main's behavio }); }); +describe("the fallback helpers resolve an unvalidated start model first (A.0.10, PR #662 round 2)", () => { + // pdfInsights.generatePDFInsights casts a user-supplied `args.model` to ApprovedModel. Persona: a + // signed-in user posts a retired free alias, a typo, or the OpenRouter wire id of the free lane. + // Before: priorityIndexOrThrow called isFreeModel on the raw string, so each one died with + // "Cannot read properties of undefined (reading 'pricing')" before the callback ran. + const FREE_WIRE = MODEL_SPECS[FREE_LANE_MODEL].sdkId; + beforeEach(() => { + vi.stubEnv("OPENROUTER_API_KEY", "test-key-not-real"); + vi.spyOn(console, "warn").mockImplementation(() => {}); + }); + afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + }); + + it.each(["x-free", "retired-model-free", "someone/new:free"])("free-looking unknown %s resolves to the free lane in all three helpers", async (id) => { + const r = await executeWithModelFallback(async (_m, used) => used, { startModel: id as never }); + expect(r.modelUsed).toBe(FREE_LANE_MODEL); + expect(r.isFree).toBe(true); + const chain = getFallbackChain(id as never); + expect(chain[0]).toBe(FREE_LANE_MODEL); + expect(chain.every((m) => isFreeModel(m))).toBe(true); + expect(getLanguageModelWithFallback(id as never).modelId).toBe(FREE_LANE_MODEL); + }); + + it("the registered free-lane wire id resolves through the alias table", async () => { + expect(FREE_WIRE).toBe("nvidia/nemotron-3-super-120b-a12b:free"); + const r = await executeWithModelFallback(async (_m, used) => used, { startModel: FREE_WIRE as never }); + expect(r.modelUsed).toBe(FREE_LANE_MODEL); + expect(getFallbackChain(FREE_WIRE as never)[0]).toBe(FREE_LANE_MODEL); + expect(getLanguageModelWithFallback(FREE_WIRE as never).modelId).toBe(FREE_LANE_MODEL); + }); + + it("any other unknown id throws UnknownModelError before the callback runs", async () => { + const fn = vi.fn(async () => "x"); + await expect(executeWithModelFallback(fn, { startModel: "not-a-model" as never })).rejects.toBeInstanceOf(UnknownModelError); + expect(fn).not.toHaveBeenCalled(); + expect(() => getFallbackChain("not-a-model" as never)).toThrow(UnknownModelError); + expect(() => getLanguageModelWithFallback("not-a-model" as never)).toThrow(UnknownModelError); + }); + + it("an omitted start model still means DEFAULT_MODEL", () => { + expect(getLanguageModelWithFallback().modelId).toBe(DEFAULT_MODEL); + expect(getFallbackChain()).toEqual(getFallbackChain(DEFAULT_MODEL)); + }); +}); + describe("normalizeNodeBenchRuntimeModel is the first resolver and fails closed", () => { beforeEach(() => { vi.spyOn(console, "warn").mockImplementation(() => {}); diff --git a/backend/convex/domains/agents/mcp_tools/models/modelResolver.ts b/backend/convex/domains/agents/mcp_tools/models/modelResolver.ts index 5cb5ea6d0..96e02466b 100644 --- a/backend/convex/domains/agents/mcp_tools/models/modelResolver.ts +++ b/backend/convex/domains/agents/mcp_tools/models/modelResolver.ts @@ -1280,7 +1280,9 @@ export function getLanguageModelWithFallback(preferredModel?: ApprovedModel): { isFree: boolean; fallbackChain: ApprovedModel[]; } { - const modelId = preferredModel ?? DEFAULT_MODEL; + // The start model arrives from public actions as an unvalidated string (pdfInsights): resolve it + // first, so an unknown id throws UnknownModelError / a free-looking one takes the free lane. + const modelId = resolveOrFailClosed(preferredModel, undefined, "getLanguageModelWithFallback"); const spec = MODEL_SPECS[modelId]; const index = priorityIndexOrThrow(modelId, "getLanguageModelWithFallback"); @@ -1310,7 +1312,7 @@ export async function executeWithModelFallback( ): Promise<{ result: T; modelUsed: ApprovedModel; isFree: boolean; fallbacksUsed: number }> { const { startModel = DEFAULT_MODEL, maxRetries = RETRY_CONFIG.maxRetries, onFallback } = options ?? {}; - let currentModel = startModel; + let currentModel = resolveOrFailClosed(startModel, undefined, "executeWithModelFallback"); let fallbacksUsed = 0; let lastError: Error | null = null; @@ -1393,5 +1395,6 @@ export function getBestFreeModel(): ApprovedModel { * Get the fallback chain starting from a given model */ export function getFallbackChain(startModel: ApprovedModel = DEFAULT_MODEL): ApprovedModel[] { - return MODEL_PRIORITY_ORDER.slice(Math.max(0, priorityIndexOrThrow(startModel, "getFallbackChain"))); + const start = resolveOrFailClosed(startModel, undefined, "getFallbackChain"); + return MODEL_PRIORITY_ORDER.slice(Math.max(0, priorityIndexOrThrow(start, "getFallbackChain"))); } diff --git a/backend/convex/domains/agents/runtimeTierFallback.ts b/backend/convex/domains/agents/runtimeTierFallback.ts index c1dec2099..db3c65944 100644 --- a/backend/convex/domains/agents/runtimeTierFallback.ts +++ b/backend/convex/domains/agents/runtimeTierFallback.ts @@ -1,5 +1,6 @@ import { isApprovedModel, + isFreeModel, resolveModelAlias, type ApprovedModel, } from "./mcp_tools/models/modelResolver"; @@ -238,8 +239,14 @@ export async function selectTierEligibleRuntimeModel(args: { excludedModels?: readonly ApprovedModel[]; checkModel: (model: ApprovedModel) => Promise; isModelAvailable?: (model: ApprovedModel) => boolean; + /** + * A.0.10: a free request never becomes a paid one. Defaults to the primary's + * cost class; a caller falling back from a free run passes true explicitly. + */ + requireFree?: boolean; }): Promise { const isModelAvailable = args.isModelAvailable ?? (() => true); + const requireFree = args.requireFree ?? isFreeModel(args.primaryModel); const queue: ApprovedModel[] = []; const seen = new Set(); const excluded = new Set(args.excludedModels ?? []); @@ -247,6 +254,7 @@ export async function selectTierEligibleRuntimeModel(args: { const enqueue = (model: ApprovedModel, index?: number) => { if (seen.has(model) || excluded.has(model)) return; + if (requireFree && !isFreeModel(model)) return; seen.add(model); if (typeof index === "number") { queue.splice(index, 0, model); @@ -257,7 +265,8 @@ export async function selectTierEligibleRuntimeModel(args: { enqueue(args.primaryModel); for (const model of args.fallbackModels ?? []) enqueue(model); - for (const model of TIER_SAFE_RUNTIME_FALLBACKS) enqueue(model); + // The tier-safe list is all paid; a free request must fail with the tier reason instead. + if (!requireFree) for (const model of TIER_SAFE_RUNTIME_FALLBACKS) enqueue(model); for (let index = 0; index < queue.length; index += 1) { const model = queue[index]; diff --git a/backend/convex/domains/ai/models/modelRouter.requestShape.test.ts b/backend/convex/domains/ai/models/modelRouter.requestShape.test.ts index c35ad0319..a1b31f95e 100644 --- a/backend/convex/domains/ai/models/modelRouter.requestShape.test.ts +++ b/backend/convex/domains/ai/models/modelRouter.requestShape.test.ts @@ -118,6 +118,26 @@ describe("modelRouter.route request shape and cost class", () => { expect(logged).toMatchObject({ modelId: "nemotron-3-super-free", actualTier: "free", costUsd: 0 }); }); + it.each(["auto-free", "nvidia/nemotron-3-super-120b-a12b:free", "invented-free"])( + "a pinned free alias or wire id (%s) is priced, classified, cached and logged as the free lane it dispatched to", + async (pinnedModel) => { + // Before: the raw string was priced ($1/$5 default), isFree false, actualTier "premium" even + // though the request went to the free lane (audit data, PR #662 round 2). + const ctx = makeCtx(); + const result = await call(ctx, { tier: "free", pinnedModel, cacheKey: `pin-${pinnedModel}` }); + expect((generateText.mock.calls[0][0].model as { modelId: string }).modelId).toBe("nvidia/nemotron-3-super-120b-a12b:free"); + expect(result).toMatchObject({ modelId: "nemotron-3-super-free", isFree: true, costUsd: 0 }); + expect(result.routing.actualTier).toBe("free"); + const logged = (ctx.runMutation.mock.calls[0] as unknown as [unknown, Record])[1]; + expect(logged).toMatchObject({ modelId: "nemotron-3-super-free", actualTier: "free", costUsd: 0, pinnedModel }); + + generateText.mockClear(); + const cached = await call(makeCtx(), { tier: "free", pinnedModel, cacheKey: `pin-${pinnedModel}` }); + expect(generateText).not.toHaveBeenCalled(); + expect(cached).toMatchObject({ modelId: "nemotron-3-super-free", isFree: true, fromCache: true }); + }, + ); + it("a cheap-tier outage still falls through the cheap list (the paid list is only for paid tiers)", async () => { resolveChain.mockReturnValue({ chain: [], reason: "no_matches_for_requirement", diagnostics: { primaryOutcome: "not_supplied", candidatePoolSize: 0 } }); generateText.mockRejectedValue(Object.assign(new Error("provider down"), { status: 400 })); diff --git a/backend/convex/domains/ai/models/modelRouter.ts b/backend/convex/domains/ai/models/modelRouter.ts index 9446c575c..cbdf71d76 100644 --- a/backend/convex/domains/ai/models/modelRouter.ts +++ b/backend/convex/domains/ai/models/modelRouter.ts @@ -16,7 +16,7 @@ import { v } from "convex/values"; import { internalAction } from "../../../_generated/server"; import { internal } from "../../../_generated/api"; import { generateText } from "ai"; -import { getLanguageModelSafe, getProviderForModel, isFreeModel, resolveModelAlias } from "../../agents/mcp_tools/models"; +import { getLanguageModelSafe, getProviderForModel, isFreeModel, normalizeModelInput, resolveModelAlias } from "../../agents/mcp_tools/models"; // B-PR4: capability-aware fallback chain. // resolveChain replaces the static `[...TIER_MODELS[tier]]` candidate list // with a tier-floor-enforced ordering that respects requested capabilities @@ -436,8 +436,13 @@ export const route = internalAction({ failedWith: number | string; }> = []; - for (const modelId of candidates) { + for (const candidate of candidates) { + // Resolve once: dispatch, pricing, isFree, actualTier, cache and audit all use the model that + // actually answers. A pinned 'auto-free', the free-lane wire id or an unknown '-free' id is + // dispatched to the free lane and must be recorded as free, not priced as the raw string. + let modelId = candidate; try { + modelId = normalizeModelInput(candidate); const model = getLanguageModelSafe(modelId); if (!model) continue; @@ -556,7 +561,7 @@ export const route = internalAction({ typeof rawStatus === "number" ? rawStatus : (lastError.message || "error").slice(0, 80); - failedAttempts.push({ modelId, failedWith }); + failedAttempts.push({ modelId: candidate, failedWith }); // Retry on 429/503 with backoff if ( diff --git a/backend/convex/domains/research/dailyBriefModelBoundary.test.ts b/backend/convex/domains/research/dailyBriefModelBoundary.test.ts new file mode 100644 index 000000000..84caf4942 --- /dev/null +++ b/backend/convex/domains/research/dailyBriefModelBoundary.test.ts @@ -0,0 +1,54 @@ +// @vitest-environment node +/** + * The daily-brief public actions take `model: v.string()` from an authenticated browser and used to + * hand it unchanged to executeFeature -> generateWithProvider -> getModelWithFailover -> the provider + * SDK (SPEC-v9 A.0.10; PR #662 review round 2). Persona: a signed-in user who types a model name the + * repo does not know ("gpt-5.5-pro"). The action must reject it before any internal action runs, so + * no provider call and no spend. Known ids and free-looking ids still go through, normalised. + * Fake ctx; no network. + */ + +import { describe, expect, it, vi } from "vitest"; + +vi.mock("@convex-dev/auth/server", () => ({ getAuthUserId: vi.fn(async () => "user1") })); + +import { runNextPersonalTask } from "./dailyBriefPersonalOverlay"; +import { runNextTask } from "./dailyBriefWorker"; +import { UnknownModelError } from "../agents/mcp_tools/models"; + +const actions = [ + ["dailyBriefWorker.runNextTask", runNextTask], + ["dailyBriefPersonalOverlay.runNextPersonalTask", runNextPersonalTask], +] as const; + +describe.each(actions)("%s model boundary", (_name, action) => { + const run = (args: Record) => { + const ctx = { runAction: vi.fn(async () => ({ done: true })) }; + const result = (action as any)._handler(ctx, args); + return { ctx, result }; + }; + + it.each(["gpt-not-real", "gpt-5.5-pro", "not-a-model"])("rejects unknown id %s before any internal action runs", async (model) => { + vi.spyOn(console, "warn").mockImplementation(() => {}); + const { ctx, result } = run({ model }); + await expect(result).rejects.toBeInstanceOf(UnknownModelError); + expect(ctx.runAction).not.toHaveBeenCalled(); + }); + + it("resolves a free-looking unknown id to the free lane, never a paid model", async () => { + vi.spyOn(console, "warn").mockImplementation(() => {}); + const { ctx, result } = run({ model: "retired-model-free", taskId: "t1" }); + await result; + expect(ctx.runAction).toHaveBeenCalledWith(expect.anything(), { model: "nemotron-3-super-free", taskId: "t1" }); + }); + + it("forwards a known id as its approved alias and leaves an omitted model omitted", async () => { + const known = run({ model: "gpt-5.4-mini" }); + await known.result; + expect(known.ctx.runAction).toHaveBeenCalledWith(expect.anything(), { model: "gpt-5.4-mini" }); + + const omitted = run({ taskId: "t1" }); + await omitted.result; + expect(omitted.ctx.runAction).toHaveBeenCalledWith(expect.anything(), { taskId: "t1" }); + }); +}); diff --git a/backend/convex/domains/research/dailyBriefPersonalOverlay.ts b/backend/convex/domains/research/dailyBriefPersonalOverlay.ts index 8df91f3ca..8994f33d0 100644 --- a/backend/convex/domains/research/dailyBriefPersonalOverlay.ts +++ b/backend/convex/domains/research/dailyBriefPersonalOverlay.ts @@ -11,6 +11,7 @@ import { v } from "convex/values"; import { internalAction, action } from "../../_generated/server"; import { internal, api } from "../../_generated/api"; import { getAuthUserId } from "@convex-dev/auth/server"; +import { normalizeModelInput } from "../agents/mcp_tools/models/modelResolver"; import OpenAI from "openai"; import Anthropic from "@anthropic-ai/sdk"; @@ -373,9 +374,11 @@ export const runNextPersonalTask = action({ handler: async (ctx, args): Promise => { const userId = await getAuthUserId(ctx); if (!userId) throw new Error("Not authenticated"); + // A.0.10: an unknown model id throws here, before any provider is reached. + const model = args.model === undefined ? undefined : normalizeModelInput(args.model); return await ctx.runAction( internal.domains.research.dailyBriefPersonalOverlay.runNextPersonalTaskInternal, - args, + model === undefined ? args : { ...args, model }, ); }, }); diff --git a/backend/convex/domains/research/dailyBriefWorker.ts b/backend/convex/domains/research/dailyBriefWorker.ts index 1d6d2c322..075bde86f 100644 --- a/backend/convex/domains/research/dailyBriefWorker.ts +++ b/backend/convex/domains/research/dailyBriefWorker.ts @@ -10,6 +10,7 @@ import { v } from "convex/values"; import { internalAction, action } from "../../_generated/server"; import { internal, api } from "../../_generated/api"; import { getAuthUserId } from "@convex-dev/auth/server"; +import { normalizeModelInput } from "../agents/mcp_tools/models/modelResolver"; import OpenAI from "openai"; import Anthropic from "@anthropic-ai/sdk"; @@ -598,10 +599,12 @@ export const runNextTask = action({ handler: async (ctx, args): Promise => { const userId = await getAuthUserId(ctx); if (!userId) throw new Error("Not authenticated"); + // A.0.10: an unknown model id throws here, before any provider is reached. + const model = args.model === undefined ? undefined : normalizeModelInput(args.model); const res: any = await ctx.runAction( internal.domains.research.dailyBriefWorker.runNextTaskInternal, - args, + model === undefined ? args : { ...args, model }, ); return res; }, diff --git a/backend/convex/domains/search/fusion/benchmark.requestShape.test.ts b/backend/convex/domains/search/fusion/benchmark.requestShape.test.ts index 17351781c..fa2adfa01 100644 --- a/backend/convex/domains/search/fusion/benchmark.requestShape.test.ts +++ b/backend/convex/domains/search/fusion/benchmark.requestShape.test.ts @@ -70,9 +70,13 @@ describe("benchmark.evaluateSearch judge request shape", () => { expect(captured[0].body.temperature).toBe(0); }); - it("Gemini judge keeps temperature pinned to 0 in generationConfig, with the model in the URL", async () => { + it("Gemini judge sends no sampling keys (deprecated on Gemini 3.x, SPEC-v9 s1.3), with the model in the URL", async () => { + // Round 3: this test used to require generationConfig.temperature === 0 and would have blocked + // the removal; ignored today, a 400 later. The determinism pin stays on the OpenAI and the + // other judges. const { captured } = await run("gemini-3.1-pro-preview"); expect(captured[0].url).toContain("/models/gemini-3.1-pro-preview:generateContent"); - expect(captured[0].body.generationConfig.temperature).toBe(0); + for (const key of ["temperature", "topP", "topK", "top_p", "top_k"]) expect(captured[0].body.generationConfig, key).not.toHaveProperty(key); + expect(captured[0].body.generationConfig.maxOutputTokens).toBe(1500); }); }); diff --git a/backend/convex/domains/search/fusion/benchmark.ts b/backend/convex/domains/search/fusion/benchmark.ts index 2316d8d57..30fb0e57b 100644 --- a/backend/convex/domains/search/fusion/benchmark.ts +++ b/backend/convex/domains/search/fusion/benchmark.ts @@ -610,7 +610,7 @@ export const evaluateSearch = action({ headers: { "Content-Type": "application/json" }, body: JSON.stringify({ contents: [{ parts: [{ text: `You are an expert search quality evaluator. Respond only with valid JSON.\n\n${prompt}` }] }], - generationConfig: { temperature: JUDGE_TEMPERATURE, maxOutputTokens: 1500 }, // Pinned to 0 for determinism + generationConfig: { maxOutputTokens: 1500 }, // no sampling: deprecated on Gemini 3.x (SPEC-v9 s1.3); was temperature 0 }), } ); diff --git a/scripts/__tests__/judgeScratchnodeChatVideo.requestShape.test.ts b/scripts/__tests__/judgeScratchnodeChatVideo.requestShape.test.ts new file mode 100644 index 000000000..af1c79c98 --- /dev/null +++ b/scripts/__tests__/judgeScratchnodeChatVideo.requestShape.test.ts @@ -0,0 +1,55 @@ +// @vitest-environment node +/** + * Request-shape test for the ScratchNode video judge (SPEC-v9 A.0.6, s1.3). The judge talks to + * a Gemini 3.x model, where sampling (`temperature`/`topP`/`topK`) is deprecated: ignored today, + * a 400 later. Persona: a maintainer running the judge on a recording; what the script sends is + * what Google accepts. The script runs on import, so this drives it for real with a fake + * `@google/genai` client and asserts the outgoing `generateContent` request. No network, no spend. + */ + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const generateContent = vi.hoisted(() => vi.fn()); +vi.mock("@google/genai", () => ({ + GoogleGenAI: class { + files = { + upload: async () => ({ name: "files/1" }), + get: async () => ({ state: "ACTIVE", uri: "https://files/1" }), + delete: async () => undefined, + }; + models = { generateContent }; + }, +})); + +describe("judgeScratchnodeChatVideo.mjs Gemini request", () => { + let video: string; + beforeEach(() => { + video = path.join(fs.mkdtempSync(path.join(os.tmpdir(), "judge-")), "clip.webm"); + fs.writeFileSync(video, "not a real video"); + vi.stubEnv("GEMINI_API_KEY", "test-key-not-real"); + vi.spyOn(process, "argv", "get").mockReturnValue(["node", "judge.mjs", "--video", video]); + vi.spyOn(console, "log").mockImplementation(() => {}); + generateContent.mockReset(); + generateContent.mockResolvedValue({ text: '{"readiness_score": 70}' }); + vi.resetModules(); + }); + afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + }); + + it("sends thinkingLevel low and no sampling keys", async () => { + await import("../ui/judgeScratchnodeChatVideo.mjs"); + await vi.waitFor(() => expect(generateContent).toHaveBeenCalledTimes(1)); + const request = generateContent.mock.calls[0][0]; + expect(request.model).toBe("gemini-3.5-flash"); + for (const key of ["temperature", "topP", "topK", "top_p", "top_k"]) { + expect(request.config, key).not.toHaveProperty(key); + } + expect(request.config.thinkingConfig).toEqual({ thinkingLevel: "low" }); + expect(request.config.maxOutputTokens).toBe(3500); + }); +}); diff --git a/scripts/ui/judgeScratchnodeChatVideo.mjs b/scripts/ui/judgeScratchnodeChatVideo.mjs index 0901e3d84..c9c8fa630 100644 --- a/scripts/ui/judgeScratchnodeChatVideo.mjs +++ b/scripts/ui/judgeScratchnodeChatVideo.mjs @@ -73,7 +73,7 @@ Return ONLY JSON: const result = await ai.models.generateContent({ model: MODEL, contents: [{ role: "user", parts: [{ fileData: { fileUri: file.uri, mimeType: "video/webm" } }, { text: PROMPT }] }], - config: { responseMimeType: "application/json", temperature: 0.2, maxOutputTokens: 3500, thinkingConfig: { thinkingLevel: "low" } }, + config: { responseMimeType: "application/json", maxOutputTokens: 3500, thinkingConfig: { thinkingLevel: "low" } }, }); const text = (result.text || "").trim(); let parsed; diff --git a/shared/llm/anthropicWireIds.test.ts b/shared/llm/anthropicWireIds.test.ts index 9800f8aa4..32bcab67a 100644 --- a/shared/llm/anthropicWireIds.test.ts +++ b/shared/llm/anthropicWireIds.test.ts @@ -91,9 +91,15 @@ describe("getModelWithFailover sends wire ids to Anthropic", () => { }); }); - it("passes an operator-supplied claude string through untouched (env override precedence)", () => { + it("passes a known wire id through untouched", () => { + // Round 3 (A.0.10): this used to pass the unknown snapshot "claude-haiku-4-5-20251001" through + // on its `claude-` prefix alone. No caller supplies one (getLlmModel has no env override), the + // public actions normalise user input to the alias first, and a prefix is not knowledge. vi.stubEnv("ANTHROPIC_API_KEY", "sk-ant-test-key-not-real"); - expect(getModelWithFailover("claude-haiku-4-5-20251001").model).toBe("claude-haiku-4-5-20251001"); + for (const wire of Object.values(ANTHROPIC_WIRE_IDS)) { + expect(getModelWithFailover(wire), wire).toMatchObject({ model: wire, provider: "anthropic", isFallback: false }); + } + expect(() => getModelWithFailover("claude-haiku-4-5-20251001")).toThrow(/Unknown model/); }); it("leaves non-Anthropic providers alone", () => { @@ -180,6 +186,21 @@ describe("getModelWithFailover fails closed", () => { for (const id of named) expect(() => getModelWithFailover(resolveModelAlias(id)), id).not.toThrow(); }); + it("an invented or unlisted paid id throws even when its provider is configured", () => { + // Round 3 (A.0.10): with OPENAI_API_KEY set these returned {provider:"openai", isFallback:false} + // and reached openai.chat.completions.create from the public daily-brief actions. + for (const id of ["gpt-not-real", "gpt-5.5-pro", "claude-not-real", "gemini-not-real"]) { + expect(() => getModelWithFailover(id), id).toThrow(/Unknown model/); + } + }); + + it("every resolver spec the catalog can route resolves by its sdkId", () => { + for (const spec of Object.values(MODEL_SPECS)) { + if (spec.provider !== "openai" && spec.provider !== "google") continue; + expect(() => getModelWithFailover(spec.sdkId), spec.sdkId).not.toThrow(); + } + }); + it("throws for any other unknown id", () => { for (const id of ["not-a-model", "freeloader-pro", "mistral-unknown"]) { expect(() => getModelWithFailover(id), id).toThrow(/Unknown model/); diff --git a/shared/llm/modelCatalog.ts b/shared/llm/modelCatalog.ts index 5bf62f71e..49102be48 100644 --- a/shared/llm/modelCatalog.ts +++ b/shared/llm/modelCatalog.ts @@ -870,8 +870,8 @@ export const FREE_LANE_WIRE_ID = "nvidia/nemotron-3-super-120b-a12b:free"; * send `model` straight to the provider SDK, so for Anthropic it must be the * wire id, not the dotted catalog alias (api.anthropic.com 404s on * "claude-sonnet-4.6"), and the free-lane alias becomes its OpenRouter wire id. - * An unrecognised string is passed through unchanged: it may already be an - * operator-supplied wire id. + * Only known ids reach here (`getModelWithFailover` throws on the rest), so an id + * with no table entry is already its own wire id (openai, gemini: sdkId == alias). */ function toProviderWireId(model: string, provider: LlmProvider): string { if (provider === "openrouter" && model === FREE_LANE_CATALOG_ID) return FREE_LANE_WIRE_ID; @@ -895,9 +895,9 @@ function isFreeLookingModel(model: string): boolean { } /** - * A model this catalog knows by name. A provider prefix (`gpt-`, `claude-`, `gemini-`) is enough to - * route an operator-supplied wire id to its own provider (env overrides keep working), but it is not - * knowledge: an id that is only prefix-shaped never fails over to a different, paid provider. + * A model this catalog knows by name or by wire id (`ANTHROPIC_WIRE_IDS` values; for openai and + * gemini the resolver's sdkId equals the catalog name, pinned by anthropicWireIds.test.ts). A provider + * prefix (`gpt-`, `claude-`, `gemini-`) is not knowledge: an id that is only prefix-shaped is unknown. */ function isKnownCatalogModel(model: string): boolean { return modelPricing[model] != null || modelEquivalents[model] != null || anthropicWireId(model) !== null; @@ -911,8 +911,8 @@ function isFreeCatalogModel(model: string): boolean { /** * Get a working model with failover. * An unknown id fails closed (SPEC-v9 A.0.10): free-looking -> the free lane (wire id), - * anything else is passed only to the provider its prefix names when that provider is - * configured, and throws otherwise. A free model never fails over to a paid provider. + * anything else throws before any provider is consulted, so a public API argument can + * never reach a paid provider unvalidated. A free model never fails over to a paid provider. * Tries the preferred model's provider first, then falls back to alternatives. * The returned `model` is the provider wire id (see `toProviderWireId`). */ @@ -928,8 +928,8 @@ export function getModelWithFailover(preferredModel: string): { // The wire id, not the alias: the caller sends `model` to the provider as is. return { model: FREE_LANE_WIRE_ID, provider: "openrouter", isFallback: true }; } - if (!provider) { - throw new Error(`[getModelWithFailover] Unknown model "${preferredModel}"`); + if (!known || !provider) { + throw new Error(`[getModelWithFailover] Unknown model "${preferredModel}": an unrecognised id never fails over to a paid default and is never sent to a provider`); } // Check if preferred provider is configured @@ -937,10 +937,6 @@ export function getModelWithFailover(preferredModel: string): { return { model: toProviderWireId(preferredModel, provider), provider, isFallback: false }; } - if (!known) { - throw new Error(`[getModelWithFailover] Unknown model "${preferredModel}": ${provider} is not configured and an unrecognised id never fails over to a paid default`); - } - if (isFreeCatalogModel(preferredModel)) { throw new Error(`[getModelWithFailover] Free model "${preferredModel}": ${provider} is not configured and a free request never fails over to a paid provider`); } diff --git a/shared/search/triSearch.test.ts b/shared/search/triSearch.test.ts index 91801df78..4bae16e62 100644 --- a/shared/search/triSearch.test.ts +++ b/shared/search/triSearch.test.ts @@ -182,6 +182,14 @@ describe("condenseQuery — multi-turn query rewrite", () => { }); }); +// Sampling is deprecated on Gemini 3.x (ignored today, a 400 later; SPEC-v9 s1.3). Both calls used to +// send generationConfig.temperature 0. +const expectNoSampling = (body: any) => { + for (const key of ["temperature", "topP", "topK", "top_p", "top_k"]) { + expect(body.generationConfig, key).not.toHaveProperty(key); + } +}; + describe("rerankWithGemini - request shape (Gemini 3.x)", () => { // SPEC-v9 s1.3: Gemini 3.x replaced thinkingBudget with thinkingLevel (`minimal` unsupported). // thinkingBudget:0 is how this reranker used to keep a thinking model from spending its @@ -208,6 +216,7 @@ describe("rerankWithGemini - request shape (Gemini 3.x)", () => { const thinking = bodies[0].generationConfig.thinkingConfig; expect(thinking).toEqual({ thinkingLevel: "low" }); expect(thinking).not.toHaveProperty("thinkingBudget"); + expectNoSampling(bodies[0]); }); it("condenseQuery sends the same thinking shape to the same model (thinkingBudget is removed on 3.6+)", async () => { @@ -227,5 +236,6 @@ describe("rerankWithGemini - request shape (Gemini 3.x)", () => { expect(thinking).toEqual({ thinkingLevel: "low" }); expect(thinking).not.toHaveProperty("thinkingBudget"); expect(JSON.stringify(bodies[0])).not.toContain("thinkingBudget"); + expectNoSampling(bodies[0]); }); }); diff --git a/shared/search/triSearch.ts b/shared/search/triSearch.ts index fd213771b..31e921e77 100644 --- a/shared/search/triSearch.ts +++ b/shared/search/triSearch.ts @@ -141,7 +141,7 @@ export async function rerankWithGemini( contents: [{ parts: [{ text: prompt }] }], // thinkingLevel "low" keeps thinking models (3.x) from spending the budget on a // reasoning preamble so they emit the bare index array; 256 tokens leaves room. - generationConfig: { temperature: 0, maxOutputTokens: 256, thinkingConfig: { thinkingLevel: "low" } }, + generationConfig: { maxOutputTokens: 256, thinkingConfig: { thinkingLevel: "low" } }, // no sampling: deprecated on Gemini 3.x (SPEC-v9 s1.3) }), signal: AbortSignal.timeout(opts.timeoutMs ?? RERANK_TIMEOUT_MS), }, @@ -267,7 +267,7 @@ export async function condenseQuery( // Same Gemini 3.x model as the reranker: thinkingBudget is removed there (SPEC-v9 s1.3), so // thinkingLevel "low" replaces it. Thinking counts toward the cap, so 256 (the rerank sizing) // leaves room for a 400-char rewrite; a truncated reply would silently fall back to the raw question. - generationConfig: { temperature: 0, maxOutputTokens: 256, thinkingConfig: { thinkingLevel: "low" } }, + generationConfig: { maxOutputTokens: 256, thinkingConfig: { thinkingLevel: "low" } }, // no sampling: deprecated on Gemini 3.x (SPEC-v9 s1.3) }), signal: AbortSignal.timeout(opts.timeoutMs ?? RERANK_TIMEOUT_MS), }, From d70d188aab2162f05aa60a2cea4924c064c6cb4f Mon Sep 17 00:00:00 2001 From: homen Date: Thu, 15 Oct 2026 16:50:09 -0700 Subject: [PATCH 5/8] fix(models): close the round 4 review findings (PR 1) A.0.4 (PR 1 moves no ids): - dailyBriefWorker.runNextTask / dailyBriefPersonalOverlay.runNextPersonalTask validate model with normalizeModelInput and forward the caller's string unchanged; round 3 forwarded the resolver's lossy alias (claude-sonnet-4-5-20250929 -> retired claude-sonnet-4, gpt-4o -> gpt-5.4). - modelCatalog: LEGACY_PASSTHROUGH_IDS (16 resolver-recognised ids main sent unchanged: 3 dated Claude snapshots, 10 gpt-*, 3 gemini-*) count as known and pass through; anthropicWireId returns the Claude snapshots, so extendedThinkingRunner accepts claude-haiku-4-5-20251001. Dotted claude-*-4.x and gpt-oss* stay rejected. - extendedThinkingRunner: adaptive thinking + output_config.effort only for claude-sonnet-4-6 / claude-opus-4-7; every other id keeps manual budget_tokens with no effort (Haiku 4.5 rejects effort); under 1,024 tokens no thinking block. Cost class (A.0.10): - modelRouter: free is the resolved model's cost class (isFreeAlias), not a row in a partial price table; 15 of 19 approved free aliases were billed $1/$5, recorded premium, and used up checkAgentBudget. - requestExtendedRun resolves the model (resolveCheckpointModel) before inserting a run. Tests: dailyBriefModelBoundary (16 ids byte-for-byte), anthropicWireIds (pass-through + drift guard), extendedThinking advanceRun for Haiku 4.5 and the request boundary, modelRouter x 19 free aliases through the real checkAgentBudget (15 fail on the previous source), and every request-shape suite now asserts the outgoing conversation ends on a user turn (a terminal assistant/model turn added to each source made each suite fail). CI runtime-smoke list: 47 files, 660 tests pass. Co-Authored-By: Claude Sonnet 5.5 --- .../models/modelRouter.requestShape.test.ts | 78 ++++++++++++++++++- .../convex/domains/ai/models/modelRouter.ts | 26 ++++--- .../extendedThinking.requestShape.test.ts | 65 +++++++++++++++- .../domains/product/extendedThinking.ts | 3 + .../research/dailyBriefModelBoundary.test.ts | 32 +++++++- .../research/dailyBriefPersonalOverlay.ts | 8 +- .../domains/research/dailyBriefWorker.ts | 8 +- .../fusion/benchmark.requestShape.test.ts | 5 ++ .../__tests__/anthropicRequestShape.test.ts | 2 + ...eScratchnodeChatVideo.requestShape.test.ts | 2 + shared/llm/anthropicWireIds.test.ts | 47 ++++++++++- shared/llm/modelCatalog.ts | 48 ++++++++++-- .../pipeline/extendedThinkingRunner.test.ts | 18 +++++ .../node/pipeline/extendedThinkingRunner.ts | 52 +++++++++---- 14 files changed, 352 insertions(+), 42 deletions(-) diff --git a/backend/convex/domains/ai/models/modelRouter.requestShape.test.ts b/backend/convex/domains/ai/models/modelRouter.requestShape.test.ts index a1b31f95e..8ed05803d 100644 --- a/backend/convex/domains/ai/models/modelRouter.requestShape.test.ts +++ b/backend/convex/domains/ai/models/modelRouter.requestShape.test.ts @@ -19,6 +19,8 @@ vi.mock("ai", async (importOriginal) => ({ ...(await importOriginal ({ resolveChain })); import { route } from "./modelRouter"; +import { checkAgentBudget } from "./modelRouterQueries"; +import { getFreeModels } from "../../agents/mcp_tools/models"; const baseArgs = { taskCategory: "research", @@ -29,7 +31,7 @@ const makeCtx = () => ({ runQuery: vi.fn(async () => ({ allowed: true })), runMutation: vi.fn(async () => null), }); -const call = (ctx: ReturnType, args: Record) => +const call = (ctx: object, args: Record) => (route as any)._handler(ctx, { ...baseArgs, ...args }); describe("modelRouter.route request shape and cost class", () => { @@ -59,6 +61,8 @@ describe("modelRouter.route request shape and cost class", () => { sent[pinnedModel] = "temperature" in request; dispatched[pinnedModel] = (request.model as { modelId: string }).modelId; for (const key of ["top_p", "top_k", "topP", "topK"]) expect(request, `${pinnedModel} ${key}`).not.toHaveProperty(key); + // No assistant prefill (400 on Claude 4.7+/5.x, Gemini 3.x): the outgoing conversation ends on a user turn. + expect(request.messages.at(-1).role, `${pinnedModel} last turn`).toBe("user"); } // The outgoing model string, so a wrong-model dispatch cannot pass on the temperature check alone. expect(dispatched).toEqual({ @@ -85,6 +89,7 @@ describe("modelRouter.route request shape and cost class", () => { await expect(call(makeCtx(), { tier: "free" })).rejects.toThrow(/All 3 models failed/); expect(generateText).toHaveBeenCalledTimes(3); + for (const c of generateText.mock.calls) expect(c[0].messages.at(-1).role).toBe("user"); const tried = generateText.mock.calls.map((c) => (c[0].model as { modelId: string }).modelId); expect(tried.every((id) => id.endsWith(":free")), tried.join(", ")).toBe(true); }); @@ -126,6 +131,7 @@ describe("modelRouter.route request shape and cost class", () => { const ctx = makeCtx(); const result = await call(ctx, { tier: "free", pinnedModel, cacheKey: `pin-${pinnedModel}` }); expect((generateText.mock.calls[0][0].model as { modelId: string }).modelId).toBe("nvidia/nemotron-3-super-120b-a12b:free"); + expect(generateText.mock.calls[0][0].messages.at(-1).role).toBe("user"); expect(result).toMatchObject({ modelId: "nemotron-3-super-free", isFree: true, costUsd: 0 }); expect(result.routing.actualTier).toBe("free"); const logged = (ctx.runMutation.mock.calls[0] as unknown as [unknown, Record])[1]; @@ -144,3 +150,73 @@ describe("modelRouter.route request shape and cost class", () => { await expect(call(makeCtx(), { tier: "cheap" })).rejects.toThrow(/All 4 models failed/); }); }); + +// Persona: a background agent with a $0.01 daily dollar budget that makes many free-lane calls. +// route() writes one audit row per call and checkAgentBudget sums the rows' costUsd, so a free call +// recorded as paid used the budget up: 7 laguna calls exhausted it and call 8 was refused (round 4). +// The ctx below runs the real checkAgentBudget handler over the rows route() actually logged. +describe("modelRouter.route free aliases against the agent's dollar budget", () => { + const BUDGET_USD = 0.01; + const CALLS = 20; + + function budgetCtx() { + const rows: Array> = []; + const db = { + query: (table: string) => ({ + withIndex: () => + table === "agentIdentities" + ? { first: async () => ({ budgetDailyCostUsd: BUDGET_USD, budgetDailyTokens: 10_000_000 }) } + : { filter: () => ({ collect: async () => rows }) }, + }), + }; + return { + rows, + runQuery: vi.fn(async (_ref: unknown, args: { agentId: string }) => (checkAgentBudget as any)._handler({ db }, args)), + runMutation: vi.fn(async (_ref: unknown, args: Record) => { + rows.push({ ...args, calledAt: Date.now() }); + return null; + }), + }; + } + + beforeEach(() => { + for (const key of ["OPENROUTER_API_KEY", "ANTHROPIC_API_KEY", "OPENAI_API_KEY", "GEMINI_API_KEY"]) { + vi.stubEnv(key, "test-key-not-real-0000"); + } + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "warn").mockImplementation(() => {}); + generateText.mockReset(); + resolveChain.mockReset(); + generateText.mockResolvedValue({ text: "ok", usage: { inputTokens: 1_000, outputTokens: 100 } }); + }); + afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + }); + + it("covers all 19 approved free aliases", () => { + expect(getFreeModels().length).toBeGreaterThanOrEqual(19); + }); + + it.each(getFreeModels())("%s: free calls cost nothing, are logged free, and never use the dollar budget up", async (alias) => { + const ctx = budgetCtx(); + for (let i = 0; i < CALLS; i++) { + const result = await call(ctx, { pinnedModel: alias, agentId: "agent-1" }); + expect(result).toMatchObject({ modelId: alias, isFree: true, costUsd: 0 }); + expect(result.routing.actualTier).toBe("free"); + } + expect(ctx.rows).toHaveLength(CALLS); + expect(ctx.rows.every((r) => r.costUsd === 0 && r.actualTier === "free")).toBe(true); + const budget = await (ctx.runQuery as any)(null, { agentId: "agent-1" }); + expect(budget).toMatchObject({ allowed: true, dailyCostUsd: 0 }); + }); + + it("control: a paid model does use the dollar budget up, so the check above is not vacuous", async () => { + const ctx = budgetCtx(); + // gpt-5.4 at 1,000 in / 100 out is $0.004 a call: the 4th call finds $0.012 >= $0.01 spent. + await call(ctx, { pinnedModel: "gpt-5.4", agentId: "agent-1" }); + await call(ctx, { pinnedModel: "gpt-5.4", agentId: "agent-1" }); + await call(ctx, { pinnedModel: "gpt-5.4", agentId: "agent-1" }); + await expect(call(ctx, { pinnedModel: "gpt-5.4", agentId: "agent-1" })).rejects.toThrow(/budget exceeded/); + }); +}); diff --git a/backend/convex/domains/ai/models/modelRouter.ts b/backend/convex/domains/ai/models/modelRouter.ts index cbdf71d76..2908066e3 100644 --- a/backend/convex/domains/ai/models/modelRouter.ts +++ b/backend/convex/domains/ai/models/modelRouter.ts @@ -148,9 +148,8 @@ const MODEL_PRICING: Record = { "arcee-trinity-large-free": { input: 0, output: 0 }, "nemotron-3-nano-30b-free": { input: 0, output: 0 }, "llama-3.3-70b-free": { input: 0, output: 0 }, - // FREE_LANE_MODEL: without this row calculateCost bills it at the $1/$5 default and the audit - // row records a pinned free route as premium paid usage. Prices for the new paid ids land in PR 2. - "nemotron-3-super-free": { input: 0, output: 0 }, + // Free aliases are priced and classified by isFreeAlias (the resolver's cost class), not by a row + // here. Prices for the new paid ids land in PR 2. "glm-4.7-flash": { input: 0.07, output: 0.07 }, "gemini-3-flash-preview": { input: 0.50, output: 3.00 }, "claude-haiku-4.5": { input: 1.00, output: 5.00 }, @@ -182,11 +181,23 @@ const TASK_DEFAULT_TIER: Record = { // COST CALCULATION // ═══════════════════════════════════════════════════════════════════════════ +/** + * Free is a property of the resolved model, not of this file's partial price table: 15 of the 19 + * approved free aliases (laguna-s-2.1-free included) have no MODEL_PRICING row and used to be billed + * at the $1/$5 default, recorded isFree=false / actualTier "premium", and summed into the agent's + * dollar budget by checkAgentBudget. + */ +function isFreeAlias(modelId: string): boolean { + const alias = resolveModelAlias(modelId); + return alias !== null && isFreeModel(alias); +} + function calculateCost( modelId: string, inputTokens: number, outputTokens: number ): number { + if (isFreeAlias(modelId)) return 0; const pricing = MODEL_PRICING[modelId] ?? { input: 1.0, output: 5.0 }; return ( (inputTokens / 1_000_000) * pricing.input + @@ -306,7 +317,7 @@ export const route = internalAction({ return { text: cached.text, modelId: cached.modelId, - isFree: MODEL_PRICING[cached.modelId]?.input === 0, + isFree: isFreeAlias(cached.modelId), inputTokens: cached.inputTokens, outputTokens: cached.outputTokens, costUsd: 0, @@ -416,10 +427,7 @@ export const route = internalAction({ // floor, so a capability requirement (for example requireVision) can put a paid model in it. // If no free model qualifies the call fails below instead of billing. if (tier === "free") { - candidates = candidates.filter((id) => { - const alias = resolveModelAlias(id); - return alias !== null && isFreeModel(alias); - }); + candidates = candidates.filter(isFreeAlias); } } @@ -463,7 +471,7 @@ export const route = internalAction({ const outputTokens = result.usage?.outputTokens ?? 0; const costUsd = calculateCost(modelId, inputTokens, outputTokens); const latencyMs = Date.now() - startTime; - const isFree = MODEL_PRICING[modelId]?.input === 0; + const isFree = isFreeAlias(modelId); // Determine actual tier based on model used if (isFree) actualTier = "free"; diff --git a/backend/convex/domains/product/extendedThinking.requestShape.test.ts b/backend/convex/domains/product/extendedThinking.requestShape.test.ts index fc85b3bcc..84f9fef36 100644 --- a/backend/convex/domains/product/extendedThinking.requestShape.test.ts +++ b/backend/convex/domains/product/extendedThinking.requestShape.test.ts @@ -9,12 +9,16 @@ * Opus 4.7+ the thinking block's text is omitted by default, so a charge estimated * from that text is always 0 and the budget would never run out. The charge comes * from usage.output_tokens minus the answer. + * Round 4 adds the per-model thinking shape (Haiku 4.5 takes manual thinking and rejects effort) and the + * public boundary: requestExtendedRun rejects a model with no Anthropic wire id before it inserts a run. * The fetch transport is fake; no network, no spend. */ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { advanceRun } from "./extendedThinking"; +vi.mock("@convex-dev/auth/server", () => ({ getAuthUserId: vi.fn(async () => "user1") })); + +import { advanceRun, requestExtendedRun } from "./extendedThinking"; const answer = JSON.stringify({ headline: "Funding history found", @@ -111,4 +115,63 @@ describe("extendedThinking.advanceRun", () => { const { requests } = await advance(runRow({ thinkingBudgetTokens: 40_000, thinkingTokensUsed: 37_500 }), 6_500); expect(requests[0].max_tokens).toBe(6_500); }); + + // Persona: a founder whose run is stored on Haiku 4.5 (requestExtendedRun.modelName is a public argument). + // Claude Haiku 4.5 supports only manual thinking and errors on output_config.effort, so the request keeps + // the budget_tokens shape main sent. The 400 is documented in the migration guide, not observed live. + it.each(["claude-haiku-4-5", "claude-haiku-4.5", "claude-haiku-4-5-20251001"])( + "a run stored with %s sends manual thinking with a token budget and no effort", + async (modelName) => { + const { requests } = await advance(runRow({ modelName }), 5_000); + const body = requests[0]; + expect(body.model).toBe(modelName === "claude-haiku-4.5" ? "claude-haiku-4-5" : modelName); + expect(body.thinking).toEqual({ type: "enabled", budget_tokens: 10_000 }); + expect(body).not.toHaveProperty("output_config"); + for (const key of ["temperature", "top_p", "top_k"]) expect(body, key).not.toHaveProperty(key); + expect(body.max_tokens).toBe(14_000); + expect(body.max_tokens).toBeGreaterThan(body.thinking.budget_tokens); + }, + ); + + it("a Haiku 4.5 run with less than the 1,024-token API minimum left sends no thinking block at all", async () => { + const { requests } = await advance(runRow({ modelName: "claude-haiku-4-5", thinkingBudgetTokens: 40_000, thinkingTokensUsed: 39_500 }), 600); + expect(requests[0]).not.toHaveProperty("thinking"); + expect(requests[0]).not.toHaveProperty("output_config"); + expect(requests[0].max_tokens).toBe(4_500); + }); + + it("the adaptive models keep adaptive thinking with effort", async () => { + for (const modelName of ["claude-sonnet-4.6", "claude-opus-4.7", "claude-opus-4-7"]) { + const { requests } = await advance(runRow({ modelName }), 5_000); + expect(requests[0].thinking, modelName).toEqual({ type: "adaptive" }); + expect(requests[0].output_config, modelName).toEqual({ effort: "medium" }); + } + }); +}); + +describe("extendedThinking.requestExtendedRun model boundary", () => { + const makeCtx = () => ({ + db: { insert: vi.fn(async () => "run1") }, + scheduler: { runAfter: vi.fn(async () => null) }, + }); + const request = (ctx: ReturnType, modelName?: string) => + (requestExtendedRun as any)._handler(ctx, { entitySlug: "Anthropic", goal: "Build a brief.", modelName }); + + // Persona: a signed-in user (or a script) that types a model name the repo does not know. Before, the run + // was inserted and advanceRun scheduled; the rejection came later, in the checkpoint builder. + it.each(["claude-not-real", "invented-free", "kimi-k2.6", "gpt-5.4"])("rejects %s before a run is inserted or a job is scheduled", async (modelName) => { + const ctx = makeCtx(); + await expect(request(ctx, modelName)).rejects.toThrow(/no Anthropic wire id/); + expect(ctx.db.insert).not.toHaveBeenCalled(); + expect(ctx.scheduler.runAfter).not.toHaveBeenCalled(); + }); + + it.each([undefined, "claude-sonnet-4.6", "claude-haiku-4-5", "claude-haiku-4-5-20251001"])("accepts %s and stores the name as given", async (modelName) => { + const ctx = makeCtx(); + await request(ctx, modelName); + expect(ctx.db.insert).toHaveBeenCalledTimes(1); + const inserted = (ctx.db.insert.mock.calls[0] as unknown as [string, { modelName: string }])[1]; + expect(inserted.modelName).toBe(modelName ?? "claude-sonnet-4.6"); + expect(ctx.scheduler.runAfter).toHaveBeenCalledTimes(1); + }); }); diff --git a/backend/convex/domains/product/extendedThinking.ts b/backend/convex/domains/product/extendedThinking.ts index 811340b10..04cc85d69 100644 --- a/backend/convex/domains/product/extendedThinking.ts +++ b/backend/convex/domains/product/extendedThinking.ts @@ -47,6 +47,7 @@ import { buildCheckpointRequestBody, estimateThinkingTokens, parseCheckpointResponse, + resolveCheckpointModel, shouldContinue, MAX_CHECKPOINTS, MAX_THINKING_BUDGET_TOKENS, @@ -96,6 +97,8 @@ export const requestExtendedRun = mutation({ ), ); const modelName = args.modelName ?? DEFAULT_MODEL; + // A.0.10: an id with no Anthropic wire id is rejected here, before a run is inserted or a job scheduled. + resolveCheckpointModel(modelName); const now = Date.now(); const runId = await ctx.db.insert("extendedThinkingRuns", { diff --git a/backend/convex/domains/research/dailyBriefModelBoundary.test.ts b/backend/convex/domains/research/dailyBriefModelBoundary.test.ts index 84caf4942..19b9c1155 100644 --- a/backend/convex/domains/research/dailyBriefModelBoundary.test.ts +++ b/backend/convex/domains/research/dailyBriefModelBoundary.test.ts @@ -4,7 +4,9 @@ * hand it unchanged to executeFeature -> generateWithProvider -> getModelWithFailover -> the provider * SDK (SPEC-v9 A.0.10; PR #662 review round 2). Persona: a signed-in user who types a model name the * repo does not know ("gpt-5.5-pro"). The action must reject it before any internal action runs, so - * no provider call and no spend. Known ids and free-looking ids still go through, normalised. + * no provider call and no spend. A known or free-looking id is validated and then forwarded exactly as + * the caller wrote it: normalizeModelInput returns the resolver's lossy alias, and "claude-sonnet-4-5-20250929" + * (live until 2026-11-30) must not become "claude-sonnet-4" (retired) in PR 1 (A.0.4; round 4). * Fake ctx; no network. */ @@ -15,6 +17,7 @@ vi.mock("@convex-dev/auth/server", () => ({ getAuthUserId: vi.fn(async () => "us import { runNextPersonalTask } from "./dailyBriefPersonalOverlay"; import { runNextTask } from "./dailyBriefWorker"; import { UnknownModelError } from "../agents/mcp_tools/models"; +import { FREE_LANE_WIRE_ID, LEGACY_PASSTHROUGH_IDS, getModelWithFailover, resolveModelAlias } from "../../../../shared/llm/modelCatalog"; const actions = [ ["dailyBriefWorker.runNextTask", runNextTask], @@ -35,14 +38,17 @@ describe.each(actions)("%s model boundary", (_name, action) => { expect(ctx.runAction).not.toHaveBeenCalled(); }); - it("resolves a free-looking unknown id to the free lane, never a paid model", async () => { + it("forwards a free-looking unknown id as written; the catalog then sends it to the free lane, never a paid model", async () => { vi.spyOn(console, "warn").mockImplementation(() => {}); + vi.stubEnv("OPENROUTER_API_KEY", "sk-or-test-key-not-real"); const { ctx, result } = run({ model: "retired-model-free", taskId: "t1" }); await result; - expect(ctx.runAction).toHaveBeenCalledWith(expect.anything(), { model: "nemotron-3-super-free", taskId: "t1" }); + expect(ctx.runAction).toHaveBeenCalledWith(expect.anything(), { model: "retired-model-free", taskId: "t1" }); + expect(getModelWithFailover(resolveModelAlias("retired-model-free"))).toEqual({ model: FREE_LANE_WIRE_ID, provider: "openrouter", isFallback: true }); + vi.unstubAllEnvs(); }); - it("forwards a known id as its approved alias and leaves an omitted model omitted", async () => { + it("forwards a known id exactly as given and leaves an omitted model omitted", async () => { const known = run({ model: "gpt-5.4-mini" }); await known.result; expect(known.ctx.runAction).toHaveBeenCalledWith(expect.anything(), { model: "gpt-5.4-mini" }); @@ -51,4 +57,22 @@ describe.each(actions)("%s model boundary", (_name, action) => { await omitted.result; expect(omitted.ctx.runAction).toHaveBeenCalledWith(expect.anything(), { taskId: "t1" }); }); + + // Persona: a signed-in user pins a model that is live today. Before the fix the action replaced it with + // the resolver's alias (claude-sonnet-4-5-20250929 -> claude-sonnet-4, claude-opus-4-5-20251101 -> + // claude-opus-4, gpt-4o -> gpt-5.4, gpt-5-mini / gpt-4o-mini -> gpt-5.4-mini): all 16 ids main sent unchanged. + it("forwards every legacy id main passed through byte-for-byte, and the catalog sends that same string to the provider", async () => { + vi.stubEnv("ANTHROPIC_API_KEY", "sk-ant-test-key-not-real"); + vi.stubEnv("OPENAI_API_KEY", "sk-test-key-not-real-0000"); + vi.stubEnv("GEMINI_API_KEY", "gemini-test-key-not-real"); + expect(LEGACY_PASSTHROUGH_IDS.size).toBe(16); + for (const model of LEGACY_PASSTHROUGH_IDS) { + const { ctx, result } = run({ model, taskId: "t1" }); + await result; + const forwarded = (ctx.runAction.mock.calls[0] as unknown as [unknown, { model: string }])[1].model; + expect(forwarded, model).toBe(model); + expect(getModelWithFailover(resolveModelAlias(forwarded)).model, model).toBe(model); + } + vi.unstubAllEnvs(); + }); }); diff --git a/backend/convex/domains/research/dailyBriefPersonalOverlay.ts b/backend/convex/domains/research/dailyBriefPersonalOverlay.ts index 8994f33d0..fec6df4f2 100644 --- a/backend/convex/domains/research/dailyBriefPersonalOverlay.ts +++ b/backend/convex/domains/research/dailyBriefPersonalOverlay.ts @@ -374,11 +374,13 @@ export const runNextPersonalTask = action({ handler: async (ctx, args): Promise => { const userId = await getAuthUserId(ctx); if (!userId) throw new Error("Not authenticated"); - // A.0.10: an unknown model id throws here, before any provider is reached. - const model = args.model === undefined ? undefined : normalizeModelInput(args.model); + // A.0.10: an unknown model id throws here, before any provider is reached. Validation only: + // the caller's string is forwarded as given, because normalizeModelInput returns the resolver's + // lossy alias (claude-sonnet-4-5-20250929 -> claude-sonnet-4, retired), which PR 1 must not send. + if (args.model !== undefined) normalizeModelInput(args.model); return await ctx.runAction( internal.domains.research.dailyBriefPersonalOverlay.runNextPersonalTaskInternal, - model === undefined ? args : { ...args, model }, + args, ); }, }); diff --git a/backend/convex/domains/research/dailyBriefWorker.ts b/backend/convex/domains/research/dailyBriefWorker.ts index 075bde86f..cc67cf2db 100644 --- a/backend/convex/domains/research/dailyBriefWorker.ts +++ b/backend/convex/domains/research/dailyBriefWorker.ts @@ -599,12 +599,14 @@ export const runNextTask = action({ handler: async (ctx, args): Promise => { const userId = await getAuthUserId(ctx); if (!userId) throw new Error("Not authenticated"); - // A.0.10: an unknown model id throws here, before any provider is reached. - const model = args.model === undefined ? undefined : normalizeModelInput(args.model); + // A.0.10: an unknown model id throws here, before any provider is reached. Validation only: + // the caller's string is forwarded as given, because normalizeModelInput returns the resolver's + // lossy alias (claude-sonnet-4-5-20250929 -> claude-sonnet-4, retired), which PR 1 must not send. + if (args.model !== undefined) normalizeModelInput(args.model); const res: any = await ctx.runAction( internal.domains.research.dailyBriefWorker.runNextTaskInternal, - model === undefined ? args : { ...args, model }, + args, ); return res; }, diff --git a/backend/convex/domains/search/fusion/benchmark.requestShape.test.ts b/backend/convex/domains/search/fusion/benchmark.requestShape.test.ts index fa2adfa01..be19e55a9 100644 --- a/backend/convex/domains/search/fusion/benchmark.requestShape.test.ts +++ b/backend/convex/domains/search/fusion/benchmark.requestShape.test.ts @@ -60,6 +60,8 @@ describe("benchmark.evaluateSearch judge request shape", () => { expect(captured[0].body.model).toBe("claude-sonnet-4-6"); for (const key of ["temperature", "top_p", "top_k"]) expect(captured[0].body, key).not.toHaveProperty(key); expect(captured[0].body.max_tokens).toBeGreaterThanOrEqual(1024); + // Claude 4.7+/5.x answer 400 to an assistant prefill: the conversation must end on a user turn (round 4). + expect(captured[0].body.messages.at(-1).role).toBe("user"); expect(result.judgeModel).toBe("claude-sonnet-4.6"); expect(result.judgeResult.reasoning).toBe("r"); }); @@ -68,6 +70,7 @@ describe("benchmark.evaluateSearch judge request shape", () => { const { captured } = await run("gpt-5.4"); expect(captured[0].url).toBe("https://api.openai.com/v1/chat/completions"); expect(captured[0].body.temperature).toBe(0); + expect(captured[0].body.messages.at(-1).role).toBe("user"); }); it("Gemini judge sends no sampling keys (deprecated on Gemini 3.x, SPEC-v9 s1.3), with the model in the URL", async () => { @@ -78,5 +81,7 @@ describe("benchmark.evaluateSearch judge request shape", () => { expect(captured[0].url).toContain("/models/gemini-3.1-pro-preview:generateContent"); for (const key of ["temperature", "topP", "topK", "top_p", "top_k"]) expect(captured[0].body.generationConfig, key).not.toHaveProperty(key); expect(captured[0].body.generationConfig.maxOutputTokens).toBe(1500); + // A model-turn prefill is a 400 on Gemini 3.x; a content with no role is a user turn. + expect(captured[0].body.contents.at(-1).role ?? "user").toBe("user"); }); }); diff --git a/packages/mcp-local/src/__tests__/anthropicRequestShape.test.ts b/packages/mcp-local/src/__tests__/anthropicRequestShape.test.ts index 1ec60834b..2fb3b68ec 100644 --- a/packages/mcp-local/src/__tests__/anthropicRequestShape.test.ts +++ b/packages/mcp-local/src/__tests__/anthropicRequestShape.test.ts @@ -53,6 +53,7 @@ describe("mcp-local Anthropic provider request shape", () => { for (const key of ["temperature", "top_p", "top_k"]) expect(body, key).not.toHaveProperty(key); expect(body.max_tokens).toBe(777); expect(body.messages).toEqual([{ role: "user", content: "hi" }]); + expect(body.messages.at(-1).role).toBe("user"); // no assistant prefill (400 on Claude 4.7+/5.x) expect(result.response).toBe("the answer"); expect(result.provider).toBe("anthropic"); }); @@ -64,6 +65,7 @@ describe("mcp-local Anthropic provider request shape", () => { expect(body.model).toBe("claude-haiku-4-5-20251001"); // PR 1 moves no ids for (const key of ["temperature", "top_p", "top_k"]) expect(body, key).not.toHaveProperty(key); expect(body.max_tokens).toBe(4096); + expect(body.messages.at(-1).role).toBe("user"); // no assistant prefill (400 on Claude 4.7+/5.x) expect(result.polishedText).toBe("the answer"); }); }); diff --git a/scripts/__tests__/judgeScratchnodeChatVideo.requestShape.test.ts b/scripts/__tests__/judgeScratchnodeChatVideo.requestShape.test.ts index af1c79c98..baa5fe3c9 100644 --- a/scripts/__tests__/judgeScratchnodeChatVideo.requestShape.test.ts +++ b/scripts/__tests__/judgeScratchnodeChatVideo.requestShape.test.ts @@ -51,5 +51,7 @@ describe("judgeScratchnodeChatVideo.mjs Gemini request", () => { } expect(request.config.thinkingConfig).toEqual({ thinkingLevel: "low" }); expect(request.config.maxOutputTokens).toBe(3500); + // A model-turn prefill is a 400 on Gemini 3.x: the conversation ends on a user turn. + expect(request.contents.at(-1).role).toBe("user"); }); }); diff --git a/shared/llm/anthropicWireIds.test.ts b/shared/llm/anthropicWireIds.test.ts index 32bcab67a..901a87f86 100644 --- a/shared/llm/anthropicWireIds.test.ts +++ b/shared/llm/anthropicWireIds.test.ts @@ -13,6 +13,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { ANTHROPIC_WIRE_IDS, + LEGACY_PASSTHROUGH_IDS, anthropicText, anthropicWireId, FREE_LANE_CATALOG_ID, @@ -21,7 +22,7 @@ import { llmModelCatalog, resolveModelAlias, } from "./modelCatalog"; -import { FREE_LANE_MODEL, MODEL_SPECS } from "../../backend/convex/domains/agents/mcp_tools/models/modelResolver"; +import { FREE_LANE_MODEL, LEGACY_ALIASES, MODEL_SPECS } from "../../backend/convex/domains/agents/mcp_tools/models/modelResolver"; describe("ANTHROPIC_WIRE_IDS agrees with the resolver's MODEL_SPECS", () => { const anthropicSpecs = Object.values(MODEL_SPECS).filter((spec) => spec.provider === "anthropic"); @@ -50,7 +51,11 @@ describe("anthropicWireId", () => { it("returns a string that already is a wire id unchanged, and null for anything else", () => { expect(anthropicWireId("claude-sonnet-4-6")).toBe("claude-sonnet-4-6"); - expect(anthropicWireId("claude-haiku-4-5-20251001")).toBeNull(); + // Round 3 pinned null here; main sent the dated snapshots unchanged and the resolver still + // recognises them, so they pass through (round 4, SPEC-v9 A.0.4: PR 1 moves no ids). + for (const snapshot of ["claude-haiku-4-5-20251001", "claude-sonnet-4-5-20250929", "claude-opus-4-5-20251101"]) { + expect(anthropicWireId(snapshot)).toBe(snapshot); + } expect(anthropicWireId("gpt-5.4")).toBeNull(); expect(anthropicWireId("")).toBeNull(); }); @@ -99,7 +104,43 @@ describe("getModelWithFailover sends wire ids to Anthropic", () => { for (const wire of Object.values(ANTHROPIC_WIRE_IDS)) { expect(getModelWithFailover(wire), wire).toMatchObject({ model: wire, provider: "anthropic", isFallback: false }); } - expect(() => getModelWithFailover("claude-haiku-4-5-20251001")).toThrow(/Unknown model/); + // An unrecognised snapshot-shaped id is still unknown: the prefix alone is not knowledge. + expect(() => getModelWithFailover("claude-haiku-4-5-29990101")).toThrow(/Unknown model/); + }); + + // Round 4 (A.0.4): getModelWithFailover threw for ids the resolver recognises and main sent unchanged. + describe("legacy ids the resolver recognises", () => { + // Not wire ids, so main's request for them failed at the provider: dotted Claude ids, and the + // gpt-oss* OpenRouter free aliases that the `gpt-` prefix used to route to paid OpenAI. + const NOT_WIRE_IDS = ["claude-opus-4.6", "claude-opus-4.5", "claude-sonnet-4.5", "gpt-oss-120b", "gpt-oss", "gpt-oss-20b"]; + const providerOf = (id: string) => (id.startsWith("claude-") ? "anthropic" : id.startsWith("gpt-") ? "openai" : "gemini"); + beforeEach(() => { + vi.stubEnv("ANTHROPIC_API_KEY", "sk-ant-test-key-not-real"); + vi.stubEnv("OPENAI_API_KEY", "sk-test-key-not-real-0000"); + vi.stubEnv("GEMINI_API_KEY", "gemini-test-key-not-real"); + }); + + it("every LEGACY_PASSTHROUGH_IDS entry is a resolver alias and goes out unchanged", () => { + expect(LEGACY_PASSTHROUGH_IDS.size).toBeGreaterThan(0); + for (const id of LEGACY_PASSTHROUGH_IDS) { + expect(LEGACY_ALIASES, `${id} must stay a resolver alias`).toHaveProperty(id); + expect(getModelWithFailover(resolveModelAlias(id)), id).toEqual({ model: id, provider: providerOf(id), isFallback: false }); + } + }); + + it("no provider-prefixed LEGACY_ALIASES key is rejected by the catalog (drift guard)", () => { + const keys = Object.keys(LEGACY_ALIASES).filter((id) => /^(claude-|gpt-|gemini-)/.test(id) && !NOT_WIRE_IDS.includes(id)); + expect(keys.length).toBeGreaterThan(20); + for (const id of keys) { + expect(() => getModelWithFailover(resolveModelAlias(id)), `${id}: add it to LEGACY_PASSTHROUGH_IDS`).not.toThrow(); + } + }); + + it("the ids main could not have served still fail closed", () => { + for (const id of NOT_WIRE_IDS) { + expect(() => getModelWithFailover(resolveModelAlias(id)), id).toThrow(/Unknown model/); + } + }); }); it("leaves non-Anthropic providers alone", () => { diff --git a/shared/llm/modelCatalog.ts b/shared/llm/modelCatalog.ts index 49102be48..d1a8369c6 100644 --- a/shared/llm/modelCatalog.ts +++ b/shared/llm/modelCatalog.ts @@ -586,16 +586,48 @@ export const ANTHROPIC_WIRE_IDS: Readonly> = { "claude-haiku-3.5": "claude-3-5-haiku-20241022", }; +/** + * Ids `modelResolver.LEGACY_ALIASES` recognises and that `main` sent to the provider unchanged, + * so they keep going out unchanged: PR 1 moves no ids (SPEC-v9 A.0.4). Each already is its own + * wire string; the resolver's lossy alias for it (for example `claude-sonnet-4-5-20250929` -> + * `claude-sonnet-4`, retired) is never what a caller asked for. The ids move in PR 2. + * Left out on purpose: the dotted `claude-opus-4.6`, `claude-opus-4.5`, `claude-sonnet-4.5` + * (never wire ids, api.anthropic.com 404s on them) and `gpt-oss*` (OpenRouter free aliases that + * the `gpt-` prefix used to route to paid OpenAI). `anthropicWireIds.test.ts` pins this list + * against `LEGACY_ALIASES`, so a new resolver alias that main would have passed through + * fails the test instead of being rejected at runtime. + */ +export const LEGACY_PASSTHROUGH_IDS: ReadonlySet = new Set([ + "claude-haiku-4-5-20251001", + "claude-sonnet-4-5-20250929", + "claude-opus-4-5-20251101", + "gpt-5.2", + "gpt-5.1", + "gpt-5.1-codex", + "gpt-5-mini", + "gpt-5-nano", + "gpt-5-chat-latest", + "gpt-4.1-mini", + "gpt-4.1-nano", + "gpt-4o", + "gpt-4o-mini", + "gemini-3-pro", + "gemini-3-flash", + "gemini-3.1-flash-lite-preview-lite", +]); + /** * The Anthropic wire id for a catalog alias (or short alias such as "claude"). - * A string that already is a wire id is returned unchanged. Returns null when - * the input is neither, so callers decide whether to fail or pass it through. + * A string that already is a wire id is returned unchanged, dated snapshots in + * `LEGACY_PASSTHROUGH_IDS` included. Returns null when the input is none of these, + * so callers decide whether to fail or pass it through. */ export function anthropicWireId(modelInput: string): string | null { const resolved = resolveModelAlias(modelInput.trim()); const fromAlias = ANTHROPIC_WIRE_IDS[resolved]; if (fromAlias) return fromAlias; - return Object.values(ANTHROPIC_WIRE_IDS).includes(resolved) ? resolved : null; + if (Object.values(ANTHROPIC_WIRE_IDS).includes(resolved)) return resolved; + return resolved.startsWith("claude-") && LEGACY_PASSTHROUGH_IDS.has(resolved) ? resolved : null; } /** @@ -896,11 +928,17 @@ function isFreeLookingModel(model: string): boolean { /** * A model this catalog knows by name or by wire id (`ANTHROPIC_WIRE_IDS` values; for openai and - * gemini the resolver's sdkId equals the catalog name, pinned by anthropicWireIds.test.ts). A provider + * gemini the resolver's sdkId equals the catalog name, pinned by anthropicWireIds.test.ts), or an id + * main passed through that the resolver still recognises (`LEGACY_PASSTHROUGH_IDS`). A provider * prefix (`gpt-`, `claude-`, `gemini-`) is not knowledge: an id that is only prefix-shaped is unknown. */ function isKnownCatalogModel(model: string): boolean { - return modelPricing[model] != null || modelEquivalents[model] != null || anthropicWireId(model) !== null; + return ( + modelPricing[model] != null || + modelEquivalents[model] != null || + LEGACY_PASSTHROUGH_IDS.has(model) || + anthropicWireId(model) !== null + ); } function isFreeCatalogModel(model: string): boolean { diff --git a/workers/node/pipeline/extendedThinkingRunner.test.ts b/workers/node/pipeline/extendedThinkingRunner.test.ts index 32359c472..d57254b7e 100644 --- a/workers/node/pipeline/extendedThinkingRunner.test.ts +++ b/workers/node/pipeline/extendedThinkingRunner.test.ts @@ -15,6 +15,7 @@ import { describe, it, expect } from "vitest"; import { buildCheckpointPrompt, buildCheckpointRequestBody, + resolveCheckpointModel, estimateThinkingTokens, parseCheckpointResponse, shouldContinue, @@ -323,6 +324,23 @@ describe("buildCheckpointRequestBody - Anthropic request shape (SPEC-v9 A.0.6)", expect(charged).toBeGreaterThanOrEqual(actuallySpent); }); + it("Haiku 4.5 keeps manual thinking with budget_tokens and gets no effort; an allowance under 1,024 sends no thinking", () => { + const body = (modelName: string, remaining: number) => buildCheckpointRequestBody({ modelName, prompt, remainingThinkingTokens: remaining }) as any; + for (const modelName of ["claude-haiku-4-5", "claude-haiku-4-5-20251001", "claude-sonnet-4-5-20250929", "claude-opus-4-5-20251101"]) { + expect(body(modelName, 40_000).thinking, modelName).toEqual({ type: "enabled", budget_tokens: MAX_THINKING_BUDGET_TOKENS }); + expect(body(modelName, 40_000), modelName).not.toHaveProperty("output_config"); + expect(body(modelName, 3_000).thinking, modelName).toEqual({ type: "enabled", budget_tokens: 3_000 }); + expect(body(modelName, 1_023), modelName).not.toHaveProperty("thinking"); + expect(body(modelName, 0), modelName).not.toHaveProperty("thinking"); + } + }); + + it("resolveCheckpointModel returns the wire id and throws for a name with no Anthropic id", () => { + expect(resolveCheckpointModel("claude-sonnet-4.6")).toBe("claude-sonnet-4-6"); + expect(resolveCheckpointModel("claude-haiku-4-5-20251001")).toBe("claude-haiku-4-5-20251001"); + expect(() => resolveCheckpointModel("claude-not-real")).toThrow(/no Anthropic wire id/); + }); + it("accepts an operator-supplied wire id and refuses a name with no Anthropic id", () => { expect(buildCheckpointRequestBody({ modelName: "claude-sonnet-4-6", prompt, remainingThinkingTokens: 1 }).model).toBe( "claude-sonnet-4-6", diff --git a/workers/node/pipeline/extendedThinkingRunner.ts b/workers/node/pipeline/extendedThinkingRunner.ts index 8b637bc75..71596091d 100644 --- a/workers/node/pipeline/extendedThinkingRunner.ts +++ b/workers/node/pipeline/extendedThinkingRunner.ts @@ -95,15 +95,37 @@ export type ParseOutcome = * Request construction * ------------------------------------------------------------------------ */ +/** + * Ids that take adaptive thinking with `output_config.effort`. Every other id keeps the manual + * `{type:"enabled", budget_tokens}` shape main sent: Haiku 4.5 supports only manual thinking and + * 400s on `effort` (migration guide, "Choosing an effort level"), and adaptive thinking starts at + * the 4.6 generation. Haiku 4.5 and the dated 4.5 snapshots move with their ids in PR 2. + */ +const ADAPTIVE_THINKING_WIRE_IDS: ReadonlySet = new Set(["claude-opus-4-7", "claude-sonnet-4-6"]); + +/** Smallest `budget_tokens` the Messages API accepts. */ +const MIN_MANUAL_BUDGET_TOKENS = 1_024; + +/** + * The Anthropic wire id a run's stored `modelName` is sent as. The stored name may be a dotted + * catalog alias ("claude-sonnet-4.6"), which api.anthropic.com rejects. A name with no Anthropic + * wire id throws; `requestExtendedRun` calls this before it inserts a run, so an unknown id never + * becomes a queued run or a scheduled job. + */ +export function resolveCheckpointModel(modelName: string): string { + const model = anthropicWireId(modelName); + if (!model) { + throw new Error(`no Anthropic wire id for model "${modelName}"`); + } + return model; +} + /** * The Messages API body for ONE checkpoint. - * - `model` is the Anthropic wire id from the explicit table in - * shared/llm/modelCatalog.ts; the run's stored `modelName` may be a dotted - * catalog alias ("claude-sonnet-4.6"), which api.anthropic.com rejects. - * A name with no Anthropic wire id throws instead of being sent. - * - Thinking is adaptive with `output_config.effort`: manual - * `{type:"enabled", budget_tokens}` is deprecated on Sonnet/Opus 4.6 and - * 400s on Claude 5.x. Thinking tokens count toward `max_tokens`, and adaptive + * - `model` comes from `resolveCheckpointModel`. + * - Thinking is chosen per model (`ADAPTIVE_THINKING_WIRE_IDS`). Adaptive models get + * `{type:"adaptive"}` + `output_config.effort`: manual `budget_tokens` is deprecated on + * Sonnet/Opus 4.6 and 400s on 4.7+ and 5.x. Thinking tokens count toward `max_tokens`, and adaptive * thinking has no separate thinking cap, so `max_tokens` bounds thinking PLUS * answer, not thinking alone. What it guarantees: one checkpoint spends at most * MAX_OUTPUT_TOKENS + min(MAX_THINKING_BUDGET_TOKENS, remaining) tokens in total, @@ -111,25 +133,29 @@ export type ParseOutcome = * `usage.output_tokens`) and stopped by `shouldContinue`, spends at most * `thinkingBudgetTokens + MAX_OUTPUT_TOKENS` on thinking. The budget is a * checkpoint-granular bound with that one-answer overshoot, not an exact cap. + * - Every other model gets `{type:"enabled", budget_tokens}` with the same allowance, below + * `max_tokens`, and no `effort`. Under the API minimum of 1,024 the request carries no + * `thinking` at all: there is nothing left to think with. */ export function buildCheckpointRequestBody(args: { modelName: string; prompt: Pick; remainingThinkingTokens: number; }): Record { - const model = anthropicWireId(args.modelName); - if (!model) { - throw new Error(`no Anthropic wire id for model "${args.modelName}"`); - } + const model = resolveCheckpointModel(args.modelName); const thinkingAllowance = Math.max( 0, Math.min(MAX_THINKING_BUDGET_TOKENS, args.remainingThinkingTokens), ); + const thinkingShape: Record = ADAPTIVE_THINKING_WIRE_IDS.has(model) + ? { thinking: { type: "adaptive" }, output_config: { effort: "medium" } } + : thinkingAllowance >= MIN_MANUAL_BUDGET_TOKENS + ? { thinking: { type: "enabled", budget_tokens: thinkingAllowance } } + : {}; return { model, max_tokens: MAX_OUTPUT_TOKENS + thinkingAllowance, - thinking: { type: "adaptive" }, - output_config: { effort: "medium" }, + ...thinkingShape, system: args.prompt.system, messages: [{ role: "user", content: args.prompt.user }], }; From 1397bcdd1a4bc51ba81d01bee4fce8fef308f5a4 Mon Sep 17 00:00:00 2001 From: homen Date: Fri, 16 Oct 2026 03:58:46 -0700 Subject: [PATCH 6/8] fix(models): close the round 5 review findings (PR 1) - Daily-brief public actions validate with the catalog lookup that dispatches (assertKnownModel(resolveModelAlias(model))), not the resolver's normalizeModelInput. The resolver check rejected the 33 short names the catalog dispatches (gpt5, sonnet, gemini-3 ...). assertKnownModel reads no environment, so the boundary accepts exactly what dispatch accepts. - Object.prototype keys are not models: every alias, pricing, equivalent, fallback, wire-id and LEGACY_ALIASES lookup goes through ownEntry (own-property only). toString / valueOf / hasOwnProperty no longer pass normalizeModelInput, getModelWithFailover or resolveCheckpointModel. - UnknownModelError lives in shared/llm/modelCatalog.ts so the catalog and the resolver throw the same class. FREE_LANE_WIRE_ID and FREE_LANE_CATALOG_ID are typed once there; modelResolver.ts imports them for MODEL_SPECS sdkId and FREE_LANE_MODEL. - One shared assertion (shared/llm/expectClaudeRequestShape.ts) that budget_tokens, thinking enabled/disabled and a forced tool_choice are absent, used by events, providerCopies, benchmark, mcp-local, tool-gen and modelRouter request-shape suites. Adding thinking:{enabled,budget_tokens} and tool_choice:{any} to each source made every one of those suites fail. Tests: dailyBriefModelBoundary (every catalog alias accepted; prototype keys rejected), anthropicWireIds / modelResolver / extendedThinkingRunner prototype-key tests (31 fail with the lookups reverted to bracket access). CI runtime-smoke list: 47 files, 693 tests pass; packet contracts 20/20; tsc root and backend/convex exit 0. Co-Authored-By: Claude Sonnet 5.5 --- .../mcp_tools/models/modelResolver.test.ts | 14 ++++ .../agents/mcp_tools/models/modelResolver.ts | 39 +++++----- .../models/modelRouter.requestShape.test.ts | 2 + .../research/dailyBriefModelBoundary.test.ts | 36 ++++++++- .../research/dailyBriefPersonalOverlay.ts | 10 +-- .../domains/research/dailyBriefWorker.ts | 10 +-- .../fusion/benchmark.requestShape.test.ts | 2 + backend/convex/events.requestShape.test.ts | 2 + .../providerCopies.requestShape.test.ts | 2 + .../__tests__/anthropicRequestShape.test.ts | 3 + scripts/__tests__/toolGenRequestShape.test.ts | 3 + shared/llm/anthropicWireIds.test.ts | 24 ++++++ shared/llm/expectClaudeRequestShape.ts | 20 +++++ shared/llm/modelCatalog.ts | 74 +++++++++++++------ .../pipeline/extendedThinkingRunner.test.ts | 8 ++ 15 files changed, 197 insertions(+), 52 deletions(-) create mode 100644 shared/llm/expectClaudeRequestShape.ts diff --git a/backend/convex/domains/agents/mcp_tools/models/modelResolver.test.ts b/backend/convex/domains/agents/mcp_tools/models/modelResolver.test.ts index 683826aa5..b0881eda0 100644 --- a/backend/convex/domains/agents/mcp_tools/models/modelResolver.test.ts +++ b/backend/convex/domains/agents/mcp_tools/models/modelResolver.test.ts @@ -359,3 +359,17 @@ describe("normalizeNodeBenchRuntimeModel is the first resolver and fails closed" expect(normalizeNodeBenchRuntimeModel("gpt-9-ultra", "advisor", { allowPaidFallback: true })).toBe(DEFAULT_MODEL); }); }); + +// Persona: a signed-in user types "toString" into a model field. LEGACY_ALIASES is a plain object, so the +// inherited property used to resolve to a function and pass every boundary (A.0.10, PR #662 round 5). +describe("Object.prototype keys are not models", () => { + const PROTOTYPE_KEYS = ["toString", "valueOf", "hasOwnProperty", "constructor", "__proto__", "isPrototypeOf", "toLocaleString"]; + + it.each(PROTOTYPE_KEYS)("%s resolves to nothing and every entry point rejects it", (key) => { + expect(resolveModelAlias(key)).toBeNull(); + expect(() => normalizeModelInput(key)).toThrow(UnknownModelError); + expect(() => getLanguageModelSafe(key)).toThrow(UnknownModelError); + expect(() => resolvePipelineModelSelection(key)).toThrow(UnknownModelError); + expect(() => assertKnownPipelineModel(key)).toThrow(UnknownModelError); + }); +}); diff --git a/backend/convex/domains/agents/mcp_tools/models/modelResolver.ts b/backend/convex/domains/agents/mcp_tools/models/modelResolver.ts index 96e02466b..2e5e86b91 100644 --- a/backend/convex/domains/agents/mcp_tools/models/modelResolver.ts +++ b/backend/convex/domains/agents/mcp_tools/models/modelResolver.ts @@ -19,6 +19,14 @@ import { createOpenAI, openai } from "@ai-sdk/openai"; import { anthropic } from "@ai-sdk/anthropic"; import { createGoogleGenerativeAI } from "@ai-sdk/google"; import type { LanguageModel } from "ai"; +import { + FREE_LANE_CATALOG_ID, + FREE_LANE_WIRE_ID, + UnknownModelError, + ownEntry, +} from "../../../../../../shared/llm/modelCatalog"; + +export { UnknownModelError }; // ═══════════════════════════════════════════════════════════════════════════ // OPENROUTER PROVIDER SETUP @@ -142,26 +150,17 @@ export const FALLBACK_MODEL: ApprovedModel = "gpt-5.4-mini"; /** * The one verified free lane. This is the typed resolver boundary (an approved - * alias): code that hands a value to the resolver writes this constant. The - * provider wire string is read from `MODEL_SPECS[FREE_LANE_MODEL].sdkId` and is - * never typed a second time. Evidence: NodeVideo's measured free-model benchmark + * alias): code that hands a value to the resolver writes this constant. Both the + * alias and its provider wire string (`MODEL_SPECS[FREE_LANE_MODEL].sdkId`) are typed once, + * in shared/llm/modelCatalog.ts (`FREE_LANE_CATALOG_ID`, `FREE_LANE_WIRE_ID`), and imported here. + * Evidence: NodeVideo's measured free-model benchmark * (config/openrouter-free-routing.json), the only measured free pick in the portfolio. * SPEC-v9 A.2.10 PR 1 (Codex v3 N2). */ -export const FREE_LANE_MODEL: ApprovedModel = "nemotron-3-super-free"; +export const FREE_LANE_MODEL: ApprovedModel = FREE_LANE_CATALOG_ID; -/** - * Thrown when an id is not an approved model, not a legacy alias, and not - * free-looking. Unknown ids never become a paid default (SPEC-v9 A.0.10). - */ -export class UnknownModelError extends Error { - readonly input: string; - constructor(input: string) { - super(`Unknown model "${input}". Allowed models: ${APPROVED_MODELS.join(", ")}`); - this.name = "UnknownModelError"; - this.input = input; - } -} +const unknownModelError = (input: string) => + new UnknownModelError(input, `Allowed models: ${APPROVED_MODELS.join(", ")}`); /** * Opt-in for the single kind of caller allowed to turn an unknown id into @@ -401,7 +400,7 @@ export function resolvePipelineModelSelection( usedDefault: false, }; } - throw new UnknownModelError(requestedModelId); + throw unknownModelError(requestedModelId); } /** @@ -649,7 +648,7 @@ export const MODEL_SPECS: Record = { "nemotron-3-super-free": { alias: "nemotron-3-super-free", provider: "openrouter", - sdkId: "nvidia/nemotron-3-super-120b-a12b:free", + sdkId: FREE_LANE_WIRE_ID, capabilities: { vision: false, toolUse: true, streaming: true, structuredOutputs: true, maxContext: 262_144 }, pricing: { inputPerMillion: 0.00, outputPerMillion: 0.00 }, }, @@ -985,7 +984,7 @@ export function resolveModelAlias(input: string): ApprovedModel | null { } // Check legacy aliases - return LEGACY_ALIASES[input] ?? LEGACY_ALIASES[normalized] ?? null; + return ownEntry(LEGACY_ALIASES, input) ?? ownEntry(LEGACY_ALIASES, normalized) ?? null; } /** @@ -1037,7 +1036,7 @@ function resolveOrFailClosed( ); return DEFAULT_MODEL; } - throw new UnknownModelError(input); + throw unknownModelError(input); } /** diff --git a/backend/convex/domains/ai/models/modelRouter.requestShape.test.ts b/backend/convex/domains/ai/models/modelRouter.requestShape.test.ts index 8ed05803d..5879ce0e0 100644 --- a/backend/convex/domains/ai/models/modelRouter.requestShape.test.ts +++ b/backend/convex/domains/ai/models/modelRouter.requestShape.test.ts @@ -21,6 +21,7 @@ vi.mock("./chainResolver", () => ({ resolveChain })); import { route } from "./modelRouter"; import { checkAgentBudget } from "./modelRouterQueries"; import { getFreeModels } from "../../agents/mcp_tools/models"; +import { expectNoManualThinkingOrForcedToolChoice } from "../../../../../shared/llm/expectClaudeRequestShape"; const baseArgs = { taskCategory: "research", @@ -61,6 +62,7 @@ describe("modelRouter.route request shape and cost class", () => { sent[pinnedModel] = "temperature" in request; dispatched[pinnedModel] = (request.model as { modelId: string }).modelId; for (const key of ["top_p", "top_k", "topP", "topK"]) expect(request, `${pinnedModel} ${key}`).not.toHaveProperty(key); + expectNoManualThinkingOrForcedToolChoice(request, pinnedModel); // No assistant prefill (400 on Claude 4.7+/5.x, Gemini 3.x): the outgoing conversation ends on a user turn. expect(request.messages.at(-1).role, `${pinnedModel} last turn`).toBe("user"); } diff --git a/backend/convex/domains/research/dailyBriefModelBoundary.test.ts b/backend/convex/domains/research/dailyBriefModelBoundary.test.ts index 19b9c1155..839d4e71d 100644 --- a/backend/convex/domains/research/dailyBriefModelBoundary.test.ts +++ b/backend/convex/domains/research/dailyBriefModelBoundary.test.ts @@ -17,7 +17,7 @@ vi.mock("@convex-dev/auth/server", () => ({ getAuthUserId: vi.fn(async () => "us import { runNextPersonalTask } from "./dailyBriefPersonalOverlay"; import { runNextTask } from "./dailyBriefWorker"; import { UnknownModelError } from "../agents/mcp_tools/models"; -import { FREE_LANE_WIRE_ID, LEGACY_PASSTHROUGH_IDS, getModelWithFailover, resolveModelAlias } from "../../../../shared/llm/modelCatalog"; +import { FREE_LANE_WIRE_ID, LEGACY_PASSTHROUGH_IDS, getModelWithFailover, modelAliases, resolveModelAlias } from "../../../../shared/llm/modelCatalog"; const actions = [ ["dailyBriefWorker.runNextTask", runNextTask], @@ -76,3 +76,37 @@ describe.each(actions)("%s model boundary", (_name, action) => { vi.unstubAllEnvs(); }); }); + +// Persona: a signed-in user picks a short model name from the catalog ("gpt5", "sonnet", "gemini-3"). +// Dispatch resolves it through the catalog (resolveModelAlias -> getModelWithFailover), so the boundary +// must accept exactly the names dispatch accepts. Round 5: the resolver-based check rejected 33 of them. +describe.each(actions)("%s boundary agrees with dispatch", (_name, action) => { + const run = (model: string) => { + const ctx = { runAction: vi.fn(async () => ({ done: true })) }; + return { ctx, result: (action as any)._handler(ctx, { model }) }; + }; + + it("accepts every alias the catalog resolves, exactly as dispatch does", async () => { + vi.spyOn(console, "warn").mockImplementation(() => {}); + vi.spyOn(console, "error").mockImplementation(() => {}); + const aliases = Object.keys(modelAliases); + expect(aliases.length).toBeGreaterThan(30); + for (const alias of aliases) { + expect(() => getModelWithFailover(resolveModelAlias(alias)), `dispatch ${alias}`).not.toThrow(); + const { ctx, result } = run(alias); + await result; + expect(ctx.runAction, alias).toHaveBeenCalledTimes(1); + } + }); + + // Object.prototype keys are not models (A.0.10): before the own-property lookups they counted as known. + it.each(["toString", "valueOf", "hasOwnProperty", "constructor", "__proto__", "isPrototypeOf"])( + "rejects Object.prototype key %s before any internal action runs", + async (model) => { + const { ctx, result } = run(model); + await expect(result).rejects.toBeInstanceOf(UnknownModelError); + expect(ctx.runAction).not.toHaveBeenCalled(); + expect(() => getModelWithFailover(resolveModelAlias(model))).toThrow(UnknownModelError); + }, + ); +}); diff --git a/backend/convex/domains/research/dailyBriefPersonalOverlay.ts b/backend/convex/domains/research/dailyBriefPersonalOverlay.ts index fec6df4f2..9ef51ca55 100644 --- a/backend/convex/domains/research/dailyBriefPersonalOverlay.ts +++ b/backend/convex/domains/research/dailyBriefPersonalOverlay.ts @@ -11,7 +11,6 @@ import { v } from "convex/values"; import { internalAction, action } from "../../_generated/server"; import { internal, api } from "../../_generated/api"; import { getAuthUserId } from "@convex-dev/auth/server"; -import { normalizeModelInput } from "../agents/mcp_tools/models/modelResolver"; import OpenAI from "openai"; import Anthropic from "@anthropic-ai/sdk"; @@ -21,6 +20,7 @@ import { getLlmModel, resolveModelAlias, getModelWithFailover, + assertKnownModel, anthropicText, } from "../../../../shared/llm/modelCatalog"; @@ -374,10 +374,10 @@ export const runNextPersonalTask = action({ handler: async (ctx, args): Promise => { const userId = await getAuthUserId(ctx); if (!userId) throw new Error("Not authenticated"); - // A.0.10: an unknown model id throws here, before any provider is reached. Validation only: - // the caller's string is forwarded as given, because normalizeModelInput returns the resolver's - // lossy alias (claude-sonnet-4-5-20250929 -> claude-sonnet-4, retired), which PR 1 must not send. - if (args.model !== undefined) normalizeModelInput(args.model); + // A.0.10: an unknown model id throws here, before any provider is reached. Validation only, + // by the same catalog lookup dispatch uses (resolveModelAlias -> getModelWithFailover), so the + // boundary accepts exactly what dispatch accepts. The caller's string is forwarded as given. + if (args.model !== undefined) assertKnownModel(resolveModelAlias(args.model)); return await ctx.runAction( internal.domains.research.dailyBriefPersonalOverlay.runNextPersonalTaskInternal, args, diff --git a/backend/convex/domains/research/dailyBriefWorker.ts b/backend/convex/domains/research/dailyBriefWorker.ts index cc67cf2db..b1ce8b402 100644 --- a/backend/convex/domains/research/dailyBriefWorker.ts +++ b/backend/convex/domains/research/dailyBriefWorker.ts @@ -10,7 +10,6 @@ import { v } from "convex/values"; import { internalAction, action } from "../../_generated/server"; import { internal, api } from "../../_generated/api"; import { getAuthUserId } from "@convex-dev/auth/server"; -import { normalizeModelInput } from "../agents/mcp_tools/models/modelResolver"; import OpenAI from "openai"; import Anthropic from "@anthropic-ai/sdk"; @@ -20,6 +19,7 @@ import { getLlmModel, resolveModelAlias, getModelWithFailover, + assertKnownModel, anthropicText, } from "../../../../shared/llm/modelCatalog"; @@ -599,10 +599,10 @@ export const runNextTask = action({ handler: async (ctx, args): Promise => { const userId = await getAuthUserId(ctx); if (!userId) throw new Error("Not authenticated"); - // A.0.10: an unknown model id throws here, before any provider is reached. Validation only: - // the caller's string is forwarded as given, because normalizeModelInput returns the resolver's - // lossy alias (claude-sonnet-4-5-20250929 -> claude-sonnet-4, retired), which PR 1 must not send. - if (args.model !== undefined) normalizeModelInput(args.model); + // A.0.10: an unknown model id throws here, before any provider is reached. Validation only, + // by the same catalog lookup dispatch uses (resolveModelAlias -> getModelWithFailover), so the + // boundary accepts exactly what dispatch accepts. The caller's string is forwarded as given. + if (args.model !== undefined) assertKnownModel(resolveModelAlias(args.model)); const res: any = await ctx.runAction( internal.domains.research.dailyBriefWorker.runNextTaskInternal, diff --git a/backend/convex/domains/search/fusion/benchmark.requestShape.test.ts b/backend/convex/domains/search/fusion/benchmark.requestShape.test.ts index be19e55a9..7a479262e 100644 --- a/backend/convex/domains/search/fusion/benchmark.requestShape.test.ts +++ b/backend/convex/domains/search/fusion/benchmark.requestShape.test.ts @@ -9,6 +9,7 @@ */ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { expectNoManualThinkingOrForcedToolChoice } from "../../../../../shared/llm/expectClaudeRequestShape"; import { evaluateSearch } from "./benchmark"; @@ -59,6 +60,7 @@ describe("benchmark.evaluateSearch judge request shape", () => { expect(captured[0].url).toBe("https://api.anthropic.com/v1/messages"); expect(captured[0].body.model).toBe("claude-sonnet-4-6"); for (const key of ["temperature", "top_p", "top_k"]) expect(captured[0].body, key).not.toHaveProperty(key); + expectNoManualThinkingOrForcedToolChoice(captured[0].body, "benchmark judge"); expect(captured[0].body.max_tokens).toBeGreaterThanOrEqual(1024); // Claude 4.7+/5.x answer 400 to an assistant prefill: the conversation must end on a user turn (round 4). expect(captured[0].body.messages.at(-1).role).toBe("user"); diff --git a/backend/convex/events.requestShape.test.ts b/backend/convex/events.requestShape.test.ts index f3ffeab7d..648412111 100644 --- a/backend/convex/events.requestShape.test.ts +++ b/backend/convex/events.requestShape.test.ts @@ -12,6 +12,7 @@ */ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { expectNoManualThinkingOrForcedToolChoice } from "../../shared/llm/expectClaudeRequestShape"; import { generateBouncerAdvisory, generateProviderAnswer, runJoinBouncer } from "./events"; @@ -47,6 +48,7 @@ function expectMessagesRequestShape(captured: Captured[], model: string) { expect(url).toBe("https://api.anthropic.com/v1/messages"); expect(body.model).toBe(model); for (const key of SAMPLING_KEYS) expect(body, `${key} must be absent`).not.toHaveProperty(key); + expectNoManualThinkingOrForcedToolChoice(body, "events.ts"); expect(body.max_tokens).toBeGreaterThanOrEqual(1024); // no prefill: the last turn is the user's expect(body.messages.at(-1).role).toBe("user"); diff --git a/backend/convex/providerCopies.requestShape.test.ts b/backend/convex/providerCopies.requestShape.test.ts index 324ef9d59..36e21727d 100644 --- a/backend/convex/providerCopies.requestShape.test.ts +++ b/backend/convex/providerCopies.requestShape.test.ts @@ -50,6 +50,7 @@ import { judgeClaimAgainstSource } from "./domains/verification/claimVerificatio import { handleChatResponse } from "./domains/agents/fastAgentChat"; import { generateResponse } from "./domains/ai/ai"; import { generateForDocument } from "./tags_actions"; +import { expectNoManualThinkingOrForcedToolChoice } from "../../shared/llm/expectClaudeRequestShape"; const ANSWER = "the answer"; const thinkingThen = (text: string) => ({ @@ -66,6 +67,7 @@ function expectAnthropicRequest(maxTokens?: number) { expect(body.model).toBe("claude-sonnet-4-6"); // the wire id, never "claude-sonnet-4.6" for (const key of ["temperature", "top_p", "top_k"]) expect(body, key).not.toHaveProperty(key); expect(body.messages.at(-1).role).toBe("user"); // no prefill + expectNoManualThinkingOrForcedToolChoice(body, "providerCopy"); if (maxTokens !== undefined) expect(body.max_tokens).toBe(maxTokens); return body; } diff --git a/packages/mcp-local/src/__tests__/anthropicRequestShape.test.ts b/packages/mcp-local/src/__tests__/anthropicRequestShape.test.ts index 2fb3b68ec..5072610ba 100644 --- a/packages/mcp-local/src/__tests__/anthropicRequestShape.test.ts +++ b/packages/mcp-local/src/__tests__/anthropicRequestShape.test.ts @@ -19,6 +19,7 @@ vi.mock("@anthropic-ai/sdk", () => ({ import { llmTools } from "../tools/llmTools.js"; import { researchWritingTools } from "../tools/researchWritingTools.js"; +import { expectNoManualThinkingOrForcedToolChoice } from "../../../../shared/llm/expectClaudeRequestShape"; const thinkingThenText = (text: string) => ({ content: [ @@ -51,6 +52,7 @@ describe("mcp-local Anthropic provider request shape", () => { const body = create.mock.calls[0][0]; expect(body.model).toBe("claude-haiku-4-5-20251001"); // PR 1 moves no ids for (const key of ["temperature", "top_p", "top_k"]) expect(body, key).not.toHaveProperty(key); + expectNoManualThinkingOrForcedToolChoice(body, "mcp-local"); expect(body.max_tokens).toBe(777); expect(body.messages).toEqual([{ role: "user", content: "hi" }]); expect(body.messages.at(-1).role).toBe("user"); // no assistant prefill (400 on Claude 4.7+/5.x) @@ -64,6 +66,7 @@ describe("mcp-local Anthropic provider request shape", () => { const body = create.mock.calls[0][0]; expect(body.model).toBe("claude-haiku-4-5-20251001"); // PR 1 moves no ids for (const key of ["temperature", "top_p", "top_k"]) expect(body, key).not.toHaveProperty(key); + expectNoManualThinkingOrForcedToolChoice(body, "mcp-local"); expect(body.max_tokens).toBe(4096); expect(body.messages.at(-1).role).toBe("user"); // no assistant prefill (400 on Claude 4.7+/5.x) expect(result.polishedText).toBe("the answer"); diff --git a/scripts/__tests__/toolGenRequestShape.test.ts b/scripts/__tests__/toolGenRequestShape.test.ts index 18a280259..63d8bbfe1 100644 --- a/scripts/__tests__/toolGenRequestShape.test.ts +++ b/scripts/__tests__/toolGenRequestShape.test.ts @@ -9,6 +9,7 @@ */ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { expectNoManualThinkingOrForcedToolChoice } from "../../shared/llm/expectClaudeRequestShape"; // @ts-expect-error - plain .mjs script without types import { buildClaudeRequest, callClaude, claudeText } from "../tool-gen/analyze-gap.mjs"; @@ -29,6 +30,7 @@ describe("analyze-gap.mjs Anthropic request", () => { // PR 1 moves no ids; PR 2 changes this literal to claude-haiku-5-5 with its own update. expect(body.model).toBe("claude-haiku-4-5-20251001"); for (const key of ["temperature", "top_p", "top_k"]) expect(body).not.toHaveProperty(key); + expectNoManualThinkingOrForcedToolChoice(body, "tool-gen"); expect(body.max_tokens).toBeGreaterThanOrEqual(1024); expect(body.system).toBe("system text"); expect(body.messages).toEqual([{ role: "user", content: "user text" }]); @@ -85,6 +87,7 @@ describe("generate-tool.mjs Anthropic request", () => { // PR 1 moves no ids; PR 2 changes this literal to claude-sonnet-5-5 with its own update. expect(body.model).toBe("claude-sonnet-4-6"); for (const key of ["temperature", "top_p", "top_k"]) expect(body).not.toHaveProperty(key); + expectNoManualThinkingOrForcedToolChoice(body, "tool-gen"); expect(body.max_tokens).toBe(4096); expect(body.messages).toEqual([{ role: "user", content: "user text" }]); }); diff --git a/shared/llm/anthropicWireIds.test.ts b/shared/llm/anthropicWireIds.test.ts index 901a87f86..2c38851c6 100644 --- a/shared/llm/anthropicWireIds.test.ts +++ b/shared/llm/anthropicWireIds.test.ts @@ -18,7 +18,12 @@ import { anthropicWireId, FREE_LANE_CATALOG_ID, FREE_LANE_WIRE_ID, + UnknownModelError, + getEquivalentModel, + getModelPricing, getModelWithFailover, + getNextFallback, + getProviderForModel, llmModelCatalog, resolveModelAlias, } from "./modelCatalog"; @@ -254,3 +259,22 @@ describe("getModelWithFailover fails closed", () => { expect(() => getModelWithFailover(FREE_LANE_MODEL)).toThrow(/never fails over to a paid provider/); }); }); + +// Persona: a signed-in user types "toString" into a model field. The alias, pricing, equivalent, +// fallback and wire-id tables are plain objects, so an inherited property used to count as a model +// and (with OpenRouter unset) the request went to a paid default (A.0.10, PR #662 round 5). +describe("Object.prototype keys are not models", () => { + const PROTOTYPE_KEYS = ["toString", "valueOf", "hasOwnProperty", "constructor", "__proto__", "isPrototypeOf", "toLocaleString"]; + + it.each(PROTOTYPE_KEYS)("%s is unknown in every catalog table and never dispatches", (key) => { + vi.stubEnv("OPENROUTER_API_KEY", ""); + vi.stubEnv("GEMINI_API_KEY", "gemini-test-key-not-real"); + expect(resolveModelAlias(key)).toBe(key); + expect(anthropicWireId(key)).toBeNull(); + expect(getModelPricing(key)).toBeNull(); + expect(getProviderForModel(key)).toBeNull(); + expect(getNextFallback(key, [])).toBeNull(); + expect(typeof getEquivalentModel(key, "openai")).toBe("string"); + expect(() => getModelWithFailover(resolveModelAlias(key))).toThrow(UnknownModelError); + }); +}); diff --git a/shared/llm/expectClaudeRequestShape.ts b/shared/llm/expectClaudeRequestShape.ts new file mode 100644 index 000000000..dc1c7bb43 --- /dev/null +++ b/shared/llm/expectClaudeRequestShape.ts @@ -0,0 +1,20 @@ +/** + * Shared request-shape assertion for every suite that builds a real Anthropic request through the + * repo's own code with a fake transport (SPEC-v9 A.0.6, s1.3). Claude Sonnet 5.5 and Opus 5.5 + * answer 400 to a manual thinking budget, to `thinking:{type:"disabled"}` and to a forced + * `tool_choice`; a suite that checks only sampling keys stays green when one of them is added. + * Accepts the raw Messages body (`thinking`, `tool_choice`) and the AI SDK call + * (`providerOptions.anthropic.thinking`, `toolChoice`). + */ +import { expect } from "vitest"; + +export function expectNoManualThinkingOrForcedToolChoice(request: unknown, label = "request"): void { + const r = (request ?? {}) as Record; + const thinking = r.thinking ?? r.providerOptions?.anthropic?.thinking; + const toolChoice = r.tool_choice ?? r.toolChoice; + expect(thinking?.budget_tokens, `${label}: thinking.budget_tokens must be absent`).toBeUndefined(); + expect(thinking?.budgetTokens, `${label}: thinking.budgetTokens must be absent`).toBeUndefined(); + expect(["enabled", "disabled"], `${label}: thinking.type must not be enabled/disabled`).not.toContain(thinking?.type); + const forced = typeof toolChoice === "string" ? toolChoice === "required" : ["any", "tool"].includes(toolChoice?.type); + expect(forced, `${label}: tool_choice must not be forced`).toBe(false); +} diff --git a/shared/llm/modelCatalog.ts b/shared/llm/modelCatalog.ts index d1a8369c6..b7bf60b7e 100644 --- a/shared/llm/modelCatalog.ts +++ b/shared/llm/modelCatalog.ts @@ -49,6 +49,30 @@ export type LlmTask = export type UserTier = "anonymous" | "free" | "pro" | "team" | "enterprise"; +/** + * Own-property lookup for the alias, pricing, equivalent, fallback and wire-id tables below. + * These tables are plain objects keyed by untrusted model ids, so `table["toString"]` would + * otherwise return `Object.prototype.toString` and a name nobody registered would count as known + * (SPEC-v9 A.0.10). `Object.prototype.hasOwnProperty.call` rather than `Object.hasOwn`: backend/convex compiles with lib ES2021. + */ +export function ownEntry(table: Readonly>, key: string): T | undefined { + return Object.prototype.hasOwnProperty.call(table, key) ? table[key] : undefined; +} + +/** + * Thrown when an id is not a model this repo knows and is not free-looking. Unknown ids never + * become a paid default (SPEC-v9 A.0.10). Defined here, not in modelResolver.ts, so the catalog's + * `getModelWithFailover` and the resolver reject with the same class. + */ +export class UnknownModelError extends Error { + readonly input: string; + constructor(input: string, detail?: string) { + super(`Unknown model "${input}"${detail ? `. ${detail}` : ""}`); + this.name = "UnknownModelError"; + this.input = input; + } +} + // ═══════════════════════════════════════════════════════════════════════════ // MODEL PRICING (per 1M tokens, USD) // ═══════════════════════════════════════════════════════════════════════════ @@ -332,7 +356,7 @@ export function calculateRequestCost( outputTokens: number, useCachedInput: boolean = false ): number { - const pricing = modelPricing[modelName]; + const pricing = ownEntry(modelPricing, modelName); if (!pricing) { console.warn(`[calculateRequestCost] No pricing for model "${modelName}", using estimate`); // Default to gpt-5.4-nano pricing as fallback @@ -350,7 +374,7 @@ export function calculateRequestCost( * Get pricing info for a model */ export function getModelPricing(modelName: string): ModelPricing | null { - return modelPricing[modelName] ?? null; + return ownEntry(modelPricing, modelName) ?? null; } /** @@ -366,7 +390,7 @@ export function getProviderForModel(modelName: string): LlmProvider | null { if (modelName.startsWith("gemini-") || modelName.startsWith("deep-research-")) { return "gemini"; } - if (modelPricing[modelName] != null) { + if (ownEntry(modelPricing, modelName) != null) { return "openrouter"; } return null; @@ -561,7 +585,7 @@ export const modelAliases: Record = { */ export function resolveModelAlias(modelInput: string): string { const normalized = modelInput.toLowerCase().trim(); - return modelAliases[normalized] ?? modelInput; + return ownEntry(modelAliases, normalized) ?? modelInput; } // ----------------------------------------------------------------------------- @@ -624,7 +648,7 @@ export const LEGACY_PASSTHROUGH_IDS: ReadonlySet = new Set([ */ export function anthropicWireId(modelInput: string): string | null { const resolved = resolveModelAlias(modelInput.trim()); - const fromAlias = ANTHROPIC_WIRE_IDS[resolved]; + const fromAlias = ownEntry(ANTHROPIC_WIRE_IDS, resolved); if (fromAlias) return fromAlias; if (Object.values(ANTHROPIC_WIRE_IDS).includes(resolved)) return resolved; return resolved.startsWith("claude-") && LEGACY_PASSTHROUGH_IDS.has(resolved) ? resolved : null; @@ -704,7 +728,7 @@ export function validateContextWindow( promptText: string, reserveOutputTokens: number = 4000 ): { fits: boolean; tokenEstimate: number; contextWindow: number; overflow: number } { - const pricing = modelPricing[modelName]; + const pricing = ownEntry(modelPricing, modelName); const contextWindow = pricing?.contextWindow ?? 128000; // Default to 128K // Rough token estimation: ~4 chars per token for English @@ -755,7 +779,7 @@ export const modelFallbackChains: Record = { }; export function getNextFallback(model: string, attempted: string[]): string | null { - const chain = modelFallbackChains[model] ?? []; + const chain = ownEntry(modelFallbackChains, model) ?? []; return chain.find((m) => !attempted.includes(m)) ?? null; } @@ -872,7 +896,7 @@ export const modelEquivalents: Record> = { * Get equivalent model for a different provider (for failover) */ export function getEquivalentModel(modelName: string, targetProvider: LlmProvider): string { - const equivalents = modelEquivalents[modelName]; + const equivalents = ownEntry(modelEquivalents, modelName); if (equivalents?.[targetProvider]) { return equivalents[targetProvider]; } @@ -890,10 +914,9 @@ export function getEquivalentModel(modelName: string, targetProvider: LlmProvide } /** - * The verified free lane's OpenRouter wire id. Must equal - * `MODEL_SPECS[FREE_LANE_MODEL].sdkId` in modelResolver.ts (this file cannot import from - * backend/convex); anthropicWireIds.test.ts pins the two together. SPEC-v9 A.2.10: the - * alias `nemotron-3-super-free` never reaches a provider. + * The verified free lane's OpenRouter wire id, typed once: modelResolver.ts imports it for + * `MODEL_SPECS[FREE_LANE_MODEL].sdkId`. SPEC-v9 A.2.10: the alias `nemotron-3-super-free` + * never reaches a provider. */ export const FREE_LANE_WIRE_ID = "nvidia/nemotron-3-super-120b-a12b:free"; @@ -911,9 +934,7 @@ function toProviderWireId(model: string, provider: LlmProvider): string { } /** - * The verified free lane as the catalog names it. Must equal `FREE_LANE_MODEL` - * in modelResolver.ts (this file cannot import from backend/convex); - * anthropicWireIds.test.ts pins the two together. + * The verified free lane's alias, typed once: modelResolver.ts imports it as `FREE_LANE_MODEL`. */ export const FREE_LANE_CATALOG_ID = "nemotron-3-super-free"; @@ -934,15 +955,27 @@ function isFreeLookingModel(model: string): boolean { */ function isKnownCatalogModel(model: string): boolean { return ( - modelPricing[model] != null || - modelEquivalents[model] != null || + ownEntry(modelPricing, model) != null || + ownEntry(modelEquivalents, model) != null || LEGACY_PASSTHROUGH_IDS.has(model) || anthropicWireId(model) !== null ); } +/** + * Throws `UnknownModelError` unless `getModelWithFailover` would dispatch `model` (a known model or + * a free-looking id). Public actions call it with `resolveModelAlias(input)` before they hand the + * caller's string on, so the boundary accepts exactly what dispatch accepts. It reads no + * environment: whether a provider is configured is dispatch's concern, not validation's. + */ +export function assertKnownModel(model: string): void { + if (!isKnownCatalogModel(model) && !isFreeLookingModel(model)) { + throw new UnknownModelError(model, "an unrecognised id never fails over to a paid default and is never sent to a provider"); + } +} + function isFreeCatalogModel(model: string): boolean { - const price = modelPricing[model]; + const price = ownEntry(modelPricing, model); return price != null && price.inputPer1M === 0 && price.outputPer1M === 0; } @@ -959,6 +992,7 @@ export function getModelWithFailover(preferredModel: string): { provider: LlmProvider; isFallback: boolean; } { + assertKnownModel(preferredModel); const known = isKnownCatalogModel(preferredModel); const provider = getProviderForModel(preferredModel); @@ -966,9 +1000,7 @@ export function getModelWithFailover(preferredModel: string): { // The wire id, not the alias: the caller sends `model` to the provider as is. return { model: FREE_LANE_WIRE_ID, provider: "openrouter", isFallback: true }; } - if (!known || !provider) { - throw new Error(`[getModelWithFailover] Unknown model "${preferredModel}": an unrecognised id never fails over to a paid default and is never sent to a provider`); - } + if (!provider) throw new UnknownModelError(preferredModel, "no provider for this id"); // Check if preferred provider is configured if (isProviderConfigured(provider)) { diff --git a/workers/node/pipeline/extendedThinkingRunner.test.ts b/workers/node/pipeline/extendedThinkingRunner.test.ts index d57254b7e..3b5938017 100644 --- a/workers/node/pipeline/extendedThinkingRunner.test.ts +++ b/workers/node/pipeline/extendedThinkingRunner.test.ts @@ -341,6 +341,14 @@ describe("buildCheckpointRequestBody - Anthropic request shape (SPEC-v9 A.0.6)", expect(() => resolveCheckpointModel("claude-not-real")).toThrow(/no Anthropic wire id/); }); + it.each(["toString", "valueOf", "hasOwnProperty", "constructor", "__proto__"])( + "resolveCheckpointModel rejects Object.prototype key %s instead of returning a function", + (key) => { + expect(() => resolveCheckpointModel(key)).toThrow(/no Anthropic wire id/); + expect(() => buildCheckpointRequestBody({ modelName: key, prompt, remainingThinkingTokens: 5_000 })).toThrow(/no Anthropic wire id/); + }, + ); + it("accepts an operator-supplied wire id and refuses a name with no Anthropic id", () => { expect(buildCheckpointRequestBody({ modelName: "claude-sonnet-4-6", prompt, remainingThinkingTokens: 1 }).model).toBe( "claude-sonnet-4-6", From ad001caf82fb47c49ab869153f53e672eaecf279 Mon Sep 17 00:00:00 2001 From: homen Date: Fri, 16 Oct 2026 12:40:56 -0700 Subject: [PATCH 7/8] fix(models): close the round 4 review findings (PR 1, #662) D1: SPEC-v9 1.2 native wire ids (claude-*-5-5, gemini-3.x, gpt-6.x, gpt-5.4-mini) are known again and go out unchanged, as on main; the 5.5 Claude ids take adaptive thinking in extendedThinking. Registry ids the catalog cannot dispatch (OpenRouter-only, no branch until PR 2) are refused with a reason, never rerouted; an unrecognised free-looking id still takes the free lane but now logs it. A.0.10: startComposedPipelineRun and startComposedPipelineRunInternal validate modelId before admission. A.0.6: assert no forced tool_choice / no prefill on extendedThinking, no model-turn prefill on triSearch. Co-Authored-By: Claude Sonnet 5.5 --- .../pipelines/pipelineModelBoundary.test.ts | 42 +++++++ .../domains/pipelines/pipelineWorkflow.ts | 3 + .../extendedThinking.requestShape.test.ts | 18 ++- .../research/dailyBriefModelBoundary.test.ts | 37 +++++- shared/llm/anthropicWireIds.test.ts | 113 +++++++++++++++++- shared/llm/modelCatalog.ts | 62 +++++++++- shared/search/triSearch.test.ts | 3 + .../pipeline/extendedThinkingRunner.test.ts | 24 ++++ .../node/pipeline/extendedThinkingRunner.ts | 12 +- 9 files changed, 308 insertions(+), 6 deletions(-) diff --git a/backend/convex/domains/pipelines/pipelineModelBoundary.test.ts b/backend/convex/domains/pipelines/pipelineModelBoundary.test.ts index 5dd9abc98..a6eaa385b 100644 --- a/backend/convex/domains/pipelines/pipelineModelBoundary.test.ts +++ b/backend/convex/domains/pipelines/pipelineModelBoundary.test.ts @@ -111,6 +111,48 @@ describe.skipIf(!convexTestAvailable)("startPipelineRun model boundary", () => { }, 20_000); }); +// Round 4: the composed entry points reserved two admission units and started a workflow for any modelId +// ("constructor" included). Same-domain pair, same rule: validate first. +describe.skipIf(!convexTestAvailable)("startComposedPipelineRun model boundary", () => { + const composed = { composition: "research_then_code", spec: "Research then build" }; + + it("rejects an unknown modelId before admission: with the launch window full it still says UnknownModel", async () => { + const t = convexTest(schema, convexModules); + const userId = await t.run(async (ctx: any) => ctx.db.insert("users", { email: "composed@example.com" })); + const user = t.withIdentity({ subject: String(userId) }); + await fillLaunchWindow(t, `user:${String(userId)}`); + const before = await admissionCounts(t); + + // Control: a launch with an omitted model hits the full window, so the window really is full. + await expect(user.mutation(pipelines.pipelineWorkflow.startComposedPipelineRun, composed)).rejects.toThrow(/Pipeline launch limit reached/); + + for (const modelId of ["gpt-9-ultra", "constructor"]) { + await expect( + user.mutation(pipelines.pipelineWorkflow.startComposedPipelineRun, { ...composed, modelId }), + modelId, + ).rejects.toThrow(/Unknown model/); + } + expect(await admissionCounts(t)).toEqual(before); + }, 20_000); + + it("rejects an unknown modelId on the trusted internal composed entry point before admission too", async () => { + const t = convexTest(schema, convexModules); + await fillLaunchWindow(t, "user:cron-owner"); + const before = await admissionCounts(t); + await expect( + t.mutation(pipelines.pipelineWorkflow.startComposedPipelineRunInternal, { ...composed, ownerKey: "user:cron-owner" }), + ).rejects.toThrow(/Pipeline launch limit reached/); // control: the window is full + + for (const modelId of ["openai:gpt-9", "constructor"]) { + await expect( + t.mutation(pipelines.pipelineWorkflow.startComposedPipelineRunInternal, { ...composed, modelId, ownerKey: "user:cron-owner" }), + modelId, + ).rejects.toThrow(/Unknown model/); + } + expect(await admissionCounts(t)).toEqual(before); + }, 20_000); +}); + describe("piRuntime.parseModelKey", () => { it("throws for an unknown provider:model instead of returning a raw handle", async () => { // Before: the catch fell through to parseRawModelKey, which forwarded "gpt-9" to pi-ai. diff --git a/backend/convex/domains/pipelines/pipelineWorkflow.ts b/backend/convex/domains/pipelines/pipelineWorkflow.ts index 7c67b08dd..f8c0cf2c1 100644 --- a/backend/convex/domains/pipelines/pipelineWorkflow.ts +++ b/backend/convex/domains/pipelines/pipelineWorkflow.ts @@ -271,6 +271,8 @@ export const startComposedPipelineRun = mutation({ returns: v.object({ workflowId: v.string() }), handler: async (ctx, args): Promise<{ workflowId: string }> => { const ownerKey = await requireAuthenticatedPipelineOwnerKey(ctx); + // Fail closed before admission, as startPipelineRun does (SPEC-v9 A.0.10). + assertKnownPipelineModel(args.modelId); await reservePipelineLaunchAdmission(ctx, ownerKey, { units: 2 }); const launchArgs = normalizePipelineLaunchText(args); const attemptKey = launchArgs.forceFresh @@ -306,6 +308,7 @@ export const startComposedPipelineRunInternal = internalMutation({ handler: async (ctx, args): Promise<{ workflowId: string }> => { const ownerKey = normalizePipelineOwnerKey(args.ownerKey); const { ownerKey: _ownerKey, ...rawLaunchArgs } = args; + assertKnownPipelineModel(args.modelId); if (ownerKey.startsWith("user:")) { await reservePipelineLaunchAdmission(ctx, ownerKey, { units: 2 }); } diff --git a/backend/convex/domains/product/extendedThinking.requestShape.test.ts b/backend/convex/domains/product/extendedThinking.requestShape.test.ts index 84f9fef36..beb7b5dd8 100644 --- a/backend/convex/domains/product/extendedThinking.requestShape.test.ts +++ b/backend/convex/domains/product/extendedThinking.requestShape.test.ts @@ -91,6 +91,22 @@ describe("extendedThinking.advanceRun", () => { expect(body.thinking).not.toHaveProperty("budget_tokens"); for (const key of ["temperature", "top_p", "top_k"]) expect(body, key).not.toHaveProperty(key); expect(body.max_tokens).toBe(14_000); + // A.0.6: no forced tool choice (400 on Sonnet/Opus 5.5) and no prefill: the last message is the user's. + expect(body).not.toHaveProperty("tool_choice"); + expect(body).not.toHaveProperty("tools"); + expect((body.messages as Array<{ role: string }>).at(-1)?.role).toBe("user"); + }); + + // Persona: a founder whose run is stored on a 5.5 id (requestExtendedRun.modelName is a public argument). + // main sent the id unchanged; the wire id is already the stored string. + it.each(["claude-haiku-5-5", "claude-sonnet-5-5", "claude-opus-5-5"])("a run stored with %s sends adaptive thinking with effort, the id as written and no forced tool_choice", async (modelName) => { + const { requests } = await advance(runRow({ modelName }), 5_000); + const body = requests[0]; + expect(body.model).toBe(modelName); + expect(body.thinking).toEqual({ type: "adaptive" }); + expect(body.output_config).toEqual({ effort: "medium" }); + for (const key of ["temperature", "top_p", "top_k", "tool_choice"]) expect(body, key).not.toHaveProperty(key); + expect((body.messages as Array<{ role: string }>).at(-1)?.role).toBe("user"); }); it("charges thinking from usage.output_tokens even though the thinking text is omitted", async () => { @@ -166,7 +182,7 @@ describe("extendedThinking.requestExtendedRun model boundary", () => { expect(ctx.scheduler.runAfter).not.toHaveBeenCalled(); }); - it.each([undefined, "claude-sonnet-4.6", "claude-haiku-4-5", "claude-haiku-4-5-20251001"])("accepts %s and stores the name as given", async (modelName) => { + it.each([undefined, "claude-sonnet-4.6", "claude-haiku-4-5", "claude-haiku-4-5-20251001", "claude-haiku-5-5", "claude-sonnet-5-5", "claude-opus-5-5"])("accepts %s and stores the name as given", async (modelName) => { const ctx = makeCtx(); await request(ctx, modelName); expect(ctx.db.insert).toHaveBeenCalledTimes(1); diff --git a/backend/convex/domains/research/dailyBriefModelBoundary.test.ts b/backend/convex/domains/research/dailyBriefModelBoundary.test.ts index 839d4e71d..d8f4f35f3 100644 --- a/backend/convex/domains/research/dailyBriefModelBoundary.test.ts +++ b/backend/convex/domains/research/dailyBriefModelBoundary.test.ts @@ -17,7 +17,7 @@ vi.mock("@convex-dev/auth/server", () => ({ getAuthUserId: vi.fn(async () => "us import { runNextPersonalTask } from "./dailyBriefPersonalOverlay"; import { runNextTask } from "./dailyBriefWorker"; import { UnknownModelError } from "../agents/mcp_tools/models"; -import { FREE_LANE_WIRE_ID, LEGACY_PASSTHROUGH_IDS, getModelWithFailover, modelAliases, resolveModelAlias } from "../../../../shared/llm/modelCatalog"; +import { FREE_LANE_WIRE_ID, LEGACY_PASSTHROUGH_IDS, OPENROUTER_ONLY_REGISTRY_IDS, SPEC_NATIVE_WIRE_IDS, getModelWithFailover, modelAliases, resolveModelAlias } from "../../../../shared/llm/modelCatalog"; const actions = [ ["dailyBriefWorker.runNextTask", runNextTask], @@ -110,3 +110,38 @@ describe.each(actions)("%s boundary agrees with dispatch", (_name, action) => { }, ); }); + +// Homen D1 / PR #662 round 4: both public actions accepted every 1.2 native id on main (it sent them on their +// prefix) and must still forward each byte-for-byte; the registry's OpenRouter-only ids are refused up front. +describe.each(actions)("%s D1 boundary", (_name, action) => { + const run = (model: string) => { + const ctx = { runAction: vi.fn(async () => ({ done: true })) }; + return { ctx, result: (action as any)._handler(ctx, { model, taskId: "t1" }) }; + }; + + it("forwards every SPEC-v9 1.2 native id exactly as given, and dispatch sends that same string to its provider", async () => { + vi.stubEnv("ANTHROPIC_API_KEY", "sk-ant-test-key-not-real"); + vi.stubEnv("OPENAI_API_KEY", "sk-test-key-not-real-0000"); + vi.stubEnv("GEMINI_API_KEY", "gemini-test-key-not-real"); + expect(SPEC_NATIVE_WIRE_IDS.size).toBeGreaterThan(8); + for (const model of SPEC_NATIVE_WIRE_IDS) { + const { ctx, result } = run(model); + await result; + expect(ctx.runAction, model).toHaveBeenCalledWith(expect.anything(), { model, taskId: "t1" }); + expect(getModelWithFailover(resolveModelAlias(model)), model).toMatchObject({ model, isFallback: false }); + } + vi.unstubAllEnvs(); + }); + + it("refuses every OpenRouter-only registry id before any internal action runs, with every provider key set", async () => { + vi.spyOn(console, "warn").mockImplementation(() => {}); + vi.stubEnv("OPENROUTER_API_KEY", "sk-or-test-key-not-real"); + vi.stubEnv("OPENAI_API_KEY", "sk-test-key-not-real-0000"); + for (const model of OPENROUTER_ONLY_REGISTRY_IDS) { + const { ctx, result } = run(model); + await expect(result, model).rejects.toBeInstanceOf(UnknownModelError); + expect(ctx.runAction, model).not.toHaveBeenCalled(); + } + vi.unstubAllEnvs(); + }); +}); diff --git a/shared/llm/anthropicWireIds.test.ts b/shared/llm/anthropicWireIds.test.ts index 2c38851c6..239502c4d 100644 --- a/shared/llm/anthropicWireIds.test.ts +++ b/shared/llm/anthropicWireIds.test.ts @@ -14,6 +14,11 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { ANTHROPIC_WIRE_IDS, LEGACY_PASSTHROUGH_IDS, + OPENROUTER_ONLY_REGISTRY_IDS, + SPEC_NATIVE_WIRE_IDS, + assertKnownModel, + modelEquivalents, + modelPricing, anthropicText, anthropicWireId, FREE_LANE_CATALOG_ID, @@ -27,7 +32,7 @@ import { llmModelCatalog, resolveModelAlias, } from "./modelCatalog"; -import { FREE_LANE_MODEL, LEGACY_ALIASES, MODEL_SPECS } from "../../backend/convex/domains/agents/mcp_tools/models/modelResolver"; +import { APPROVED_MODELS, FREE_LANE_MODEL, LEGACY_ALIASES, MODEL_SPECS } from "../../backend/convex/domains/agents/mcp_tools/models/modelResolver"; describe("ANTHROPIC_WIRE_IDS agrees with the resolver's MODEL_SPECS", () => { const anthropicSpecs = Object.values(MODEL_SPECS).filter((spec) => spec.provider === "anthropic"); @@ -278,3 +283,109 @@ describe("Object.prototype keys are not models", () => { expect(() => getModelWithFailover(resolveModelAlias(key))).toThrow(UnknownModelError); }); }); + +// Homen D1 (2026-10-16): "known" = SPEC-v9 section 1.2 plus the repo's own registries, and a known id is +// never silently rerouted. PR 1 moves no ids (A.0.4), so every 1.2 native id that main sent unchanged to +// its own provider is still sent unchanged (PR #662 review round 4, P1). +describe("D1: SPEC-v9 1.2 native wire ids are known and go out unchanged", () => { + const providerOf = (id: string) => (id.startsWith("claude-") ? "anthropic" : id.startsWith("gemini-") ? "gemini" : "openai"); + beforeEach(() => { + vi.spyOn(console, "warn").mockImplementation(() => {}); + vi.spyOn(console, "error").mockImplementation(() => {}); + vi.stubEnv("ANTHROPIC_API_KEY", "sk-ant-test-key-not-real"); + vi.stubEnv("OPENAI_API_KEY", "sk-test-key-not-real-0000"); + vi.stubEnv("GEMINI_API_KEY", "gemini-test-key-not-real"); + }); + afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + }); + + it("names the 1.2 anthropic, google and openai columns, including the ids the review listed", () => { + for (const id of ["claude-sonnet-5-5", "claude-haiku-5-5", "claude-opus-5-5", "gemini-3.8-flash", "gemini-3.6-flash", "gemini-3.5-flash-lite", "gpt-6.1-sol", "gpt-6-luna"]) { + expect(SPEC_NATIVE_WIRE_IDS.has(id), id).toBe(true); + } + for (const id of SPEC_NATIVE_WIRE_IDS) expect(id).not.toContain("/"); + }); + + it("every one is accepted by the boundary check and dispatched to its own provider as written", () => { + for (const id of SPEC_NATIVE_WIRE_IDS) { + expect(() => assertKnownModel(resolveModelAlias(id)), id).not.toThrow(); + expect(getModelWithFailover(resolveModelAlias(id)), id).toEqual({ model: id, provider: providerOf(id), isFallback: false }); + } + }); + + it("the Claude ones have an Anthropic wire id, themselves", () => { + for (const id of SPEC_NATIVE_WIRE_IDS) { + if (id.startsWith("claude-")) expect(anthropicWireId(id), id).toBe(id); + } + }); +}); + +// D1 for the resolver-registry ids the catalog cannot dispatch: it has no OpenRouter branch until PR 2. +// Disposition: REFUSED with a reason. Before this change they either threw as "unknown" (paid-looking) or, +// when free-looking, were silently rerouted to the free lane; on main they went to the paid gpt-5.4-mini. +describe("D1: resolver-registry ids the catalog cannot dispatch are refused, never rerouted", () => { + beforeEach(() => { + vi.spyOn(console, "warn").mockImplementation(() => {}); + vi.spyOn(console, "error").mockImplementation(() => {}); + vi.stubEnv("ANTHROPIC_API_KEY", "sk-ant-test-key-not-real"); + vi.stubEnv("OPENAI_API_KEY", "sk-test-key-not-real-0000"); + vi.stubEnv("GEMINI_API_KEY", "gemini-test-key-not-real"); + vi.stubEnv("OPENROUTER_API_KEY", "sk-or-test-key-not-real"); + }); + afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + }); + + // The set the resolver registers on OpenRouter (its ids, its legacy-alias keys and its wire strings), + // less what the catalog already dispatches and the free lane itself. + const derived = (): Set => { + const catalogKnows = (name: string) => { + const id = resolveModelAlias(name); // the boundary resolves catalog aliases first ("kimi" -> kimi-k2.6) + return Object.hasOwn(modelPricing, id) || Object.hasOwn(modelEquivalents, id); + }; + const out = new Set(); + for (const id of new Set([...APPROVED_MODELS, ...Object.keys(LEGACY_ALIASES)])) { + const target = (LEGACY_ALIASES as Record)[id] ?? id; + const spec = (MODEL_SPECS as Record)[target]; + if (spec?.provider !== "openrouter" || target === FREE_LANE_MODEL) continue; + for (const name of [id, target, spec.sdkId]) if (!catalogKnows(name) && name !== FREE_LANE_WIRE_ID) out.add(name); + } + return out; + }; + + it("OPENROUTER_ONLY_REGISTRY_IDS is exactly the registry's OpenRouter ids (no stale entry, none missing)", () => { + const want = derived(); + expect(want.size).toBeGreaterThan(50); + expect([...OPENROUTER_ONLY_REGISTRY_IDS].sort()).toEqual([...want].sort()); + }); + + it("every one is refused with its reason by the boundary check and by dispatch, with every provider configured", () => { + for (const id of OPENROUTER_ONLY_REGISTRY_IDS) { + expect(() => assertKnownModel(id), id).toThrow(UnknownModelError); + expect(() => getModelWithFailover(id), id).toThrow(/OpenRouter branch/); + } + }); + + it("the named examples are among them, free-looking ones included (no silent reroute to nemotron)", () => { + for (const id of ["z-ai/glm-4.7", "z-ai/glm-4.7-flash", "qwen3-coder", "llama-3.3-70b", "laguna-s-2.1-free", "poolside/laguna-s-2.1:free", "poolside/laguna-xs-2.1:free"]) { + expect(OPENROUTER_ONLY_REGISTRY_IDS.has(id), id).toBe(true); + expect(() => getModelWithFailover(id), id).toThrow(UnknownModelError); + } + }); + + it("the free lane itself is not refused, and an id the registry does not know still takes the free lane (logged)", () => { + expect(getModelWithFailover(FREE_LANE_MODEL)).toEqual({ model: FREE_LANE_WIRE_ID, provider: "openrouter", isFallback: false }); + expect(getModelWithFailover("brand-new-model:free")).toEqual({ model: FREE_LANE_WIRE_ID, provider: "openrouter", isFallback: true }); + expect(console.warn).toHaveBeenCalledWith(expect.stringContaining("brand-new-model:free")); + }); + + // Not wire ids: main sent them to api.anthropic.com, which 404s on a dotted Claude id. They stay refused. + it("dotted Claude ids that were never wire ids stay refused", () => { + for (const id of ["claude-opus-4.6", "claude-opus-4.5", "claude-sonnet-4.5"]) { + expect(() => getModelWithFailover(id), id).toThrow(UnknownModelError); + } + }); +}); diff --git a/shared/llm/modelCatalog.ts b/shared/llm/modelCatalog.ts index b7bf60b7e..dd95a13b7 100644 --- a/shared/llm/modelCatalog.ts +++ b/shared/llm/modelCatalog.ts @@ -640,6 +640,61 @@ export const LEGACY_PASSTHROUGH_IDS: ReadonlySet = new Set([ "gemini-3.1-flash-lite-preview-lite", ]); +/** + * The ids SPEC-v9 section 1.2 lists as their own wire string on the `anthropic`, `google` and + * `openai` transports. Each is already what the provider accepts, so `main` sent every one + * unchanged (it routed on the `claude-` / `gemini-` / `gpt-` prefix); PR 1 moves no ids + * (A.0.4) and Homen D1 makes them known. They are dispatched as written. The resolver registers + * them in PR 2; `anthropicWireIds.test.ts` iterates this set through every public boundary. + */ +export const SPEC_NATIVE_WIRE_IDS: ReadonlySet = new Set([ + "claude-haiku-5-5", + "claude-sonnet-5-5", + "claude-opus-5-5", + "claude-fable-5-1", + "gemini-3.8-flash", + "gemini-3.6-flash", + "gemini-3.5-flash-lite", + "gemini-3.1-pro-preview", + "gemini-3.8-flash-tts", + "gemini-3.8-live", + "gpt-6.1-sol", + "gpt-6-luna", + "gpt-5.4-mini", +]); + +/** + * Resolver-registry ids (`APPROVED_MODELS`, `LEGACY_ALIASES`, and their wire strings) whose + * provider is OpenRouter and that this catalog cannot dispatch: it has no OpenRouter branch + * (PR 2 adds it). Homen D1: a known id is never silently rerouted, so these are REFUSED with this + * reason instead of falling to the paid default (`main`: gpt-5.4-mini) or to the free lane + * (head, for the free-looking ones). `anthropicWireIds.test.ts` derives the same set from the + * resolver and pins this one to it in both directions. + */ +export const OPENROUTER_ONLY_REGISTRY_IDS: ReadonlySet = new Set([ + "arcee-ai/trinity-large-preview:free", "arcee-ai/trinity-mini:free", "chimera", + "cognitivecomputations/dolphin-mistral-24b-venice-edition:free", "deepseek-chimera", + "deepseek/deepseek-r1-0528:free", "deepseek/deepseek-r1:free", "deepseek/deepseek-v3.2", + "deepseek/deepseek-v3.2-speciale", "devstral", "devstral-2", "devstral-2-free", "dolphin-mistral", "gemma", + "gemma-3-27b", "glm", "glm-4.5-air", "google/gemma-3-27b-it:free", "gpt-oss", "gpt-oss-120b", "gpt-oss-20b", + "grok", "grok-4-fast", "grok-4-fast-free", "kat-coder", "kat-coder-pro", "kat-coder-pro-free", + "kwaipilot/kat-coder-pro:free", "laguna-s-2.1", "laguna-s-2.1-free", "laguna-xs-2.1", "laguna-xs-2.1-free", + "llama-3.3", "llama-3.3-70b", "llama-4", "llama-4-maverick", "llama-4-maverick-free", "llama-4-scout", + "llama-4-scout-free", "meta-llama/llama-3.3-70b-instruct:free", "meta-llama/llama-4-maverick:free", + "meta-llama/llama-4-scout:free", "mimo", "mimo-v2", "mimo-v2-flash-free", "mistral-small", + "mistral-small-3.1", "mistralai/devstral-2:free", "mistralai/mistral-large-2411", + "mistralai/mistral-small-3.1-24b-instruct:free", "nemotron", "nemotron-3-nano", "nemotron-70b", + "nemotron-free", "nemotron-nano-12b", "nvidia/llama-3.1-nemotron-70b-instruct:free", + "nvidia/nemotron-3-nano-30b-a3b:free", "nvidia/nemotron-nano-12b-v2-vl:free", "openai/gpt-oss-120b:free", + "openai/gpt-oss-20b:free", "poolside/laguna-s-2.1", "poolside/laguna-s-2.1:free", "poolside/laguna-xs-2.1", + "poolside/laguna-xs-2.1:free", "qwen/qwen3-235b-a22b", "qwen/qwen3-coder:free", + "qwen/qwen3-next-80b-a3b-instruct:free", "qwen3-coder", "qwen3-next", "step-3.5-flash", + "stepfun/step-3.5-flash:free", "tngtech/deepseek-r1t2-chimera:free", "trinity", "trinity-large", + "trinity-mini", "venice-dolphin", "x-ai/grok-4-fast:free", "xiaomi/mimo-v2-flash", + "xiaomi/mimo-v2-flash:free", "z-ai/glm-4.5-air:free", "z-ai/glm-4.7", "z-ai/glm-4.7-20251222", + "z-ai/glm-4.7-flash", "z-ai/glm-4.7-flash-20260119", "zhipu/glm-4.5-air:free", +]); + /** * The Anthropic wire id for a catalog alias (or short alias such as "claude"). * A string that already is a wire id is returned unchanged, dated snapshots in @@ -651,7 +706,7 @@ export function anthropicWireId(modelInput: string): string | null { const fromAlias = ownEntry(ANTHROPIC_WIRE_IDS, resolved); if (fromAlias) return fromAlias; if (Object.values(ANTHROPIC_WIRE_IDS).includes(resolved)) return resolved; - return resolved.startsWith("claude-") && LEGACY_PASSTHROUGH_IDS.has(resolved) ? resolved : null; + return resolved.startsWith("claude-") && (LEGACY_PASSTHROUGH_IDS.has(resolved) || SPEC_NATIVE_WIRE_IDS.has(resolved)) ? resolved : null; } /** @@ -958,6 +1013,7 @@ function isKnownCatalogModel(model: string): boolean { ownEntry(modelPricing, model) != null || ownEntry(modelEquivalents, model) != null || LEGACY_PASSTHROUGH_IDS.has(model) || + SPEC_NATIVE_WIRE_IDS.has(model) || anthropicWireId(model) !== null ); } @@ -969,6 +1025,9 @@ function isKnownCatalogModel(model: string): boolean { * environment: whether a provider is configured is dispatch's concern, not validation's. */ export function assertKnownModel(model: string): void { + if (OPENROUTER_ONLY_REGISTRY_IDS.has(model)) { + throw new UnknownModelError(model, "a registered OpenRouter id this catalog cannot dispatch until PR 2 adds the OpenRouter branch; it is refused, never rerouted to another model"); + } if (!isKnownCatalogModel(model) && !isFreeLookingModel(model)) { throw new UnknownModelError(model, "an unrecognised id never fails over to a paid default and is never sent to a provider"); } @@ -998,6 +1057,7 @@ export function getModelWithFailover(preferredModel: string): { if (!known && isFreeLookingModel(preferredModel)) { // The wire id, not the alias: the caller sends `model` to the provider as is. + console.warn(`[getModelWithFailover] unrecognised free-looking id "${preferredModel}" -> the verified free lane ${FREE_LANE_WIRE_ID}`); return { model: FREE_LANE_WIRE_ID, provider: "openrouter", isFallback: true }; } if (!provider) throw new UnknownModelError(preferredModel, "no provider for this id"); diff --git a/shared/search/triSearch.test.ts b/shared/search/triSearch.test.ts index 4bae16e62..a79f8c6df 100644 --- a/shared/search/triSearch.test.ts +++ b/shared/search/triSearch.test.ts @@ -217,6 +217,8 @@ describe("rerankWithGemini - request shape (Gemini 3.x)", () => { expect(thinking).toEqual({ thinkingLevel: "low" }); expect(thinking).not.toHaveProperty("thinkingBudget"); expectNoSampling(bodies[0]); + // Gemini 3.x rejects a model-turn prefill (400, SPEC-v9 1.3): the last content is the user's. + expect(bodies[0].contents.at(-1).role).not.toBe("model"); }); it("condenseQuery sends the same thinking shape to the same model (thinkingBudget is removed on 3.6+)", async () => { @@ -237,5 +239,6 @@ describe("rerankWithGemini - request shape (Gemini 3.x)", () => { expect(thinking).not.toHaveProperty("thinkingBudget"); expect(JSON.stringify(bodies[0])).not.toContain("thinkingBudget"); expectNoSampling(bodies[0]); + expect(bodies[0].contents.at(-1).role).not.toBe("model"); }); }); diff --git a/workers/node/pipeline/extendedThinkingRunner.test.ts b/workers/node/pipeline/extendedThinkingRunner.test.ts index 3b5938017..cfb129ca7 100644 --- a/workers/node/pipeline/extendedThinkingRunner.test.ts +++ b/workers/node/pipeline/extendedThinkingRunner.test.ts @@ -291,6 +291,30 @@ describe("buildCheckpointRequestBody - Anthropic request shape (SPEC-v9 A.0.6)", expect(body.system).toBe("sys"); }); + // Sonnet/Opus 5.5 reject a forced tool_choice and any prefill (400, SPEC-v9 1.3), and Haiku 5.5 drops + // thinking when one is forced. The checkpoint is a plain user turn for every id it can send. + it.each([ + "claude-sonnet-4.6", "claude-opus-4.7", "claude-haiku-4-5", "claude-haiku-5-5", "claude-sonnet-5-5", "claude-opus-5-5", "claude-fable-5-1", + ])("%s: no forced tool_choice, no tools, and the last message is the user's (no prefill)", (modelName) => { + const body = buildCheckpointRequestBody({ modelName, prompt, remainingThinkingTokens: 40_000 }) as any; + expect(body).not.toHaveProperty("tool_choice"); + expect(body).not.toHaveProperty("tools"); + expect((body.messages as Array<{ role: string }>).at(-1)?.role).toBe("user"); + }); + + it.each(["claude-haiku-5-5", "claude-sonnet-5-5", "claude-opus-5-5", "claude-fable-5-1"])( + "%s (SPEC-v9 1.2 wire id, sent as written) takes adaptive thinking with effort, never budget_tokens", + (modelName) => { + const body = buildCheckpointRequestBody({ modelName, prompt, remainingThinkingTokens: 40_000 }) as any; + expect(body.model).toBe(modelName); + expect(body.thinking).toEqual({ type: "adaptive" }); + expect(body.output_config).toEqual({ effort: "medium" }); + for (const key of ["temperature", "top_p", "top_k"]) expect(body).not.toHaveProperty(key); + // and under the 1,024 allowance the adaptive shape is unchanged: there is no manual budget to drop + expect((buildCheckpointRequestBody({ modelName, prompt, remainingThinkingTokens: 0 }) as any).thinking).toEqual({ type: "adaptive" }); + }, + ); + it("caps total output per checkpoint: max_tokens = answer cap + min(allowance, remaining)", () => { // This bounds thinking PLUS answer. Adaptive thinking has no separate thinking cap, so with 2,500 // tokens left a checkpoint may still think for up to 6,500 (the 4,000 answer room is not reserved). diff --git a/workers/node/pipeline/extendedThinkingRunner.ts b/workers/node/pipeline/extendedThinkingRunner.ts index 71596091d..2841a32d4 100644 --- a/workers/node/pipeline/extendedThinkingRunner.ts +++ b/workers/node/pipeline/extendedThinkingRunner.ts @@ -99,9 +99,17 @@ export type ParseOutcome = * Ids that take adaptive thinking with `output_config.effort`. Every other id keeps the manual * `{type:"enabled", budget_tokens}` shape main sent: Haiku 4.5 supports only manual thinking and * 400s on `effort` (migration guide, "Choosing an effort level"), and adaptive thinking starts at - * the 4.6 generation. Haiku 4.5 and the dated 4.5 snapshots move with their ids in PR 2. + * the 4.6 generation; the 5.5 ids (SPEC-v9 1.3: `budget_tokens` 400s on them) take adaptive. + * Haiku 4.5 and the dated 4.5 snapshots move with their ids in PR 2. */ -const ADAPTIVE_THINKING_WIRE_IDS: ReadonlySet = new Set(["claude-opus-4-7", "claude-sonnet-4-6"]); +const ADAPTIVE_THINKING_WIRE_IDS: ReadonlySet = new Set([ + "claude-opus-4-7", + "claude-sonnet-4-6", + "claude-haiku-5-5", + "claude-sonnet-5-5", + "claude-opus-5-5", + "claude-fable-5-1", +]); /** Smallest `budget_tokens` the Messages API accepts. */ const MIN_MANUAL_BUDGET_TOKENS = 1_024; From af017607caa6e6d87597a5d91c1db9e99081d088 Mon Sep 17 00:00:00 2001 From: homen Date: Fri, 16 Oct 2026 14:04:11 -0700 Subject: [PATCH 8/8] fix(models): close the review round 5 findings against ad001caf (PR 1, #662) D1: the 15 SPEC-v9 1.2 live ':free' ids are known. getLanguageModelSafe sends an unregistered one unchanged on OpenRouter; the alias-returning resolver entry points and the catalog (no OpenRouter branch until PR 2) refuse them with a reason instead of swapping in nemotron. The 1.2 native ids the resolver does not register yet are refused with RefusedModelError (an UnknownModelError subclass) that says why. A.0.6: the Gemini thinking setting follows the selected model (gemini-3* thinkingLevel, others keep thinkingBudget:0) in triSearch rerank/condenseQuery and the video judge. A.0.10: anonymous FastAgent requests are validated before the anonymous override (normalizeFastAgentModel); createSchedule, requestLlmJudge and startCoordinatorWorkflow reject unknown models at the boundary. run-comprehensive-eval aborts only on --models values that do not resolve; identityViolations still rejects alias rows as receipts. Tests: runtime-smoke list 48 files / 744 tests pass (new queuedModelBoundary.test.ts added to it); packet contracts 20/20; tsc root and backend/convex exit 0; ten mutations each fail their tests. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/ci.yml | 1 + .../actions/coordinatorWorkflowActions.ts | 2 + .../domains/agents/fastAgentPanelStreaming.ts | 20 +-- .../domains/agents/mcp_tools/models/index.ts | 2 + .../mcp_tools/models/modelResolver.test.ts | 140 ++++++++++++++++++ .../agents/mcp_tools/models/modelResolver.ts | 64 +++++++- .../pipelines/pipelineModelBoundary.test.ts | 23 +++ .../domains/pipelines/pipelineSchedule.ts | 5 +- .../domains/product/diligenceLlmJudgeRuns.ts | 4 +- .../product/queuedModelBoundary.test.ts | 103 +++++++++++++ .../__tests__/comprehensiveEvalModels.test.ts | 24 ++- ...eScratchnodeChatVideo.requestShape.test.ts | 13 ++ scripts/run-comprehensive-eval.ts | 17 ++- scripts/ui/judgeScratchnodeChatVideo.mjs | 7 +- shared/llm/anthropicWireIds.test.ts | 36 +++++ shared/llm/modelCatalog.ts | 51 +++++++ shared/search/triSearch.test.ts | 48 +++++- shared/search/triSearch.ts | 17 ++- 18 files changed, 537 insertions(+), 40 deletions(-) create mode 100644 backend/convex/domains/product/queuedModelBoundary.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b3b32fa88..2ce2c8bd0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -101,6 +101,7 @@ jobs: backend/convex/domains/mcp/mcpExecutionTraceEndpoints.integration.test.ts backend/convex/domains/agents/mcp_tools/models/modelResolver.test.ts backend/convex/domains/pipelines/pipelineModelBoundary.test.ts + backend/convex/domains/product/queuedModelBoundary.test.ts backend/convex/events.requestShape.test.ts shared/llm/anthropicWireIds.test.ts shared/search/triSearch.test.ts diff --git a/backend/convex/actions/coordinatorWorkflowActions.ts b/backend/convex/actions/coordinatorWorkflowActions.ts index 5362358c7..0690abeb9 100644 --- a/backend/convex/actions/coordinatorWorkflowActions.ts +++ b/backend/convex/actions/coordinatorWorkflowActions.ts @@ -4,6 +4,7 @@ import { action } from "../_generated/server"; import { v } from "convex/values"; import { internal } from "../_generated/api"; import type { Id } from "../_generated/dataModel"; +import { assertKnownModelInput } from "../domains/agents/mcp_tools/models"; /** * Enqueue a durable coordinator workflow run. @@ -18,6 +19,7 @@ export const startCoordinatorWorkflow = action({ runId: v.string(), }), handler: async (ctx, args): Promise<{ runId: string }> => { + assertKnownModelInput(args.model); // fail closed before anything is queued (SPEC-v9 A.0.10) const runId = await ctx.scheduler.runAfter( 0, diff --git a/backend/convex/domains/agents/fastAgentPanelStreaming.ts b/backend/convex/domains/agents/fastAgentPanelStreaming.ts index a289b7737..865e10a61 100644 --- a/backend/convex/domains/agents/fastAgentPanelStreaming.ts +++ b/backend/convex/domains/agents/fastAgentPanelStreaming.ts @@ -88,10 +88,7 @@ import { canReadAgentRunPresentation, projectAgentRunPresentation, } from "./agentRunPresentation"; -import { - FAST_AGENT_SIGN_IN_BENEFIT_COPY, - resolveFastAgentRequestedModel, -} from "../../../../shared/llm/fastAgentRuntimeContract"; +import { FAST_AGENT_SIGN_IN_BENEFIT_COPY } from "../../../../shared/llm/fastAgentRuntimeContract"; // Import tools import { fusionSearch, quickSearch } from "../../tools/search"; @@ -445,6 +442,7 @@ import { getLanguageModelSafe, normalizeModelInput, normalizeNodeBenchRuntimeModel, + normalizeFastAgentModel, isFreeModel, DEFAULT_MODEL, isModelAvailable, @@ -2228,11 +2226,8 @@ export const createThread = action({ throw new Error("Anonymous users must provide a session ID"); } - const modelName = normalizeNodeBenchRuntimeModel( - resolveFastAgentRequestedModel({ - isAnonymous, - requestedModel: args.model, - }), + const modelName = normalizeFastAgentModel( + { isAnonymous, requestedModel: args.model }, isAnonymous ? "executor" : "advisor", ); @@ -2968,11 +2963,8 @@ export const initiateAsyncStreaming = mutation({ const runtimeProfile = isAnonymous ? "executor" : args.useCoordinator !== false ? "advisor" : "executor"; - const modelName = normalizeNodeBenchRuntimeModel( - resolveFastAgentRequestedModel({ - isAnonymous, - requestedModel: args.model, - }), + const modelName = normalizeFastAgentModel( + { isAnonymous, requestedModel: args.model }, runtimeProfile, ); const chatAgent = createChatAgent(modelName); diff --git a/backend/convex/domains/agents/mcp_tools/models/index.ts b/backend/convex/domains/agents/mcp_tools/models/index.ts index 61d461c5f..b4cf439b2 100644 --- a/backend/convex/domains/agents/mcp_tools/models/index.ts +++ b/backend/convex/domains/agents/mcp_tools/models/index.ts @@ -32,6 +32,7 @@ export { // Fail-closed resolution (SPEC-v9 A.0.10) UnknownModelError, type UnknownModelOptions, + assertKnownModelInput, assertKnownPipelineModel, getPaidFallbackCount, @@ -51,6 +52,7 @@ export { // Normalization normalizeModelInput, normalizeNodeBenchRuntimeModel, + normalizeFastAgentModel, isApprovedModel, // Free model helpers diff --git a/backend/convex/domains/agents/mcp_tools/models/modelResolver.test.ts b/backend/convex/domains/agents/mcp_tools/models/modelResolver.test.ts index b0881eda0..11975acd2 100644 --- a/backend/convex/domains/agents/mcp_tools/models/modelResolver.test.ts +++ b/backend/convex/domains/agents/mcp_tools/models/modelResolver.test.ts @@ -16,11 +16,14 @@ import { getPaidFallbackCount, isApprovedModel, isFreeModel, + normalizeFastAgentModel, normalizeModelInput, normalizeNodeBenchRuntimeModel, resolveModelAlias, resolvePipelineModelSelection, } from "./modelResolver"; +import { ANONYMOUS_FAST_AGENT_MODEL_ID } from "../../../../../../shared/llm/fastAgentRuntimeContract"; +import { RefusedModelError, SPEC_LIVE_FREE_WIRE_IDS, SPEC_NATIVE_WIRE_IDS } from "../../../../../../shared/llm/modelCatalog"; describe("pipeline model route resolution", () => { it("maps NodeBench auto routes to approved runtime aliases", () => { @@ -373,3 +376,140 @@ describe("Object.prototype keys are not models", () => { expect(() => assertKnownPipelineModel(key)).toThrow(UnknownModelError); }); }); + +// Persona: a safety engineer asks for the content-safety classifier by its OpenRouter id. It is one of the +// 15 live `:free` ids SPEC-v9 section 1.2 lists, so Homen D1 makes it known: it must reach OpenRouter as +// that id, never come back as the general chat model nvidia/nemotron-3-super (PR #662 review round 6, P2). +describe("D1: the SPEC-v9 1.2 live :free ids are known and are never rerouted", () => { + // Typed out here, not read from SPEC_LIVE_FREE_WIRE_IDS, so the set cannot drift from the spec unseen. + const SPEC_1_2_FREE = [ + "apodex/apodex-1.1-mini:free", "dots-studio/dots-3-note-preview:free", "liquid/lfm-2.5-2.6b:free", + "nvidia/nemotron-3.5-lightning:free", "thinkingmachines/inkling-small:free", "poolside/laguna-s-2.1:free", + "thinkingmachines/inkling:free", "poolside/laguna-xs-2.1:free", "cohere/north-mini-code:free", + "nvidia/nemotron-3.5-content-safety:free", "nvidia/nemotron-3-ultra-550b-a55b:free", + "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free", "google/gemma-4-26b-a4b-it:free", + "google/gemma-4-31b-it:free", "nvidia/nemotron-3-super-120b-a12b:free", + ]; + beforeEach(() => { + vi.stubEnv("OPENROUTER_API_KEY", "test-key-not-real"); + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "warn").mockImplementation(() => {}); + }); + afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + }); + + it("the set is exactly the 15 ids of the spec", () => { + expect([...SPEC_LIVE_FREE_WIRE_IDS].sort()).toEqual([...SPEC_1_2_FREE].sort()); + }); + + it("getLanguageModelSafe sends every one unchanged on OpenRouter", () => { + for (const id of SPEC_1_2_FREE) { + const model = getLanguageModelSafe(id) as { modelId: string; provider?: string }; + expect(model.modelId, id).toBe(id); + expect(String(model.provider ?? "")).toMatch(/openai|chat/); + } + }); + + it("is case and whitespace tolerant, like the alias lookup", () => { + expect((getLanguageModelSafe(" Google/Gemma-4-31B-IT:free ") as { modelId: string }).modelId).toBe("google/gemma-4-31b-it:free"); + }); + + it("an entry point that returns an alias refuses the unregistered ones with a reason; it never returns another model", () => { + for (const id of SPEC_1_2_FREE) { + const registered = resolveModelAlias(id); + for (const call of [ + () => normalizeModelInput(id), + () => resolvePipelineModelSelection(id).resolvedModelId, + () => resolvePipelineModelSelection(`openrouter:${id}`).resolvedModelId, + ]) { + if (registered) { + expect(call(), id).toBe(registered); // nemotron-super and laguna s/xs are registered aliases + } else { + expect(call, id).toThrow(RefusedModelError); + } + } + if (registered) expect(() => assertKnownPipelineModel(id), id).not.toThrow(); + else expect(() => assertKnownPipelineModel(id), id).toThrow(RefusedModelError); + } + expect(() => normalizeModelInput("nvidia/nemotron-3.5-content-safety:free")).toThrow(/known id, refused/); + }); + + it("an id that is merely free-looking still takes the free lane", () => { + expect((getLanguageModelSafe("nvidia/nemotron-9-invented:free") as { modelId: string }).modelId).toBe(FREE_LANE_WIRE_ID); + }); +}); + +// The review's sub-note: the native ids D1 calls known were reported as "Unknown model". +describe("D1: SPEC-v9 1.2 native ids the resolver registers in PR 2 are refused with the reason, not called unknown", () => { + beforeEach(() => { + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "warn").mockImplementation(() => {}); + }); + afterEach(() => vi.restoreAllMocks()); + + it("says why, and is still an UnknownModelError for every boundary that catches that class", () => { + for (const id of ["claude-sonnet-5-5", "gemini-3.8-flash"]) { + expect(SPEC_NATIVE_WIRE_IDS.has(id)).toBe(true); + for (const call of [() => getLanguageModelSafe(id), () => normalizeModelInput(id), () => resolvePipelineModelSelection(id)]) { + let error: unknown; + try { call(); } catch (e) { error = e; } + expect(error, id).toBeInstanceOf(RefusedModelError); + expect(error, id).toBeInstanceOf(UnknownModelError); + expect((error as Error).message, id).toMatch(/registers in PR 2/); + expect((error as Error).message, id).not.toMatch(/^Unknown model/); + } + } + }); +}); + +// Persona: a visitor who has not signed in opens the FastAgent panel and a script (or a stale client) sends +// a model field. The anonymous lane is bounded, but what was ASKED for is still validated first: an unknown +// id throws at the public boundary, and a free request never becomes the paid bounded model +// (SPEC-v9 A.0.10; PR #662 review round 6, P2). createThread and initiateAsyncStreaming both route here. +describe("FastAgent anonymous callers: the requested model is validated before the anonymous override", () => { + beforeEach(() => { + vi.spyOn(console, "warn").mockImplementation(() => {}); + }); + afterEach(() => vi.restoreAllMocks()); + const anon = (requestedModel?: string) => normalizeFastAgentModel({ isAnonymous: true, requestedModel }, "executor"); + + it("keeps a free request on the free lane, never the paid bounded model", () => { + for (const id of ["auto-free", "invented-free", "x:free", FREE_LANE_MODEL, "qwen3-coder-free"]) { + const resolved = anon(id); + expect(isFreeModel(resolved), id).toBe(true); + expect(resolved, id).not.toBe(ANONYMOUS_FAST_AGENT_MODEL_ID); + } + expect(anon("auto-free")).toBe(FREE_LANE_MODEL); + }); + + it("throws for an unknown id, including Object.prototype keys, instead of accepting it silently", () => { + for (const id of ["constructor", "toString", "gpt-not-real", "freeloader-pro"]) { + expect(() => anon(id), id).toThrow(UnknownModelError); + } + }); + + it("refuses a known id the resolver has not registered yet, with the reason", () => { + expect(() => anon("claude-sonnet-5-5")).toThrow(RefusedModelError); + expect(() => anon("google/gemma-4-31b-it:free")).toThrow(RefusedModelError); + }); + + it("keeps the anonymous default when no model is passed, and the bounded lane for a known paid request", () => { + expect(anon(undefined)).toBe(ANONYMOUS_FAST_AGENT_MODEL_ID); + expect(anon("")).toBe(ANONYMOUS_FAST_AGENT_MODEL_ID); + expect(anon("claude-opus-4.7")).toBe(ANONYMOUS_FAST_AGENT_MODEL_ID); + expect(anon("gpt-5.4-mini")).toBe(ANONYMOUS_FAST_AGENT_MODEL_ID); + }); + + it("does not change a signed-in caller", () => { + for (const id of [undefined, "auto-free", "gpt-5.4-mini", "claude-opus-4.7", "kimi-k2.6"]) { + for (const profile of ["advisor", "executor"] as const) { + expect(normalizeFastAgentModel({ isAnonymous: false, requestedModel: id }, profile), `${profile}/${id}`).toBe( + normalizeNodeBenchRuntimeModel(id, profile), + ); + } + } + expect(() => normalizeFastAgentModel({ isAnonymous: false, requestedModel: "constructor" }, "advisor")).toThrow(UnknownModelError); + }); +}); diff --git a/backend/convex/domains/agents/mcp_tools/models/modelResolver.ts b/backend/convex/domains/agents/mcp_tools/models/modelResolver.ts index 2e5e86b91..9921e64cc 100644 --- a/backend/convex/domains/agents/mcp_tools/models/modelResolver.ts +++ b/backend/convex/domains/agents/mcp_tools/models/modelResolver.ts @@ -19,11 +19,15 @@ import { createOpenAI, openai } from "@ai-sdk/openai"; import { anthropic } from "@ai-sdk/anthropic"; import { createGoogleGenerativeAI } from "@ai-sdk/google"; import type { LanguageModel } from "ai"; +import { resolveFastAgentRequestedModel } from "../../../../../../shared/llm/fastAgentRuntimeContract"; import { FREE_LANE_CATALOG_ID, FREE_LANE_WIRE_ID, + RefusedModelError, + SPEC_NATIVE_WIRE_IDS, UnknownModelError, ownEntry, + specLiveFreeWireId, } from "../../../../../../shared/llm/modelCatalog"; export { UnknownModelError }; @@ -286,6 +290,23 @@ export function normalizeNodeBenchRuntimeModel( return fallback; } +/** + * FastAgent public entry points (`createThread`, `initiateAsyncStreaming`). An anonymous caller is + * pinned to the bounded lane, but what it ASKED for is validated first (SPEC-v9 A.0.10): an unknown + * id throws, and a free request stays on its free model instead of being swapped for the paid lane. + * A known paid request still takes the bounded lane, as before. No model passed: the bounded lane. + */ +export function normalizeFastAgentModel( + { isAnonymous, requestedModel }: { isAnonymous: boolean; requestedModel?: string }, + profile: NodeBenchRuntimeProfile, +): ApprovedModel { + if (isAnonymous && requestedModel) { + const requested = normalizeNodeBenchRuntimeModel(requestedModel, profile); + if (isFreeModel(requested)) return requested; + } + return normalizeNodeBenchRuntimeModel(resolveFastAgentRequestedModel({ isAnonymous, requestedModel }), profile); +} + export type NodeBenchModelRouteTier = "balanced" | "free" | "frontier" | "fast" | "small"; export interface PipelineModelSelectionResolution { @@ -388,6 +409,8 @@ export function resolvePipelineModelSelection( }; } + refuseKnownUnregisteredModel(requestedModelId); + refuseKnownUnregisteredModel(stripped); // Fail closed (SPEC-v9 A.0.10): a free-looking unknown id stays in the free // lane; any other unknown id throws. It never becomes DEFAULT_MODEL, because // the pipelines domain holds a launch counter, not a financial reservation. @@ -932,7 +955,7 @@ export const LEGACY_ALIASES: Record = { /** * Build a LanguageModel instance from a ModelSpec */ -function buildLanguageModel(spec: ModelSpec): LanguageModel { +function buildLanguageModel(spec: { alias: string; provider: Provider; sdkId: string }): LanguageModel { switch (spec.provider) { case "openai": return openai(spec.sdkId); @@ -1009,6 +1032,24 @@ export function getPaidFallbackCount(): number { return paidFallbackCount; } +/** + * Homen D1: an id SPEC-v9 section 1.2 lists is KNOWN, so it is never reported as unknown and never + * rerouted (to the free lane or to a paid default). The ones this resolver has not registered are + * the live `:free` ids and the native Claude / Gemini / GPT ids; PR 2 registers them. Until then an + * ApprovedModel-returning entry point has nothing to return for them, so it refuses with the reason. + * (`getLanguageModelSafe` returns a model, so it serves the unregistered `:free` ids itself.) + * Call after the alias lookup missed. + */ +function refuseKnownUnregisteredModel(input: string): void { + const normalized = input.trim().toLowerCase(); + if (specLiveFreeWireId(normalized) !== null) { + throw new RefusedModelError(input, "a live free OpenRouter id of SPEC-v9 section 1.2 that this entry point registers in PR 2; it is never rerouted to the free lane"); + } + if (SPEC_NATIVE_WIRE_IDS.has(normalized)) { + throw new RefusedModelError(input, "a SPEC-v9 section 1.2 id that this resolver registers in PR 2 (PR 1 moves no ids); it is never rerouted to another model"); + } +} + /** * Resolve untrusted input to an approved alias or fail closed (SPEC-v9 A.0.10): * - empty input -> DEFAULT_MODEL @@ -1025,6 +1066,7 @@ function resolveOrFailClosed( if (!input) return DEFAULT_MODEL; const resolved = resolveModelAlias(input); if (resolved) return resolved; + refuseKnownUnregisteredModel(input); if (isFreeLookingModelInput(input)) { console.warn(`[modelResolver] ${caller}: unknown free-looking model "${input}" -> ${FREE_LANE_MODEL}`); return FREE_LANE_MODEL; @@ -1048,6 +1090,13 @@ export function getLanguageModelSafe( input: string | undefined | null, opts?: UnknownModelOptions, ): LanguageModel { + // A live `:free` id (SPEC-v9 1.2) the registry has no alias for is known: send it unchanged on OpenRouter. + const liveFree = input ? specLiveFreeWireId(input) : null; + if (input && liveFree !== null && resolveModelAlias(input) === null) { + const spec = { alias: null, provider: "openrouter" as const, sdkId: liveFree }; + logModelResolution(input, spec, "getLanguageModelSafe"); + return buildLanguageModel({ ...spec, alias: liveFree }); + } const resolved = resolveOrFailClosed(input, opts, "getLanguageModelSafe"); const substituted = Boolean(input) && resolveModelAlias(input as string) === null; const spec = MODEL_SPECS[resolved]; @@ -1055,6 +1104,17 @@ export function getLanguageModelSafe( return buildLanguageModel(spec); } +/** + * Public-boundary check for a model id a caller supplied (SPEC-v9 A.0.10): throws exactly when + * `getLanguageModelSafe(input)` would throw, builds no model and needs no API key. Use it where an + * id is queued or stored and only resolved later, so the caller hears about a bad id now. + */ +export function assertKnownModelInput(input: string | undefined | null): void { + if (!input) return; + if (specLiveFreeWireId(input) !== null && resolveModelAlias(input) === null) return; // served unchanged + resolveOrFailClosed(input, undefined, "assertKnownModelInput"); +} + /** * Get the ModelSpec for an approved model */ @@ -1090,7 +1150,7 @@ export interface ModelResolutionEvent { function logModelResolution( requestedAlias: string, - spec: ModelSpec, + spec: { alias: ApprovedModel | null; provider: Provider; sdkId: string }, caller: string, fallback?: "unknown_input", ): void { diff --git a/backend/convex/domains/pipelines/pipelineModelBoundary.test.ts b/backend/convex/domains/pipelines/pipelineModelBoundary.test.ts index a6eaa385b..ec3b03089 100644 --- a/backend/convex/domains/pipelines/pipelineModelBoundary.test.ts +++ b/backend/convex/domains/pipelines/pipelineModelBoundary.test.ts @@ -153,6 +153,29 @@ describe.skipIf(!convexTestAvailable)("startComposedPipelineRun model boundary", }, 20_000); }); +// Round 6: createSchedule stored an enabled hourly schedule for "constructor". The row counted toward the +// owner's schedule limit and startPipelineRunInternal then rejected it on every run. +describe.skipIf(!convexTestAvailable)("createSchedule model boundary", () => { + const schedule = { pipelineKind: "research", spec: "Weekly research", cadence: "hourly" }; + const scheduleCount = (t: any) => t.run(async (ctx: any) => (await ctx.db.query("scheduledPipelineRuns").collect()).length); + + it("rejects an unknown modelId before inserting, and still accepts an omitted or known one", async () => { + const t = convexTest(schema, convexModules); + const userId = await t.run(async (ctx: any) => ctx.db.insert("users", { email: "schedule@example.com" })); + const user = t.withIdentity({ subject: String(userId) }); + + for (const modelId of ["constructor", "gpt-9-ultra", "openai:gpt-9"]) { + await expect(user.mutation(pipelines.pipelineSchedule.createSchedule, { ...schedule, modelId }), modelId).rejects.toThrow(/Unknown model/); + } + expect(await scheduleCount(t)).toBe(0); + + await user.mutation(pipelines.pipelineSchedule.createSchedule, schedule); + await user.mutation(pipelines.pipelineSchedule.createSchedule, { ...schedule, modelId: "gpt-5.4-mini" }); + await user.mutation(pipelines.pipelineSchedule.createSchedule, { ...schedule, modelId: "x:free" }); + expect(await scheduleCount(t)).toBe(3); + }, 20_000); +}); + describe("piRuntime.parseModelKey", () => { it("throws for an unknown provider:model instead of returning a raw handle", async () => { // Before: the catch fell through to parseRawModelKey, which forwarded "gpt-9" to pi-ai. diff --git a/backend/convex/domains/pipelines/pipelineSchedule.ts b/backend/convex/domains/pipelines/pipelineSchedule.ts index e689bdeec..a5f1301f3 100644 --- a/backend/convex/domains/pipelines/pipelineSchedule.ts +++ b/backend/convex/domains/pipelines/pipelineSchedule.ts @@ -26,7 +26,7 @@ import { } from "../../_generated/server"; import { internal } from "../../_generated/api"; import type { Id } from "../../_generated/dataModel"; -import { DEFAULT_PIPELINE_MODEL_ROUTE } from "../agents/mcp_tools/models/modelResolver"; +import { DEFAULT_PIPELINE_MODEL_ROUTE, assertKnownPipelineModel } from "../agents/mcp_tools/models/modelResolver"; import { pipelineOwnerMatches, requireAuthenticatedPipelineOwnerKey, @@ -71,6 +71,9 @@ export const createSchedule = mutation({ handler: async (ctx, args) => { const ownerKey = await requireAuthenticatedPipelineOwnerKey(ctx); const launchText = normalizePipelineLaunchText(args); + // Same rule as startPipelineRun: an enabled schedule with an unknown model would count toward the + // owner's limit and be rejected by startPipelineRunInternal on every run (SPEC-v9 A.0.10). + assertKnownPipelineModel(launchText.modelId); const existingSchedules = await ctx.db .query("scheduledPipelineRuns") .withIndex("by_owner_createdAt", (q) => q.eq("ownerKey", ownerKey)) diff --git a/backend/convex/domains/product/diligenceLlmJudgeRuns.ts b/backend/convex/domains/product/diligenceLlmJudgeRuns.ts index 45c8085fc..990137503 100644 --- a/backend/convex/domains/product/diligenceLlmJudgeRuns.ts +++ b/backend/convex/domains/product/diligenceLlmJudgeRuns.ts @@ -33,7 +33,7 @@ import { import { internal } from "../../_generated/api"; import type { Doc } from "../../_generated/dataModel"; import { generateText } from "ai"; -import { getLanguageModelSafe } from "../agents/mcp_tools/models"; +import { assertKnownModelInput, getLanguageModelSafe } from "../agents/mcp_tools/models"; import { JUDGE_PROMPT_VERSION, buildLlmJudgePrompt, @@ -502,6 +502,8 @@ export const requestLlmJudge = mutation({ }, handler: async (ctx, args) => { const identity = await requireAuthenticatedProductIdentity(ctx); + // Fail closed at the boundary: scoreVerdictWithLlm would only reject it later (SPEC-v9 A.0.10). + assertKnownModelInput(args.modelName); // Sanity — verdict must exist. const verdict = await ctx.db.get(args.verdictId); if (!verdict || verdict.ownerKey !== identity.ownerKey) { diff --git a/backend/convex/domains/product/queuedModelBoundary.test.ts b/backend/convex/domains/product/queuedModelBoundary.test.ts new file mode 100644 index 000000000..011238dc0 --- /dev/null +++ b/backend/convex/domains/product/queuedModelBoundary.test.ts @@ -0,0 +1,103 @@ +/// +/** + * Public queue boundaries reject an unknown model id at the call, not later (SPEC-v9 A.0.10). + * + * Persona: a signed-in user (or a stale client) clicks "Score with LLM", or starts a coordinator run, with a + * model id nobody registered ("constructor"). Before this change both calls answered "queued" / a run id and + * the run failed closed minutes later in getLanguageModelSafe: nothing was spent, but the caller was told yes. + */ + +import { describe, expect, it } from "vitest"; + +import schema from "../../schema"; +import { api } from "../../_generated/api"; + +const DIR_SEGMENTS = ["domains", "product"]; +function rerootGlobKey(key: string): string { + const parts = key.replace(/^\.\//, "").split("/"); + const base = [...DIR_SEGMENTS]; + while (parts[0] === "..") { + parts.shift(); + base.pop(); + } + return [...base, ...parts].join("/"); +} + +const convexModules = Object.fromEntries( + Object.entries(import.meta.glob("../../**/*.{ts,js}")).map(([key, loader]) => [rerootGlobKey(key), loader]), +); + +let convexTest: any; +let convexTestAvailable = false; +try { + const mod = await import(/* @vite-ignore */ "convex-test"); + convexTest = mod.convexTest; + convexTestAvailable = typeof convexTest === "function"; +} catch { + convexTestAvailable = false; +} + +const scheduledCount = (t: any) => t.run(async (ctx: any) => (await ctx.db.system.query("_scheduled_functions").collect()).length); + +describe.skipIf(!convexTestAvailable)("requestLlmJudge model boundary", () => { + it("rejects an unknown modelName before scheduling, and still queues an omitted or known one", async () => { + const t = convexTest(schema, convexModules); + const { userId, verdictId } = await t.run(async (ctx: any) => { + const userId = await ctx.db.insert("users", { email: "judge-boundary@example.com" }); + const now = Date.now(); + const telemetryId = await ctx.db.insert("diligenceRunTelemetry", { + entitySlug: "acme", + ownerKey: `user:${String(userId)}`, + blockType: "overview", + scratchpadRunId: "run-1", + version: 1, + overallTier: "pass", + headerText: "h", + status: "done", + startedAt: now, + endedAt: now, + elapsedMs: 0, + }); + const verdictId = await ctx.db.insert("diligenceJudgeVerdicts", { + telemetryId, + entitySlug: "acme", + ownerKey: `user:${String(userId)}`, + blockType: "overview", + scratchpadRunId: "run-1", + verdict: "pass", + passCount: 1, + failCount: 0, + skipCount: 0, + score: 1, + latencyBudgetMs: 1000, + gatesJson: "[]", + judgedAt: Date.now(), + }); + return { userId, verdictId }; + }); + const user = t.withIdentity({ subject: String(userId) }); + const request = api.domains.product.diligenceLlmJudgeRuns.requestLlmJudge; + + for (const modelName of ["constructor", "gpt-9-ultra"]) { + await expect(user.mutation(request, { verdictId, modelName }), modelName).rejects.toThrow(/Unknown model/); + } + expect(await scheduledCount(t)).toBe(0); + + await expect(user.mutation(request, { verdictId })).resolves.toEqual({ status: "queued" }); + await expect(user.mutation(request, { verdictId, modelName: "gpt-5.4-mini" })).resolves.toEqual({ status: "queued" }); + expect(await scheduledCount(t)).toBe(2); + }, 20_000); +}); + +describe.skipIf(!convexTestAvailable)("startCoordinatorWorkflow model boundary", () => { + it("rejects an unknown model before anything is queued", async () => { + const t = convexTest(schema, convexModules); + for (const model of ["constructor", "gpt-9-ultra"]) { + await expect( + t.action(api.actions.coordinatorWorkflowActions.startCoordinatorWorkflow, { threadId: "t1", prompt: "go", model }), + model, + ).rejects.toThrow(/Unknown model/); + } + expect(await scheduledCount(t)).toBe(0); + }, 20_000); +}); diff --git a/scripts/__tests__/comprehensiveEvalModels.test.ts b/scripts/__tests__/comprehensiveEvalModels.test.ts index 15f338e2b..0d00ef0b9 100644 --- a/scripts/__tests__/comprehensiveEvalModels.test.ts +++ b/scripts/__tests__/comprehensiveEvalModels.test.ts @@ -13,20 +13,28 @@ import { describe, expect, it } from "vitest"; -import { identityViolations, nonCanonicalModels, runSingleEpisode } from "../run-comprehensive-eval"; +import { identityViolations, runSingleEpisode, unresolvableModels } from "../run-comprehensive-eval"; -describe("nonCanonicalModels (--models must be an approved id exactly, or the run aborts before spending)", () => { +describe("unresolvableModels (--models aborts only on a value the resolver does not resolve)", () => { it("accepts approved ids", () => { - expect(nonCanonicalModels(["kimi-k2.6", "claude-sonnet-4.6", "gpt-5.4-mini", "nemotron-3-super-free"])).toEqual([]); + expect(unresolvableModels(["kimi-k2.6", "claude-sonnet-4.6", "gpt-5.4-mini", "nemotron-3-super-free"])).toEqual([]); }); - it("flags ids the resolver does not know, including the removed minimax-m2.1", () => { - expect(nonCanonicalModels(["kimi-k2.6", "minimax-m2.1", "claude-opus-9"])).toEqual(["minimax-m2.1", "claude-opus-9"]); + it("flags ids the resolver does not know, including the removed minimax-m2.1 and Object.prototype keys", () => { + expect(unresolvableModels(["kimi-k2.6", "minimax-m2.1", "claude-opus-9", "constructor"])).toEqual(["minimax-m2.1", "claude-opus-9", "constructor"]); }); - it("flags a legacy alias, which resolves to a different id and would disqualify every row after the paid run", () => { - // gpt-4o-mini resolves to gpt-5.4-mini; the previous check (resolveModelAlias(m) === null) accepted it. - expect(nonCanonicalModels(["gpt-4o-mini", "GPT-5.4-MINI"])).toEqual(["gpt-4o-mini", "GPT-5.4-MINI"]); + // Round 6 (SPEC-v9 PR 1: abort only when a value "does not resolve"; Homen D1: CLI overrides are pass-through). + // Round 5 pinned these two as refused because resolveModelAlias(m) !== m. Persona: an operator reruns an old + // command line with --models gpt-4o-mini. It resolves, so the run starts; identityViolations (below) still + // disqualifies every row that resolved to another id, so the alias can never pass as a receipt. + it("lets a known alias through, as the operator typed it", () => { + expect(unresolvableModels(["gpt-4o-mini", "claude-sonnet-4-5-20250929", "GPT-5.4-MINI"])).toEqual([]); + }); + + it("the alias run is still not evidence: its rows resolve to another id and are disqualified", () => { + const aliasRow = { model: "gpt-4o-mini", resolvedModelId: "gpt-5.4-mini", fallbackUsed: false, scenarioId: "s1", scenarioName: "s", passed: true, elapsed: 1, failureReasons: [] } as never; + expect(identityViolations([aliasRow])).toHaveLength(1); }); }); diff --git a/scripts/__tests__/judgeScratchnodeChatVideo.requestShape.test.ts b/scripts/__tests__/judgeScratchnodeChatVideo.requestShape.test.ts index baa5fe3c9..6319a1553 100644 --- a/scripts/__tests__/judgeScratchnodeChatVideo.requestShape.test.ts +++ b/scripts/__tests__/judgeScratchnodeChatVideo.requestShape.test.ts @@ -41,6 +41,19 @@ describe("judgeScratchnodeChatVideo.mjs Gemini request", () => { vi.restoreAllMocks(); }); + // Persona: a maintainer judges a recording with --model gemini-2.5-flash. thinkingLevel is a Gemini 3 field; + // 2.5 documents thinkingBudget, which main sent (PR #662 review round 6, P2). Fake client: not observed at Google. + for (const [model, want] of [["gemini-2.5-flash", { thinkingBudget: 0 }], ["gemini-3.8-flash", { thinkingLevel: "low" }]] as const) { + it(`--model ${model} sends ${JSON.stringify(want)}`, async () => { + vi.spyOn(process, "argv", "get").mockReturnValue(["node", "judge.mjs", "--video", video, "--model", model]); + await import("../ui/judgeScratchnodeChatVideo.mjs"); + await vi.waitFor(() => expect(generateContent).toHaveBeenCalledTimes(1)); + const request = generateContent.mock.calls[0][0]; + expect(request.model).toBe(model); + expect(request.config.thinkingConfig).toEqual(want); + }); + } + it("sends thinkingLevel low and no sampling keys", async () => { await import("../ui/judgeScratchnodeChatVideo.mjs"); await vi.waitFor(() => expect(generateContent).toHaveBeenCalledTimes(1)); diff --git a/scripts/run-comprehensive-eval.ts b/scripts/run-comprehensive-eval.ts index 53a3a4a72..9b5eeee5e 100644 --- a/scripts/run-comprehensive-eval.ts +++ b/scripts/run-comprehensive-eval.ts @@ -246,13 +246,14 @@ function percentile(arr: number[], p: number): number { } /** - * `--models` values that are not a canonical approved id: unknown ids, and legacy - * aliases such as `gpt-4o-mini` (which resolve to another model, so every row would be - * disqualified only after the paid run finished). Each one aborts the run before any - * request is made (SPEC-v9 A.0.10). + * `--models` values the resolver does not resolve at all. Each one aborts the run before any + * request is made (SPEC-v9 PR 1: abort only when a value "does not resolve"; A.0.10). A known + * alias such as `gpt-4o-mini` resolves, so it passes through as the operator typed it (Homen D1: + * eval-script CLI overrides are pass-through). If it resolves to another id, `identityViolations` + * disqualifies those rows as receipts after the run and exits 1, so no alias can pass as evidence. */ -export function nonCanonicalModels(models: readonly string[]): string[] { - return models.filter((m) => resolveModelAlias(m) !== m); +export function unresolvableModels(models: readonly string[]): string[] { + return models.filter((m) => resolveModelAlias(m) === null); } /** @@ -598,9 +599,9 @@ async function main() { // Fail closed: an unknown --models value used to resolve silently to the default // model, so a "candidate" run could measure a different model. - const unknown = nonCanonicalModels(modelsToTest); + const unknown = unresolvableModels(modelsToTest); if (unknown.length > 0) { - throw new Error(`Non-canonical --models value(s): ${unknown.join(", ")} (each must be an approved id exactly; aliases and unknown ids are refused)`); + throw new Error(`Unresolvable --models value(s): ${unknown.join(", ")} (no approved id or alias resolves them)`); } // Select scenarios based on suite diff --git a/scripts/ui/judgeScratchnodeChatVideo.mjs b/scripts/ui/judgeScratchnodeChatVideo.mjs index c9c8fa630..8e90d476f 100644 --- a/scripts/ui/judgeScratchnodeChatVideo.mjs +++ b/scripts/ui/judgeScratchnodeChatVideo.mjs @@ -19,7 +19,10 @@ const args = process.argv.slice(2); const getArg = (f, d) => { const i = args.indexOf(f); return i >= 0 && args[i + 1] ? args[i + 1] : d; }; const VIDEO = getArg("--video", ""); const SURFACE = getArg("--surface", "desktop"); -const MODEL = getArg("--model", "gemini-3.5-flash"); // latest Gemini 3.5 Flash (thinking model → thinkingBudget:0 set below) +const MODEL = getArg("--model", "gemini-3.5-flash"); // latest Gemini 3.5 Flash (thinking model → thinking control chosen by THINKING below) +// Keyed by the model selected (SPEC-v9 s1.3): Gemini 3.x takes thinkingLevel; any other --model (gemini-2.5-*) +// keeps the thinkingBudget:0 main sent. Same rule as geminiThinkingConfig in shared/search/triSearch.ts. +const THINKING = /^(models\/)?gemini-3/i.test(MODEL.trim()) ? { thinkingLevel: "low" } : { thinkingBudget: 0 }; if (!VIDEO || !fs.existsSync(VIDEO)) { console.error("ERR: --video required"); process.exit(1); } function loadKey() { @@ -73,7 +76,7 @@ Return ONLY JSON: const result = await ai.models.generateContent({ model: MODEL, contents: [{ role: "user", parts: [{ fileData: { fileUri: file.uri, mimeType: "video/webm" } }, { text: PROMPT }] }], - config: { responseMimeType: "application/json", maxOutputTokens: 3500, thinkingConfig: { thinkingLevel: "low" } }, + config: { responseMimeType: "application/json", maxOutputTokens: 3500, thinkingConfig: THINKING }, }); const text = (result.text || "").trim(); let parsed; diff --git a/shared/llm/anthropicWireIds.test.ts b/shared/llm/anthropicWireIds.test.ts index 239502c4d..b30190f31 100644 --- a/shared/llm/anthropicWireIds.test.ts +++ b/shared/llm/anthropicWireIds.test.ts @@ -23,6 +23,8 @@ import { anthropicWireId, FREE_LANE_CATALOG_ID, FREE_LANE_WIRE_ID, + RefusedModelError, + SPEC_LIVE_FREE_WIRE_IDS, UnknownModelError, getEquivalentModel, getModelPricing, @@ -389,3 +391,37 @@ describe("D1: resolver-registry ids the catalog cannot dispatch are refused, nev } }); }); + +// Homen D1 for the 15 live `:free` ids of SPEC-v9 section 1.2 (PR #662 review round 6, P2). This catalog has no +// OpenRouter branch until PR 2: it serves the free lane's own wire id as asked and refuses the other 14 with a +// reason. It never answers a request for a known free id with a different model. +describe("D1: the SPEC-v9 1.2 live :free ids through getModelWithFailover", () => { + beforeEach(() => { + vi.spyOn(console, "warn").mockImplementation(() => {}); + vi.stubEnv("OPENROUTER_API_KEY", "sk-or-test-key-not-real"); + }); + afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + }); + + it("each is served as itself or refused with a reason, and none comes back as another model", () => { + expect(SPEC_LIVE_FREE_WIRE_IDS.size).toBe(15); + for (const id of SPEC_LIVE_FREE_WIRE_IDS) { + if (id === FREE_LANE_WIRE_ID) { + expect(getModelWithFailover(id), id).toEqual({ model: FREE_LANE_WIRE_ID, provider: "openrouter", isFallback: false }); + expect(() => assertKnownModel(id), id).not.toThrow(); + } else { + expect(() => assertKnownModel(id), id).toThrow(UnknownModelError); + expect(() => getModelWithFailover(id), id).toThrow(UnknownModelError); + let error: unknown; + try { getModelWithFailover(id); } catch (e) { error = e; } + expect((error as Error).message, id).toMatch(/until PR 2/); + } + } + // the ones the review named + for (const id of ["apodex/apodex-1.1-mini:free", "google/gemma-4-31b-it:free", "nvidia/nemotron-3.5-content-safety:free"]) { + expect(() => getModelWithFailover(id), id).toThrow(RefusedModelError); + } + }); +}); diff --git a/shared/llm/modelCatalog.ts b/shared/llm/modelCatalog.ts index dd95a13b7..2b0257a29 100644 --- a/shared/llm/modelCatalog.ts +++ b/shared/llm/modelCatalog.ts @@ -73,6 +73,19 @@ export class UnknownModelError extends Error { } } +/** + * A KNOWN id (SPEC-v9 section 1.2 / Homen D1) that a particular transport does not serve yet. It is + * refused with its reason instead of being called unknown, and never rerouted to another model. + * It extends `UnknownModelError`, so every boundary that already catches that class still catches it. + */ +export class RefusedModelError extends UnknownModelError { + constructor(input: string, reason: string) { + super(input); + this.message = `Model "${input}" is a known id, refused here: ${reason}`; + this.name = "RefusedModelError"; + } +} + // ═══════════════════════════════════════════════════════════════════════════ // MODEL PRICING (per 1M tokens, USD) // ═══════════════════════════════════════════════════════════════════════════ @@ -663,6 +676,36 @@ export const SPEC_NATIVE_WIRE_IDS: ReadonlySet = new Set([ "gpt-5.4-mini", ]); +/** + * The 15 live `:free` ids of SPEC-v9 section 1.2 (read 2026-10-10), the only free ids Part A may + * write. Homen D1: each is known, so a request for one is never rerouted to the free lane. The + * resolver sends the unregistered ones unchanged on OpenRouter; this catalog has no OpenRouter + * branch until PR 2, so it refuses them with a reason (`FREE_LANE_WIRE_ID` is the one it serves). + */ +export const SPEC_LIVE_FREE_WIRE_IDS: ReadonlySet = new Set([ + "apodex/apodex-1.1-mini:free", + "dots-studio/dots-3-note-preview:free", + "liquid/lfm-2.5-2.6b:free", + "nvidia/nemotron-3.5-lightning:free", + "thinkingmachines/inkling-small:free", + "poolside/laguna-s-2.1:free", + "thinkingmachines/inkling:free", + "poolside/laguna-xs-2.1:free", + "cohere/north-mini-code:free", + "nvidia/nemotron-3.5-content-safety:free", + "nvidia/nemotron-3-ultra-550b-a55b:free", + "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free", + "google/gemma-4-26b-a4b-it:free", + "google/gemma-4-31b-it:free", + "nvidia/nemotron-3-super-120b-a12b:free", +]); + +/** The input, trimmed and lower-cased, when it is one of the 15 live free ids; else null. */ +export function specLiveFreeWireId(input: string): string | null { + const normalized = input.trim().toLowerCase(); + return SPEC_LIVE_FREE_WIRE_IDS.has(normalized) ? normalized : null; +} + /** * Resolver-registry ids (`APPROVED_MODELS`, `LEGACY_ALIASES`, and their wire strings) whose * provider is OpenRouter and that this catalog cannot dispatch: it has no OpenRouter branch @@ -1028,6 +1071,10 @@ export function assertKnownModel(model: string): void { if (OPENROUTER_ONLY_REGISTRY_IDS.has(model)) { throw new UnknownModelError(model, "a registered OpenRouter id this catalog cannot dispatch until PR 2 adds the OpenRouter branch; it is refused, never rerouted to another model"); } + const liveFree = specLiveFreeWireId(model); + if (liveFree !== null && liveFree !== FREE_LANE_WIRE_ID) { + throw new RefusedModelError(model, "a live free OpenRouter id of SPEC-v9 section 1.2 that this catalog cannot dispatch until PR 2 adds the OpenRouter branch; it is never rerouted to another model"); + } if (!isKnownCatalogModel(model) && !isFreeLookingModel(model)) { throw new UnknownModelError(model, "an unrecognised id never fails over to a paid default and is never sent to a provider"); } @@ -1052,6 +1099,10 @@ export function getModelWithFailover(preferredModel: string): { isFallback: boolean; } { assertKnownModel(preferredModel); + // The free lane's own wire id is known (SPEC-v9 1.2): served as asked, not as a substitution. + if (specLiveFreeWireId(preferredModel) === FREE_LANE_WIRE_ID) { + return { model: FREE_LANE_WIRE_ID, provider: "openrouter", isFallback: false }; + } const known = isKnownCatalogModel(preferredModel); const provider = getProviderForModel(preferredModel); diff --git a/shared/search/triSearch.test.ts b/shared/search/triSearch.test.ts index a79f8c6df..bc52c7600 100644 --- a/shared/search/triSearch.test.ts +++ b/shared/search/triSearch.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect } from "vitest"; +import { afterEach, describe, it, expect, vi } from "vitest"; import { rrfFuse, rerankWithGemini, triSearch, condenseQuery, RRF_K, type TriCandidate } from "./triSearch"; // Scenario-based tests (.claude/rules/scenario_testing.md). @@ -242,3 +242,49 @@ describe("rerankWithGemini - request shape (Gemini 3.x)", () => { expect(bodies[0].contents.at(-1).role).not.toBe("model"); }); }); + +// Persona: an operator pins TRI_SEARCH_RERANK_MODEL to a gemini-2.5 id (still in the repo's registry) while +// the default moves to Gemini 3.x. thinkingLevel is a Gemini 3 field; sent to 2.5 it is rejected or ignored, +// and both calls would quietly fall back to unranked order (PR #662 review round 6, P2). Not observed against +// Google: the shape follows Google's thinking guide, as the Gemini 3.x cases above do. +describe("thinking control follows the selected model (SPEC-v9 s1.3: keyed by exact id)", () => { + afterEach(() => { + vi.unstubAllEnvs(); + vi.resetModules(); + }); + const run = async (model: string) => { + vi.stubEnv("TRI_SEARCH_RERANK_MODEL", model); + vi.resetModules(); + const mod = await import("./triSearch"); + const bodies: any[] = []; + const urls: string[] = []; + const spy = (async (url: string, init: { body: string }) => { + urls.push(String(url)); + bodies.push(JSON.parse(init.body)); + return { ok: true, status: 200, json: async () => ({ candidates: [{ content: { parts: [{ text: "[1,0]" }] } }] }) } as unknown as Response; + }) as unknown as typeof fetch; + await mod.rerankWithGemini("entity funding", mod.rrfFuse([leg("linkup", ["a", "b"])]), { topN: 2, apiKey: "test", fetchImpl: spy }); + await mod.condenseQuery("and the cost?", ["audit tool calls"], { apiKey: "test", fetchImpl: spy }); + return { bodies, urls }; + }; + + it("a gemini-2.5 override keeps thinkingBudget:0 and never gets thinkingLevel, in both calls", async () => { + const { bodies, urls } = await run("gemini-2.5-flash"); + expect(bodies).toHaveLength(2); + for (const [i, body] of bodies.entries()) { + expect(urls[i]).toContain("/models/gemini-2.5-flash:generateContent"); + expect(body.generationConfig.thinkingConfig).toEqual({ thinkingBudget: 0 }); + expect(JSON.stringify(body)).not.toContain("thinkingLevel"); + } + }); + + it("a gemini-3.x override gets thinkingLevel low and never thinkingBudget, in both calls", async () => { + const { bodies, urls } = await run("gemini-3.8-flash"); + expect(bodies).toHaveLength(2); + for (const [i, body] of bodies.entries()) { + expect(urls[i]).toContain("/models/gemini-3.8-flash:generateContent"); + expect(body.generationConfig.thinkingConfig).toEqual({ thinkingLevel: "low" }); + expect(JSON.stringify(body)).not.toContain("thinkingBudget"); + } + }); +}); diff --git a/shared/search/triSearch.ts b/shared/search/triSearch.ts index 31e921e77..884768d2e 100644 --- a/shared/search/triSearch.ts +++ b/shared/search/triSearch.ts @@ -56,8 +56,19 @@ export const MAX_RERANK_CANDIDATES = 12; // below, else it spends the token budget on a reasoning preamble and never emits the bare index // array (verified with the since-removed thinkingBudget:0: thinking-on tok=96 → no array; // thinking-0 tok=256 → "[0,1,2]" in ~700ms). Gemini 3.x replaced thinkingBudget with -// thinkingConfig.thinkingLevel (`minimal` is unsupported), so the request now sends "low". +// thinkingConfig.thinkingLevel (`minimal` is unsupported), so a 3.x model is sent "low"; a 2.5 override +// still gets thinkingBudget:0 (geminiThinkingConfig). const RERANK_MODEL = process.env.TRI_SEARCH_RERANK_MODEL || "gemini-3.5-flash"; +/** + * Thinking control keyed by the model actually selected (SPEC-v9 s1.3: request shapes are keyed by + * exact model id; A.0.4: a TRI_SEARCH_RERANK_MODEL override keeps working). Gemini 3.x takes + * `thinkingLevel` and no longer accepts `thinkingBudget`; every other id, gemini-2.5-* included, + * keeps the `thinkingBudget: 0` that main sent, which is the field those models document. + */ +export function geminiThinkingConfig(model: string): { thinkingLevel: "low" } | { thinkingBudget: 0 } { + return /^(models\/)?gemini-3/i.test(model.trim()) ? { thinkingLevel: "low" } : { thinkingBudget: 0 }; +} + const RERANK_TIMEOUT_MS = Number(process.env.TRI_SEARCH_RERANK_TIMEOUT_MS) || 6000; const MAX_TITLE = 120; const MAX_SNIPPET = 280; @@ -141,7 +152,7 @@ export async function rerankWithGemini( contents: [{ parts: [{ text: prompt }] }], // thinkingLevel "low" keeps thinking models (3.x) from spending the budget on a // reasoning preamble so they emit the bare index array; 256 tokens leaves room. - generationConfig: { maxOutputTokens: 256, thinkingConfig: { thinkingLevel: "low" } }, // no sampling: deprecated on Gemini 3.x (SPEC-v9 s1.3) + generationConfig: { maxOutputTokens: 256, thinkingConfig: geminiThinkingConfig(RERANK_MODEL) }, // no sampling: deprecated on Gemini 3.x (SPEC-v9 s1.3) }), signal: AbortSignal.timeout(opts.timeoutMs ?? RERANK_TIMEOUT_MS), }, @@ -267,7 +278,7 @@ export async function condenseQuery( // Same Gemini 3.x model as the reranker: thinkingBudget is removed there (SPEC-v9 s1.3), so // thinkingLevel "low" replaces it. Thinking counts toward the cap, so 256 (the rerank sizing) // leaves room for a 400-char rewrite; a truncated reply would silently fall back to the raw question. - generationConfig: { maxOutputTokens: 256, thinkingConfig: { thinkingLevel: "low" } }, // no sampling: deprecated on Gemini 3.x (SPEC-v9 s1.3) + generationConfig: { maxOutputTokens: 256, thinkingConfig: geminiThinkingConfig(RERANK_MODEL) }, // no sampling: deprecated on Gemini 3.x (SPEC-v9 s1.3) }), signal: AbortSignal.timeout(opts.timeoutMs ?? RERANK_TIMEOUT_MS), },