chore(models): refresh stale model defaults (NodeBenchAI)

.github/workflows/ci.yml

@@ -100,6 +100,25 @@ jobs:
           backend/convex/domains/mcp/mcpSourcingDraft.integration.test.ts
           backend/convex/domains/mcp/mcpExecutionTraceEndpoints.integration.test.ts
           backend/convex/domains/agents/mcp_tools/models/modelResolver.test.ts
+          backend/convex/domains/pipelines/pipelineModelBoundary.test.ts
+          backend/convex/events.requestShape.test.ts
+          shared/llm/anthropicWireIds.test.ts
+          shared/search/triSearch.test.ts
+          workers/node/pipeline/extendedThinkingRunner.test.ts
+          scripts/eval-harness/evalHarness.test.ts
+          scripts/__tests__/toolGenRequestShape.test.ts
+          scripts/__tests__/comprehensiveEvalModels.test.ts
+          apps/web/src/shared/llm/approvedModels.drift.test.ts
+          backend/convex/domains/ai/models/modelRouter.requestShape.test.ts
+          backend/convex/domains/search/fusion/benchmark.requestShape.test.ts
+          backend/convex/domains/product/extendedThinking.requestShape.test.ts
+          backend/convex/providerCopies.requestShape.test.ts
+          backend/convex/domains/agents/runtimeRouting.test.ts
+          backend/convex/domains/verification/instagramClaimVerification.test.ts
+          backend/convex/domains/agents/agentChatActions.failClosed.test.ts
+          backend/convex/domains/research/dailyBriefModelBoundary.test.ts
+          scripts/__tests__/judgeScratchnodeChatVideo.requestShape.test.ts
+          packages/mcp-local/src/__tests__/anthropicRequestShape.test.ts
           apps/web/src/features/agents/components/FastAgentPanel/__tests__/RunState.test.tsx
           backend/convex/domains/agents/autonomy/autonomy.integration.test.ts
           backend/convex/domains/agents/autonomy/autonomyTasteBench.integration.test.ts

apps/web/src/shared/llm/approvedModels.drift.test.ts

@@ -0,0 +1,63 @@
+/**
+ * Drift guard: apps/web/src/shared/llm/approvedModels.ts is a hand-kept mirror of
+ * backend/convex/domains/agents/mcp_tools/models/modelResolver.ts (Convex cannot import
+ * from the web tree). The two had already drifted - MODEL_PRIORITY_ORDER had 16 entries
+ * against the resolver's 24 - so the selector could offer or order models the backend
+ * would route differently (SPEC-v9 A.2.10 PR 1). Failing here means one side was edited
+ * without the other: regenerate the web side from the resolver, never loosen this test.
+ */
+
+import { describe, expect, it } from "vitest";
+
+import * as web from "./approvedModels";
+import { PIPELINE_MODEL_OPTIONS } from "./pipelineModelRoutes";
+import * as backend from "../../../../../backend/convex/domains/agents/mcp_tools/models/modelResolver";
+
+describe("web approvedModels mirrors modelResolver", () => {
+  it("lists the same approved ids in the same order", () => {
+    expect([...web.APPROVED_MODELS]).toEqual([...backend.APPROVED_MODELS]);
+  });
+
+  it("has a UI row for every approved id, agreeing on provider and free status", () => {
+    expect(Object.keys(web.MODEL_UI_INFO).sort()).toEqual([...backend.APPROVED_MODELS].sort());
+    for (const id of backend.APPROVED_MODELS) {
+      const ui = web.MODEL_UI_INFO[id];
+      expect(ui.id, id).toBe(id);
+      expect(ui.provider, `${id} provider`).toBe(backend.MODEL_SPECS[id].provider);
+      expect(Boolean(ui.isFree), `${id} isFree`).toBe(backend.isFreeModel(id));
+    }
+  });
+
+  it("agrees on the default, fallback and free-lane constants", () => {
+    expect(web.DEFAULT_MODEL).toBe(backend.DEFAULT_MODEL);
+    expect(web.FALLBACK_MODEL).toBe(backend.FALLBACK_MODEL);
+    expect(web.MODEL_UI_INFO[backend.FREE_LANE_MODEL].isFree).toBe(true);
+  });
+
+  it("agrees on the priority order and the advisor/executor/background lanes", () => {
+    expect(web.MODEL_PRIORITY_ORDER).toEqual(backend.MODEL_PRIORITY_ORDER);
+    expect(web.NODEBENCH_ADVISOR_MODEL).toBe(backend.NODEBENCH_ADVISOR_MODEL);
+    expect(web.NODEBENCH_EXECUTOR_MODELS).toEqual(backend.NODEBENCH_EXECUTOR_MODELS);
+    expect(web.NODEBENCH_BACKGROUND_MODELS).toEqual(backend.NODEBENCH_BACKGROUND_MODELS);
+  });
+
+  it("normalizes every approved id to the same lane model", () => {
+    for (const profile of ["advisor", "executor", "background"] as const) {
+      for (const id of backend.APPROVED_MODELS) {
+        expect(web.normalizeNodeBenchRuntimeModel(id, profile), `${profile}/${id}`).toBe(
+          backend.normalizeNodeBenchRuntimeModel(id, profile),
+        );
+      }
+    }
+  });
+
+  it("pins every selector option, including the auto-free literal, to the backend route resolver", () => {
+    // pipelineModelRoutes.ts types its resolved ids by hand. With getBestFreeModel reverted to
+    // laguna, the web tests still passed and only modelResolver.test.ts failed; this closes that gap.
+    for (const option of PIPELINE_MODEL_OPTIONS) {
+      const resolved = backend.resolvePipelineModelSelection(option.value);
+      expect(resolved.resolvedModelId, `${option.value} resolvedModelId`).toBe(option.resolvedModelId);
+      expect(resolved.isFreeRoute, `${option.value} isFree`).toBe(option.isFree);
+    }
+  });
+});

apps/web/src/shared/llm/approvedModels.ts

@@ -32,6 +32,7 @@ export const APPROVED_MODELS = [
   "qwen3-235b",
   "minimax-m2.7",
   "mistral-large",
+  "nemotron-3-super-free",
   "laguna-s-2.1-free",
   "laguna-xs-2.1-free",
   "qwen3-coder-free",
@@ -293,6 +294,16 @@ export const MODEL_UI_INFO: Record<ApprovedModel, ModelUIInfo> = {
     contextWindow: "131K",
     icon: "OR",
   },
+  "nemotron-3-super-free": {
+    id: "nemotron-3-super-free",
+    name: "Nemotron 3 Super (Free)",
+    provider: "openrouter",
+    description: "Verified free lane with tools and structured outputs",
+    tier: "powerful",
+    contextWindow: "262K",
+    icon: "OR",
+    isFree: true,
+  },
   "laguna-s-2.1-free": {
     id: "laguna-s-2.1-free",
     name: "Laguna S 2.1 (Free)",
@@ -478,23 +489,33 @@ export const MODEL_UI_INFO: Record<ApprovedModel, ModelUIInfo> = {
 export const DEFAULT_MODEL: ApprovedModel = "kimi-k2.6";
 export const FALLBACK_MODEL: ApprovedModel = "gpt-5.4-mini";
 
+// Mirrors modelResolver.ts; approvedModels.drift.test.ts fails when they diverge.
 export const MODEL_PRIORITY_ORDER: ApprovedModel[] = [
   "kimi-k2.6",
-  "minimax-m2.7",
   "gemini-3.1-flash-lite-preview",
-  "gpt-5.4-mini",
   "gemini-3-flash-preview",
   "gemini-3.1-pro-preview",
+  "minimax-m2.7",
+  "gpt-5.4-mini",
   "gpt-5.4",
   "claude-sonnet-4.6",
   "claude-opus-4.7",
   "claude-haiku-4.5",
   "gpt-5.4-nano",
+  "nemotron-3-super-free",
+  "laguna-s-2.1-free",
+  "laguna-xs-2.1-free",
   "qwen3-coder-free",
   "step-3.5-flash-free",
   "gpt-oss-120b-free",
   "trinity-large-free",
   "nemotron-3-nano-free",
+  "qwen3-next-free",
+  "llama-3.3-70b-free",
+  "glm-4.5-air-free",
+  "deepseek-r1-free",
+  "deepseek-chimera-free",
+  "venice-dolphin-free",
 ];
 
 export type NodeBenchRuntimeProfile = "advisor" | "executor" | "background";
@@ -503,16 +524,16 @@ export const NODEBENCH_ADVISOR_MODEL: ApprovedModel = "kimi-k2.6";
 
 export const NODEBENCH_EXECUTOR_MODELS: ApprovedModel[] = [
   "gemini-3.1-flash-lite-preview",
-  "gpt-5.4-mini",
-  "minimax-m2.7",
   "gemini-3-flash-preview",
+  "minimax-m2.7",
+  "gpt-5.4-mini",
   "kimi-k2.6",
 ];
 
 export const NODEBENCH_BACKGROUND_MODELS: ApprovedModel[] = [
   "gemini-3.1-pro-preview",
-  "kimi-k2.6",
   "gpt-5.4",
+  "kimi-k2.6",
   "gemini-3-flash-preview",
 ];
 
@@ -549,6 +570,11 @@ export function normalizeNodeBenchRuntimeModel(
     return fallback;
   }
 
+  // A free model keeps its cost class in every profile (mirrors modelResolver.ts).
+  if (MODEL_UI_INFO[candidate].isFree) {
+    return candidate;
+  }
+
   if (profile === "advisor" && NODEBENCH_ADVISOR_ALLOWED.has(candidate)) {
     return candidate;
   }

apps/web/src/shared/llm/pipelineModelRoutes.test.ts

@@ -25,10 +25,12 @@ describe("pipeline model routes", () => {
     expect(option).toMatchObject({
       label: "Auto free",
       provider: "openrouter",
-      resolvedModelId: "laguna-s-2.1-free",
+      // Was laguna-s-2.1-free: that lane expires on OpenRouter 2026-10-31, so auto-free
+      // resolves to FREE_LANE_MODEL (SPEC-v9 A.2.10 PR 1; NodeVideo's measured pick).
+      resolvedModelId: "nemotron-3-super-free",
       isFree: true,
     });
-    expect(getPipelineModelRuntimeId(PIPELINE_MODEL_AUTO_FREE)).toBe("laguna-s-2.1-free");
+    expect(getPipelineModelRuntimeId(PIPELINE_MODEL_AUTO_FREE)).toBe("nemotron-3-super-free");
   });
 
   it("keeps direct models and route models in one selector bank", () => {

apps/web/src/shared/llm/pipelineModelRoutes.ts

@@ -47,7 +47,9 @@ const AUTO_ROUTE_OPTIONS: PipelineModelOption[] = [
     shortLabel: "Auto free",
     detail: "Free-capable OpenRouter route for budget-sensitive runs.",
     provider: "openrouter",
-    resolvedModelId: "laguna-s-2.1-free",
+    // Must equal routeAliasToModel("auto-free") in modelResolver.ts (FREE_LANE_MODEL):
+    // laguna-s-2.1:free expires on OpenRouter 2026-10-31 (SPEC-v9 A.1).
+    resolvedModelId: "nemotron-3-super-free",
     isRoute: true,
     isFree: true,
   },

backend/convex/__tests__/agentRuntimeTierFallback.test.ts

@@ -17,6 +17,13 @@ import {
   formatPublicAgentRunError,
   projectAgentRunPresentation,
 } from "../domains/agents/agentRunPresentation";
+import {
+  getBestModelForTier,
+  getProviderForModel,
+  isModelAllowedForTier,
+  type UserTier,
+} from "../../../shared/llm/modelCatalog";
+import { FREE_LANE_MODEL } from "../domains/agents/mcp_tools/models/modelResolver";
 import { setEffectiveModel } from "../domains/agents/orchestrator/queueProtocol";
 import {
   checkRequestAllowed,
@@ -176,6 +183,59 @@ describe("selectTierEligibleRuntimeModel", () => {
     });
   });
 
+  // A.0.10: the real tier policy (checkRequestAccess' own calls), not a stub.
+  const realTierCheck = (tier: UserTier) => async (model: string) =>
+    isModelAllowedForTier(model, tier)
+      ? allowed()
+      : denied(
+          `Model "${model}" is not available on the ${tier} tier`,
+          getBestModelForTier("chat", tier, getProviderForModel(model) || "openai"),
+        );
+
+  it.each(["anonymous", "free"] as const)(
+    "never answers a free-lane request with a paid model on the %s tier",
+    async (tier) => {
+      const checkModel = vi.fn(realTierCheck(tier));
+      const result = await selectTierEligibleRuntimeModel({
+        primaryModel: FREE_LANE_MODEL,
+        fallbackModels: [],
+        checkModel,
+      });
+
+      expect(result.model).toBeNull();
+      expect(checkModel.mock.calls.map(([m]) => m)).toEqual([FREE_LANE_MODEL]);
+      expect(getTierSelectionFailureReason(result)).toContain(
+        `not available on the ${tier} tier`,
+      );
+    },
+  );
+
+  it("still serves the free lane where the tier allows it", async () => {
+    const result = await selectTierEligibleRuntimeModel({
+      primaryModel: FREE_LANE_MODEL,
+      checkModel: realTierCheck("pro"),
+    });
+    expect(result.model).toBe(FREE_LANE_MODEL);
+  });
+
+  it("keeps a failed free run free when choosing a post-stream fallback", async () => {
+    const result = await selectTierEligibleRuntimeModel({
+      primaryModel: "gpt-5.4-mini",
+      excludedModels: [FREE_LANE_MODEL],
+      requireFree: true,
+      checkModel: realTierCheck("pro"),
+    });
+    expect(result.model).toBeNull();
+  });
+
+  it("preserves the paid fallbacks for a paid primary", async () => {
+    const result = await selectTierEligibleRuntimeModel({
+      primaryModel: "kimi-k2.6",
+      checkModel: realTierCheck("anonymous"),
+    });
+    expect(result.model).toBe("gpt-5.4-nano");
+  });
+
   it("skips already-attempted models when choosing a tier-eligible fallback", async () => {
     const checkModel = vi.fn(async () => allowed());
     const result = await selectTierEligibleRuntimeModel({

backend/convex/domains/agents/agentChatActions.failClosed.test.ts

@@ -0,0 +1,63 @@
+// @vitest-environment node
+/**
+ * agentChatActions.continueThread resumes a conversation on the model stored on its thread
+ * (SPEC-v9 A.0.10; PR #662 review round 1). Persona: a user returning to a thread whose stored
+ * model name no longer resolves (a retired or mistyped alias). The turn must stop with
+ * UnknownModelError. It used to catch the error, log a warning and carry on with DEFAULT_MODEL
+ * (kimi-k2.6), so the user paid for a model they never chose; logging a fallback is not a control.
+ * A transient failure to read the thread is a different thing and still uses the default.
+ * The agent is faked; no network, no spend.
+ */
+
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+const createCoordinatorAgent = vi.hoisted(() => vi.fn());
+vi.mock("./core/coordinatorAgent", () => ({ createCoordinatorAgent }));
+vi.mock("@convex-dev/auth/server", () => ({ getAuthUserId: vi.fn(async () => "user1") }));
+
+import { continueThread } from "./agentChatActions";
+import { UnknownModelError } from "./mcp_tools/models";
+
+function ctxWith(streamThreadModel: string | Error) {
+  return {
+    runQuery: vi.fn(async (_ref: unknown, args: Record<string, unknown>) => {
+      if ("agentThreadId" in args) {
+        if (streamThreadModel instanceof Error) throw streamThreadModel;
+        return { model: streamThreadModel };
+      }
+      return { agentThreadId: "agent-thread-1" };
+    }),
+  };
+}
+
+describe("agentChatActions.continueThread fails closed on a stored model", () => {
+  beforeEach(() => {
+    vi.spyOn(console, "warn").mockImplementation(() => {});
+    createCoordinatorAgent.mockReset();
+    const result = { consumeStream: vi.fn(async () => undefined) };
+    createCoordinatorAgent.mockReturnValue({
+      saveMessage: vi.fn(async () => ({ messageId: "m1" })),
+      streamText: vi.fn(async () => result),
+    });
+  });
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  it("an unknown stored model stops the turn and starts no agent", async () => {
+    await expect(
+      (continueThread as any)._handler(ctxWith("gpt-9-ultra"), { threadId: "t1", message: "hi" }),
+    ).rejects.toBeInstanceOf(UnknownModelError);
+    expect(createCoordinatorAgent).not.toHaveBeenCalled();
+  });
+
+  it("a free-looking stored model continues on the free lane, not a paid default", async () => {
+    await (continueThread as any)._handler(ctxWith("retired-model-free"), { threadId: "t1", message: "hi" });
+    expect(createCoordinatorAgent).toHaveBeenCalledWith("nemotron-3-super-free");
+  });
+
+  it("a transient failure to read the thread still falls back to the default model", async () => {
+    await (continueThread as any)._handler(ctxWith(new Error("convex timeout")), { threadId: "t1", message: "hi" });
+    expect(createCoordinatorAgent).toHaveBeenCalledWith("kimi-k2.6");
+  });
+});

backend/convex/domains/agents/agentChatActions.ts

@@ -11,7 +11,7 @@ import { api, internal } from "../../_generated/api";
 import type { Id } from "../../_generated/dataModel";
 import { getAuthUserId } from "@convex-dev/auth/server";
 import { createCoordinatorAgent } from "./core/coordinatorAgent";
-import { DEFAULT_MODEL, normalizeModelInput } from "./mcp_tools/models";
+import { DEFAULT_MODEL, normalizeModelInput, UnknownModelError } from "./mcp_tools/models";
 
 // Agent-mode chat should use the same CoordinatorAgent wiring as streaming mode.
 const createChatAgent = (model: string) => createCoordinatorAgent(model);
@@ -133,6 +133,8 @@ export const continueThread = (action as any)({
         modelName = normalizeModelInput(streamThreadByAgentId.model);
       }
     } catch (err) {
+      // An unknown stored model must stop the turn, not continue on DEFAULT_MODEL (SPEC-v9 A.0.10).
+      if (err instanceof UnknownModelError) throw err;
       console.warn("[agentChatActions.continueThread] Failed to resolve thread model, using default.", err);
     }
 

backend/convex/domains/agents/fastAgentChat.ts

@@ -12,7 +12,7 @@ import OpenAI from "openai";
 import Anthropic from "@anthropic-ai/sdk";
 import { createGoogleGenerativeAI } from "@ai-sdk/google";
 import { generateText } from "ai";
-import { getLlmModel, resolveModelAlias, getModelWithFailover } from "../../../../shared/llm/modelCatalog";
+import { getLlmModel, resolveModelAlias, getModelWithFailover, anthropicText } from "../../../../shared/llm/modelCatalog";
 
 /**
  * Modern fast agent chat action
@@ -283,7 +283,8 @@ async function handleDocumentEdit(
  * Handle chat/question requests
  * Uses direct LLM call for quick responses
  */
-async function handleChatResponse(
+// Exported for providerCopies.requestShape.test.ts (SPEC-v9 A.0.6); not a Convex function.
+export async function handleChatResponse(
   ctx: ActionCtx,
   args: {
     message: string;
@@ -326,7 +327,7 @@ async function handleChatResponse(
       system: systemPrompt,
       messages: [{ role: "user", content: message }],
     });
-    response = completion.content[0]?.type === "text" ? completion.content[0].text : "I apologize, but I couldn't generate a response.";
+    response = anthropicText(completion.content) || "I apologize, but I couldn't generate a response.";
   } else if (provider === "gemini") {
     const google = createGoogleGenerativeAI({ apiKey: process.env.GEMINI_API_KEY });
     const result = await generateText({