feat(evals): Record and replay eval model calls with Roach

.github/workflows/eval.yml

@@ -29,7 +29,8 @@ jobs:
       - name: Setup Node.js
         uses: actions/setup-node@v4
         with:
-          node-version: "22"
+          # Roach (see packages/mcp-server-evals/roach.json) needs Node 24.14+.
+          node-version: "24"
 
       # pnpm/action-setup@v4
       - uses: pnpm/action-setup@a7487c7e89a18df4991f7f222e4898a00d66ddda
@@ -62,6 +63,9 @@ jobs:
         env:
           OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
           EMBEDDED_AGENT_PROVIDER: openrouter
+          # Lets Roach record model calls it has no recording for. Fork PRs
+          # get no secrets, so they only replay.
+          ROACH_TOKEN: ${{ secrets.ROACH_TOKEN }}
 
       - name: Create eval status check
         uses: actions/github-script@v7

docs/testing/overview.md

@@ -302,6 +302,22 @@ pnpm eval
 pnpm eval tool-name
 ```
 
+### Eval Recordings
+
+Evals send their OpenRouter calls through [Roach](https://github.com/getsentry/roach),
+which replays a recorded response when it has one. `packages/mcp-server-evals/roach.json`
+names the host and the rule. Recording and replay need Node 24.14 or later.
+
+- `ROACH_TOKEN` lets a run record calls that have no recording. The Eval
+  workflow sets it. Without it, a run only replays, and a call with no
+  recording fails its eval. A replay sends nothing upstream, so any
+  `OPENROUTER_API_KEY` value works for it.
+- `ROACH_MODE=off` sends every call live and records nothing. Use it locally
+  when you change prompts, tools, or fixtures and have no token.
+- `ROACH_MODE=record` sends every call live and replaces the recordings.
+- The Roach setup file comes before `src/setup-env.ts` in `vitest.config.ts`,
+  so MSW does not see Roach's own calls.
+
 ## Test Data Management
 
 ### Using Fixtures

packages/mcp-server-evals/package.json

@@ -27,5 +27,8 @@
     "vitest": "catalog:",
     "vitest-evals": "catalog:",
     "zod": "catalog:"
+  },
+  "devDependencies": {
+    "@sentry/roach": "catalog:"
   }
 }

packages/mcp-server-evals/roach.json

@@ -0,0 +1,13 @@
+{
+  "tenant": "getsentry/toolkit",
+  "allow": ["https://openrouter.ai"],
+  "rules": [
+    {
+      "name": "model",
+      "match": {
+        "method": "POST",
+        "url": "https://openrouter.ai/api/v1/"
+      }
+    }
+  ]
+}

packages/mcp-server-evals/vitest.config.ts

@@ -10,6 +10,9 @@ export default defineConfig({
       reporter: ["text", "json", "html"],
       include: ["**/*.ts"],
     },
-    setupFiles: ["./src/setup-env.ts"],
+    // Roach records model calls and replays them on later runs. See roach.json.
+    // Its setup file loads before setup-env.ts starts MSW.
+    globalSetup: ["@sentry/roach/vitest"],
+    setupFiles: ["@sentry/roach/vitest/setup", "./src/setup-env.ts"],
   },
 });

pnpm-workspace.yaml

@@ -57,6 +57,7 @@ catalog:
   "@sentry/core": 11.6.0
   "@sentry/node": 11.6.0
   "@sentry/react": 11.6.0
+  "@sentry/roach": 0.3.0
   "@sentry/search-syntax": ^1.0.0
   "@sentry/vite-plugin": ^4.6.1
   "@tailwindcss/typography": ^0.5.16
@@ -124,6 +125,7 @@ minimumReleaseAgeExclude:
   - "@sentry/react@11.6.0"
   - "@sentry/replay-canvas@11.6.0"
   - "@sentry/replay@11.6.0"
+  - "@sentry/roach@0.3.0"
   - "@sentry/search-syntax@1.0.0"
   - "@sentry/server-runtime-injection@11.6.0"
   - "@sentry/server-utils@11.6.0"