From f99011915d5476ffb073da5a04c882a9161d2f0e Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Thu, 5 Nov 2026 01:54:50 +0000 Subject: [PATCH 1/2] feat(evals): Record and replay eval model calls with Roach The MCP evals call OpenRouter on every run. Roach, Sentry's shared recording proxy, now sits in front of those calls: it replays a call it has a recording for, and records the rest when the run has ROACH_TOKEN. The setup is config only: roach.json names the OpenRouter host and one rule for its API, vitest.config.ts adds the Roach global setup and setup file, and the Eval workflow passes ROACH_TOKEN and moves to Node 24, which Roach needs. Fork PRs get no secrets, so they only replay. The Roach setup file loads before setup-env.ts so that MSW does not catch Roach's own control calls. That needs getsentry/roach#30. Co-Authored-By: David Cramer <23610+dcramer@users.noreply.github.com> --- .github/workflows/eval.yml | 6 ++++- docs/testing/overview.md | 16 +++++++++++++ packages/mcp-server-evals/package.json | 3 +++ packages/mcp-server-evals/roach.json | 13 +++++++++++ packages/mcp-server-evals/vitest.config.ts | 5 ++++- pnpm-lock.yaml | 26 ++++++++++++++++++++++ pnpm-workspace.yaml | 2 ++ 7 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 packages/mcp-server-evals/roach.json diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index 6d54abbab..25784c9d1 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -29,7 +29,8 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v4 with: - node-version: "22" + # Roach (see packages/mcp-server-evals/roach.json) needs Node 24.14+. + node-version: "24" # pnpm/action-setup@v4 - uses: pnpm/action-setup@a7487c7e89a18df4991f7f222e4898a00d66ddda @@ -62,6 +63,9 @@ jobs: env: OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} EMBEDDED_AGENT_PROVIDER: openrouter + # Lets Roach record model calls it has no recording for. Fork PRs + # get no secrets, so they only replay. + ROACH_TOKEN: ${{ secrets.ROACH_TOKEN }} - name: Create eval status check uses: actions/github-script@v7 diff --git a/docs/testing/overview.md b/docs/testing/overview.md index 4a7891969..fe2c87c1d 100644 --- a/docs/testing/overview.md +++ b/docs/testing/overview.md @@ -302,6 +302,22 @@ pnpm eval pnpm eval tool-name ``` +### Eval Recordings + +Evals send their OpenRouter calls through [Roach](https://github.com/getsentry/roach), +which replays a recorded response when it has one. `packages/mcp-server-evals/roach.json` +names the host and the rule. Recording and replay need Node 24.14 or later. + +- `ROACH_TOKEN` lets a run record calls that have no recording. The Eval + workflow sets it. Without it, a run only replays, and a call with no + recording fails its eval. A replay sends nothing upstream, so any + `OPENROUTER_API_KEY` value works for it. +- `ROACH_MODE=off` sends every call live and records nothing. Use it locally + when you change prompts, tools, or fixtures and have no token. +- `ROACH_MODE=record` sends every call live and replaces the recordings. +- The Roach setup file comes before `src/setup-env.ts` in `vitest.config.ts`, + so MSW does not see Roach's own calls. + ## Test Data Management ### Using Fixtures diff --git a/packages/mcp-server-evals/package.json b/packages/mcp-server-evals/package.json index 778274356..3b4486166 100644 --- a/packages/mcp-server-evals/package.json +++ b/packages/mcp-server-evals/package.json @@ -27,5 +27,8 @@ "vitest": "catalog:", "vitest-evals": "catalog:", "zod": "catalog:" + }, + "devDependencies": { + "@sentry/roach": "catalog:" } } diff --git a/packages/mcp-server-evals/roach.json b/packages/mcp-server-evals/roach.json new file mode 100644 index 000000000..e82db1c94 --- /dev/null +++ b/packages/mcp-server-evals/roach.json @@ -0,0 +1,13 @@ +{ + "tenant": "getsentry/toolkit", + "allow": ["https://openrouter.ai"], + "rules": [ + { + "name": "model", + "match": { + "method": "POST", + "url": "https://openrouter.ai/api/v1/" + } + } + ] +} diff --git a/packages/mcp-server-evals/vitest.config.ts b/packages/mcp-server-evals/vitest.config.ts index cdbf92da4..c4e5a1906 100644 --- a/packages/mcp-server-evals/vitest.config.ts +++ b/packages/mcp-server-evals/vitest.config.ts @@ -10,6 +10,9 @@ export default defineConfig({ reporter: ["text", "json", "html"], include: ["**/*.ts"], }, - setupFiles: ["./src/setup-env.ts"], + // Roach records model calls and replays them on later runs. See roach.json. + // Its setup file loads before setup-env.ts starts MSW. + globalSetup: ["@sentry/roach/vitest"], + setupFiles: ["@sentry/roach/vitest/setup", "./src/setup-env.ts"], }, }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index db7fa68aa..d3e7a4cb1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -57,6 +57,9 @@ catalogs: '@sentry/react': specifier: 11.6.0 version: 11.6.0 + '@sentry/roach': + specifier: 0.2.0 + version: 0.2.0 '@sentry/search-syntax': specifier: ^1.0.0 version: 1.0.0 @@ -955,6 +958,10 @@ importers: zod: specifier: ^4.6.6 version: 4.6.7 + devDependencies: + '@sentry/roach': + specifier: 'catalog:' + version: 0.2.0(vitest@5.0.5) packages/mcp-server-mocks: dependencies: @@ -4676,6 +4683,15 @@ packages: resolution: {integrity: sha512-cJyPYnF6QJA8ADRfAW06YoU7EVIebtWezBOCU9Vag5+W1VLBkXEvH7C4LbeMUiDmrqw92sumq06NgqipIxA7BA==} engines: {node: '>=20.19.0 <22.0.0 || >=22.12.0 <23.0.0 || >=23.2.0'} + '@sentry/roach@0.2.0': + resolution: {integrity: sha512-F9DOaNnqJ1PzPzxcl+nexu10yz6LXxQOCrnHVCoAlUd+lBTMIm0zmNDO7L8X3KsotpxTUw3ROMhQDVkLhzdTrA==} + engines: {node: '>=24'} + peerDependencies: + vitest: '>=3' + peerDependenciesMeta: + vitest: + optional: true + '@sentry/search-syntax@1.0.0': resolution: {integrity: sha512-R+raAe4PixC+e7Zn6uPH7NSK7nzfHy2MA1kRWDn7dFH8zaU10edFBVPj8qO3TlSb58yts0tejPdy+s1acq80wQ==} @@ -12661,6 +12677,16 @@ snapshots: '@sentry/browser-utils': 11.6.0 '@sentry/core': 11.6.0 + '@sentry/roach@0.2.0(vitest@5.0.5)': + dependencies: + '@sentry/node': 11.6.0(@babel/core@7.28.0)(rollup@4.62.4) + optionalDependencies: + vitest: 5.0.5(@opentelemetry/api@1.9.1)(@types/node@24.13.5)(@vitest/coverage-v8@5.0.5)(jsdom@28.1.0)(msw@2.10.2(@types/node@24.13.5)(typescript@5.9.3))(vite@8.3.3(@types/node@24.13.5)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.13)(yaml@2.9.0)) + transitivePeerDependencies: + - '@babel/core' + - rollup + - webpack + '@sentry/search-syntax@1.0.0': {} '@sentry/server-runtime-injection@11.6.0': diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 0b84f03e9..ab0fc0638 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -57,6 +57,7 @@ catalog: "@sentry/core": 11.6.0 "@sentry/node": 11.6.0 "@sentry/react": 11.6.0 + "@sentry/roach": 0.2.0 "@sentry/search-syntax": ^1.0.0 "@sentry/vite-plugin": ^4.6.1 "@tailwindcss/typography": ^0.5.16 @@ -124,6 +125,7 @@ minimumReleaseAgeExclude: - "@sentry/react@11.6.0" - "@sentry/replay-canvas@11.6.0" - "@sentry/replay@11.6.0" + - "@sentry/roach@0.2.0" - "@sentry/search-syntax@1.0.0" - "@sentry/server-runtime-injection@11.6.0" - "@sentry/server-utils@11.6.0" From b34c56c0ea6c7b85dfb2c957fc2c973a3d059214 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Thu, 5 Nov 2026 19:56:01 +0000 Subject: [PATCH 2/2] build(evals): Use @sentry/roach 0.3.0 0.3.0 has the client changes the evals need, so drop the git dependency on unreleased roach main. --- pnpm-lock.yaml | 18 ++++++------------ pnpm-workspace.yaml | 4 ++-- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d3e7a4cb1..a28c21b1e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -58,8 +58,8 @@ catalogs: specifier: 11.6.0 version: 11.6.0 '@sentry/roach': - specifier: 0.2.0 - version: 0.2.0 + specifier: 0.3.0 + version: 0.3.0 '@sentry/search-syntax': specifier: ^1.0.0 version: 1.0.0 @@ -961,7 +961,7 @@ importers: devDependencies: '@sentry/roach': specifier: 'catalog:' - version: 0.2.0(vitest@5.0.5) + version: 0.3.0(vitest@5.0.5) packages/mcp-server-mocks: dependencies: @@ -4683,8 +4683,8 @@ packages: resolution: {integrity: sha512-cJyPYnF6QJA8ADRfAW06YoU7EVIebtWezBOCU9Vag5+W1VLBkXEvH7C4LbeMUiDmrqw92sumq06NgqipIxA7BA==} engines: {node: '>=20.19.0 <22.0.0 || >=22.12.0 <23.0.0 || >=23.2.0'} - '@sentry/roach@0.2.0': - resolution: {integrity: sha512-F9DOaNnqJ1PzPzxcl+nexu10yz6LXxQOCrnHVCoAlUd+lBTMIm0zmNDO7L8X3KsotpxTUw3ROMhQDVkLhzdTrA==} + '@sentry/roach@0.3.0': + resolution: {integrity: sha512-vgYiE3t/Cd5Ruff0KdCNktRNFYoXJTj10RUlcgqqhuMYTBbGlY+e/pU48M02WuueoQWx6zRZMm76jQRJQwp7KA==} engines: {node: '>=24'} peerDependencies: vitest: '>=3' @@ -12677,15 +12677,9 @@ snapshots: '@sentry/browser-utils': 11.6.0 '@sentry/core': 11.6.0 - '@sentry/roach@0.2.0(vitest@5.0.5)': - dependencies: - '@sentry/node': 11.6.0(@babel/core@7.28.0)(rollup@4.62.4) + '@sentry/roach@0.3.0(vitest@5.0.5)': optionalDependencies: vitest: 5.0.5(@opentelemetry/api@1.9.1)(@types/node@24.13.5)(@vitest/coverage-v8@5.0.5)(jsdom@28.1.0)(msw@2.10.2(@types/node@24.13.5)(typescript@5.9.3))(vite@8.3.3(@types/node@24.13.5)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.13)(yaml@2.9.0)) - transitivePeerDependencies: - - '@babel/core' - - rollup - - webpack '@sentry/search-syntax@1.0.0': {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index ab0fc0638..8686bd37d 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -57,7 +57,7 @@ catalog: "@sentry/core": 11.6.0 "@sentry/node": 11.6.0 "@sentry/react": 11.6.0 - "@sentry/roach": 0.2.0 + "@sentry/roach": 0.3.0 "@sentry/search-syntax": ^1.0.0 "@sentry/vite-plugin": ^4.6.1 "@tailwindcss/typography": ^0.5.16 @@ -125,7 +125,7 @@ minimumReleaseAgeExclude: - "@sentry/react@11.6.0" - "@sentry/replay-canvas@11.6.0" - "@sentry/replay@11.6.0" - - "@sentry/roach@0.2.0" + - "@sentry/roach@0.3.0" - "@sentry/search-syntax@1.0.0" - "@sentry/server-runtime-injection@11.6.0" - "@sentry/server-utils@11.6.0"