From 198335daeac199610863f2ba7d796ba6322df74f Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Thu, 5 Nov 2026 04:55:50 +0000 Subject: [PATCH] test(evals): Migrate evals to the vitest-evals harness API Move every eval suite off `vitest-evals/legacy` onto the root harness API: `describeEval(name, { harness, judges, judgeThreshold }, (it) => ...)`. Each eval case is the harness input. The task only receives the prompt; suite-level judges read `expectedTools` and `expected` from the case, so one `run()` per case still averages every judge against the suite threshold, as the scorer-first API did. - `createTaskHarness` adapts the existing task runners into a harness with a normalized transcript (prompt, tool calls, output). - `ToolPredictionScorer` and the local `StructuredOutputScorer` become judges with unchanged prompts and scoring. - `ToolCallJudge` wraps the built-in judge, which shares the legacy `ToolCallScorer` implementation. Cases, expected tool calls, structured-output checks, thresholds and timeouts are unchanged. The harness design follows #1132. Co-Authored-By: David Cramer <23610+dcramer@users.noreply.github.com> --- docs/contributing/adding-tools.md | 28 +- docs/testing/overview.md | 52 +- .../src/evals/autofix.eval.ts | 67 +- .../src/evals/create-dsn.eval.ts | 53 +- .../src/evals/create-project.eval.ts | 77 +- .../src/evals/create-team.eval.ts | 61 +- .../src/evals/get-issue.eval.ts | 71 +- .../src/evals/get-sentry-resource.eval.ts | 123 +-- .../src/evals/get-trace-details.eval.ts | 71 +- .../src/evals/list-dsns.eval.ts | 51 +- .../src/evals/list-issues.eval.ts | 173 ++-- .../src/evals/list-organizations.eval.ts | 44 +- .../src/evals/list-projects.eval.ts | 59 +- .../src/evals/list-releases.eval.ts | 103 +-- .../src/evals/list-tags.eval.ts | 59 +- .../src/evals/list-teams.eval.ts | 115 +-- .../src/evals/natural-language-search.eval.ts | 255 +++--- .../src/evals/search-docs.eval.ts | 94 ++- ...rch-events-agent-attribute-context.eval.ts | 775 +++++++++--------- .../evals/search-events-agent-regex.eval.ts | 354 ++++---- .../src/evals/search-events-agent.eval.ts | 478 +++++------ .../evals/search-events-environments.eval.ts | 178 ++-- .../src/evals/search-events.eval.ts | 191 +++-- .../evals/search-issue-events-agent.eval.ts | 235 +++--- .../src/evals/search-issue-events.eval.ts | 159 ++-- .../src/evals/search-issues-agent.eval.ts | 289 +++---- .../src/evals/search-issues.eval.ts | 161 ++-- .../src/evals/update-issue.eval.ts | 231 +++--- .../src/evals/update-project.eval.ts | 77 +- .../src/evals/utils/harness.ts | 95 +++ .../mcp-server-evals/src/evals/utils/index.ts | 13 +- .../src/evals/utils/runner.ts | 2 +- ...tputScorer.ts => structuredOutputJudge.ts} | 239 +++--- .../src/evals/utils/toolCallJudge.ts | 22 + ...ictionScorer.ts => toolPredictionJudge.ts} | 180 ++-- 35 files changed, 2817 insertions(+), 2418 deletions(-) create mode 100644 packages/mcp-server-evals/src/evals/utils/harness.ts rename packages/mcp-server-evals/src/evals/utils/{structuredOutputScorer.ts => structuredOutputJudge.ts} (51%) create mode 100644 packages/mcp-server-evals/src/evals/utils/toolCallJudge.ts rename packages/mcp-server-evals/src/evals/utils/{toolPredictionScorer.ts => toolPredictionJudge.ts} (69%) diff --git a/docs/contributing/adding-tools.md b/docs/contributing/adding-tools.md index 8ada19eb0..9aeed33df 100644 --- a/docs/contributing/adding-tools.md +++ b/docs/contributing/adding-tools.md @@ -296,18 +296,22 @@ See [api-patterns.md](api-patterns.md#mock-patterns) for validation examples. **⚠️ Each eval costs time and API credits. Only test core functionality!** ```typescript -describeEval("your-tool", { - data: async () => [ - { - input: `Primary use case in ${FIXTURES.organizationSlug}`, - expected: "Expected response", - }, - // Maximum 2-3 scenarios! - ], - task: TaskRunner(), - scorers: [Factuality()], - threshold: 0.6, -}); +describeEval( + "your-tool", + { + harness: createTaskHarness("your-tool", NoOpTaskRunner()), + judges: [ToolPredictionJudge()], + judgeThreshold: 0.6, + }, + (it) => + defineEvalCases(it, [ + { + input: `Primary use case in ${FIXTURES.organizationSlug}`, + expectedTools: [{ name: "your_tool", arguments: {} }], + }, + // Maximum 2-3 scenarios! + ]), +); ``` ## Testing Workflow diff --git a/docs/testing/overview.md b/docs/testing/overview.md index 4a7891969..4967d72b1 100644 --- a/docs/testing/overview.md +++ b/docs/testing/overview.md @@ -275,23 +275,45 @@ expect(result.timestamp).toMatchInlineSnapshot(); // ❌ ### Eval Test Structure ```typescript -import { describeEval } from "vitest-evals/legacy"; -import { TaskRunner, Factuality } from "./utils"; - -describeEval("tool-name", { - data: async () => [ - { - input: "Natural language request", - expected: "Expected response content", - }, - ], - task: TaskRunner(), // Uses AI to call tools - scorers: [Factuality()], // Validates output - threshold: 0.6, - timeout: 30000, -}); +import { describeEval } from "vitest-evals"; +import { + createTaskHarness, + defineEvalCases, + FIXTURES, + NoOpTaskRunner, + ToolPredictionJudge, + type EvalCase, +} from "./utils"; + +const cases: EvalCase[] = [ + { + input: `Natural language request in ${FIXTURES.organizationSlug}`, + expectedTools: [ + { + name: "tool_name", + arguments: { organizationSlug: FIXTURES.organizationSlug }, + }, + ], + }, +]; + +describeEval( + "tool-name", + { + // NoOpTaskRunner skips tool execution; the judge predicts the tool calls + harness: createTaskHarness("tool-name", NoOpTaskRunner()), + judges: [ToolPredictionJudge()], + judgeThreshold: 0.6, + }, + (it) => defineEvalCases(it, cases, { timeout: 30000 }), +); ``` +Each case is the harness input. The task receives only `input`; judges read +`expectedTools` and `expected` from the case. Embedded-agent evals pass their +own task to `createTaskHarness` and use `ToolCallJudge` and +`StructuredOutputJudge`. + ### Running Evals ```bash diff --git a/packages/mcp-server-evals/src/evals/autofix.eval.ts b/packages/mcp-server-evals/src/evals/autofix.eval.ts index cbbf20409..2c3ea20d5 100644 --- a/packages/mcp-server-evals/src/evals/autofix.eval.ts +++ b/packages/mcp-server-evals/src/evals/autofix.eval.ts @@ -1,35 +1,44 @@ -import { describeEval } from "vitest-evals/legacy"; -import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils"; +import { describeEval } from "vitest-evals"; +import { + createTaskHarness, + defineEvalCases, + FIXTURES, + NoOpTaskRunner, + ToolPredictionJudge, + type EvalCase, +} from "./utils"; -describeEval("begin-issue-fix", { - data: async () => { - return [ +const cases: EvalCase[] = [ + { + input: `Whats the status on root causing this issue in Sentry?\n${FIXTURES.testIssueUrl}`, + expectedTools: [ { - input: `Whats the status on root causing this issue in Sentry?\n${FIXTURES.testIssueUrl}`, - expectedTools: [ - { - name: "analyze_issue_with_seer", - arguments: { - issueUrl: FIXTURES.testIssueUrl, - }, - }, - ], + name: "analyze_issue_with_seer", + arguments: { + issueUrl: FIXTURES.testIssueUrl, + }, }, + ], + }, + { + input: `Can you root cause this issue and retrieve the analysis?\n${FIXTURES.testIssueUrl}`, + expectedTools: [ { - input: `Can you root cause this issue and retrieve the analysis?\n${FIXTURES.testIssueUrl}`, - expectedTools: [ - { - name: "analyze_issue_with_seer", - arguments: { - issueUrl: FIXTURES.testIssueUrl, - }, - }, - ], + name: "analyze_issue_with_seer", + arguments: { + issueUrl: FIXTURES.testIssueUrl, + }, }, - ]; + ], + }, +]; + +describeEval( + "begin-issue-fix", + { + harness: createTaskHarness("begin-issue-fix", NoOpTaskRunner()), + judges: [ToolPredictionJudge()], + judgeThreshold: 0.6, }, - task: NoOpTaskRunner(), - scorers: [ToolPredictionScorer()], - threshold: 0.6, - timeout: 30000, -}); + (it) => defineEvalCases(it, cases, { timeout: 30000 }), +); diff --git a/packages/mcp-server-evals/src/evals/create-dsn.eval.ts b/packages/mcp-server-evals/src/evals/create-dsn.eval.ts index d7cf92177..37248b5fb 100644 --- a/packages/mcp-server-evals/src/evals/create-dsn.eval.ts +++ b/packages/mcp-server-evals/src/evals/create-dsn.eval.ts @@ -1,26 +1,35 @@ -import { describeEval } from "vitest-evals/legacy"; -import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils"; +import { describeEval } from "vitest-evals"; +import { + createTaskHarness, + defineEvalCases, + FIXTURES, + NoOpTaskRunner, + ToolPredictionJudge, + type EvalCase, +} from "./utils"; -describeEval("create-dsn", { - data: async () => { - return [ +const cases: EvalCase[] = [ + { + input: `Create a new DSN named "Production" for '${FIXTURES.organizationSlug}/${FIXTURES.projectSlug}'`, + expectedTools: [ { - input: `Create a new DSN named "Production" for '${FIXTURES.organizationSlug}/${FIXTURES.projectSlug}'`, - expectedTools: [ - { - name: "create_dsn", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - projectSlug: FIXTURES.projectSlug, - name: "Production", - }, - }, - ], + name: "create_dsn", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + projectSlug: FIXTURES.projectSlug, + name: "Production", + }, }, - ]; + ], }, - task: NoOpTaskRunner(), - scorers: [ToolPredictionScorer()], - threshold: 0.6, - timeout: 30000, -}); +]; + +describeEval( + "create-dsn", + { + harness: createTaskHarness("create-dsn", NoOpTaskRunner()), + judges: [ToolPredictionJudge()], + judgeThreshold: 0.6, + }, + (it) => defineEvalCases(it, cases, { timeout: 30000 }), +); diff --git a/packages/mcp-server-evals/src/evals/create-project.eval.ts b/packages/mcp-server-evals/src/evals/create-project.eval.ts index 1eebc6eec..f0c7d50a9 100644 --- a/packages/mcp-server-evals/src/evals/create-project.eval.ts +++ b/packages/mcp-server-evals/src/evals/create-project.eval.ts @@ -1,38 +1,47 @@ -import { describeEval } from "vitest-evals/legacy"; -import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils"; +import { describeEval } from "vitest-evals"; +import { + createTaskHarness, + defineEvalCases, + FIXTURES, + NoOpTaskRunner, + ToolPredictionJudge, + type EvalCase, +} from "./utils"; -describeEval("create-project", { - data: async () => { - return [ +const cases: EvalCase[] = [ + { + input: `Create a new project in Sentry for '${FIXTURES.organizationSlug}' called '${FIXTURES.projectSlug}' with the '${FIXTURES.teamSlug}' team. Output **only** the project slug and the SENTRY_DSN in the format of:\n\n`, + expectedTools: [ { - input: `Create a new project in Sentry for '${FIXTURES.organizationSlug}' called '${FIXTURES.projectSlug}' with the '${FIXTURES.teamSlug}' team. Output **only** the project slug and the SENTRY_DSN in the format of:\n\n`, - expectedTools: [ - { - name: "find_organizations", - arguments: {}, - }, - { - name: "find_teams", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - regionUrl: "https://us.sentry.io", - }, - }, - { - name: "create_project", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - regionUrl: "https://us.sentry.io", - teamSlug: FIXTURES.teamSlug, - name: FIXTURES.projectSlug, - }, - }, - ], + name: "find_organizations", + arguments: {}, }, - ]; + { + name: "find_teams", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + regionUrl: "https://us.sentry.io", + }, + }, + { + name: "create_project", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + regionUrl: "https://us.sentry.io", + teamSlug: FIXTURES.teamSlug, + name: FIXTURES.projectSlug, + }, + }, + ], + }, +]; + +describeEval( + "create-project", + { + harness: createTaskHarness("create-project", NoOpTaskRunner()), + judges: [ToolPredictionJudge()], + judgeThreshold: 0.6, }, - task: NoOpTaskRunner(), - scorers: [ToolPredictionScorer()], - threshold: 0.6, - timeout: 30000, -}); + (it) => defineEvalCases(it, cases, { timeout: 30000 }), +); diff --git a/packages/mcp-server-evals/src/evals/create-team.eval.ts b/packages/mcp-server-evals/src/evals/create-team.eval.ts index 777cd6608..34a8fe08e 100644 --- a/packages/mcp-server-evals/src/evals/create-team.eval.ts +++ b/packages/mcp-server-evals/src/evals/create-team.eval.ts @@ -1,30 +1,39 @@ -import { describeEval } from "vitest-evals/legacy"; -import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils"; +import { describeEval } from "vitest-evals"; +import { + createTaskHarness, + defineEvalCases, + FIXTURES, + NoOpTaskRunner, + ToolPredictionJudge, + type EvalCase, +} from "./utils"; -describeEval("create-team", { - data: async () => { - return [ +const cases: EvalCase[] = [ + { + input: `Create a new team in Sentry for '${FIXTURES.organizationSlug}' called 'the-goats' response with **only** the team slug and no other text.`, + expectedTools: [ { - input: `Create a new team in Sentry for '${FIXTURES.organizationSlug}' called 'the-goats' response with **only** the team slug and no other text.`, - expectedTools: [ - { - name: "find_organizations", - arguments: {}, - }, - { - name: "create_team", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - name: "the-goats", - regionUrl: "https://us.sentry.io", - }, - }, - ], + name: "find_organizations", + arguments: {}, }, - ]; + { + name: "create_team", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + name: "the-goats", + regionUrl: "https://us.sentry.io", + }, + }, + ], + }, +]; + +describeEval( + "create-team", + { + harness: createTaskHarness("create-team", NoOpTaskRunner()), + judges: [ToolPredictionJudge()], + judgeThreshold: 0.6, }, - task: NoOpTaskRunner(), - scorers: [ToolPredictionScorer()], - threshold: 0.6, - timeout: 30000, -}); + (it) => defineEvalCases(it, cases, { timeout: 30000 }), +); diff --git a/packages/mcp-server-evals/src/evals/get-issue.eval.ts b/packages/mcp-server-evals/src/evals/get-issue.eval.ts index 658ca9d75..5b4d82c6e 100644 --- a/packages/mcp-server-evals/src/evals/get-issue.eval.ts +++ b/packages/mcp-server-evals/src/evals/get-issue.eval.ts @@ -1,37 +1,46 @@ -import { describeEval, ToolCallScorer } from "vitest-evals/legacy"; -import { FIXTURES, McpToolCallTaskRunner } from "./utils"; +import { describeEval } from "vitest-evals"; +import { + createTaskHarness, + defineEvalCases, + FIXTURES, + McpToolCallTaskRunner, + ToolCallJudge, + type EvalCase, +} from "./utils"; -describeEval("get-issue", { - data: async () => { - return [ +const cases: EvalCase[] = [ + { + input: `Explain CLOUDFLARE-MCP-41 from Sentry in ${FIXTURES.organizationSlug}.`, + expectedTools: [ { - input: `Explain CLOUDFLARE-MCP-41 from Sentry in ${FIXTURES.organizationSlug}.`, - expectedTools: [ - { - name: "get_issue_details", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - issueId: "CLOUDFLARE-MCP-41", - }, - }, - ], + name: "get_issue_details", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + issueId: "CLOUDFLARE-MCP-41", + }, }, + ], + }, + { + input: `Explain the event with ID 7ca573c0f4814912aaa9bdc77d1a7d51 from Sentry in ${FIXTURES.organizationSlug}.`, + expectedTools: [ { - input: `Explain the event with ID 7ca573c0f4814912aaa9bdc77d1a7d51 from Sentry in ${FIXTURES.organizationSlug}.`, - expectedTools: [ - { - name: "get_issue_details", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - eventId: "7ca573c0f4814912aaa9bdc77d1a7d51", - }, - }, - ], + name: "get_issue_details", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + eventId: "7ca573c0f4814912aaa9bdc77d1a7d51", + }, }, - ]; + ], + }, +]; + +describeEval( + "get-issue", + { + harness: createTaskHarness("get-issue", McpToolCallTaskRunner()), + judges: [ToolCallJudge({ ordered: true, params: "fuzzy" })], + judgeThreshold: 0.6, }, - task: McpToolCallTaskRunner(), - scorers: [ToolCallScorer({ ordered: true, params: "fuzzy" })], - threshold: 0.6, - timeout: 90000, -}); + (it) => defineEvalCases(it, cases, { timeout: 90000 }), +); diff --git a/packages/mcp-server-evals/src/evals/get-sentry-resource.eval.ts b/packages/mcp-server-evals/src/evals/get-sentry-resource.eval.ts index e9f2eeb6a..a908ad8d9 100644 --- a/packages/mcp-server-evals/src/evals/get-sentry-resource.eval.ts +++ b/packages/mcp-server-evals/src/evals/get-sentry-resource.eval.ts @@ -1,69 +1,78 @@ -import { describeEval } from "vitest-evals/legacy"; -import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils"; +import { describeEval } from "vitest-evals"; +import { + createTaskHarness, + defineEvalCases, + FIXTURES, + NoOpTaskRunner, + ToolPredictionJudge, + type EvalCase, +} from "./utils"; -describeEval("get-sentry-resource", { - data: async () => { - return [ +const cases: EvalCase[] = [ + { + input: `What's happening in this Sentry issue? ${FIXTURES.issueUrl}`, + expectedTools: [ { - input: `What's happening in this Sentry issue? ${FIXTURES.issueUrl}`, - expectedTools: [ - { - name: "get_sentry_resource", - arguments: { - url: FIXTURES.issueUrl, - }, - }, - ], + name: "get_sentry_resource", + arguments: { + url: FIXTURES.issueUrl, + }, }, + ], + }, + { + input: `Show me the breadcrumbs for ${FIXTURES.issueUrl}`, + expectedTools: [ { - input: `Show me the breadcrumbs for ${FIXTURES.issueUrl}`, - expectedTools: [ - { - name: "get_issue_breadcrumbs", - arguments: { - issueUrl: FIXTURES.issueUrl, - }, - }, - ], + name: "get_issue_breadcrumbs", + arguments: { + issueUrl: FIXTURES.issueUrl, + }, }, + ], + }, + { + input: `Fetch the breadcrumbs for issue ${FIXTURES.issueId} in ${FIXTURES.organizationSlug}.`, + expectedTools: [ { - input: `Fetch the breadcrumbs for issue ${FIXTURES.issueId} in ${FIXTURES.organizationSlug}.`, - expectedTools: [ - { - name: "get_issue_breadcrumbs", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - issueId: FIXTURES.issueId, - }, - }, - ], + name: "get_issue_breadcrumbs", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + issueId: FIXTURES.issueId, + }, }, + ], + }, + { + input: `Show me what happened in this trace: ${FIXTURES.traceUrl}`, + expectedTools: [ { - input: `Show me what happened in this trace: ${FIXTURES.traceUrl}`, - expectedTools: [ - { - name: "get_sentry_resource", - arguments: { - url: FIXTURES.traceUrl, - }, - }, - ], + name: "get_sentry_resource", + arguments: { + url: FIXTURES.traceUrl, + }, }, + ], + }, + { + input: `What widgets are on this dashboard? ${FIXTURES.dashboardUrl}`, + expectedTools: [ { - input: `What widgets are on this dashboard? ${FIXTURES.dashboardUrl}`, - expectedTools: [ - { - name: "get_sentry_resource", - arguments: { - url: FIXTURES.dashboardUrl, - }, - }, - ], + name: "get_sentry_resource", + arguments: { + url: FIXTURES.dashboardUrl, + }, }, - ]; + ], + }, +]; + +describeEval( + "get-sentry-resource", + { + harness: createTaskHarness("get-sentry-resource", NoOpTaskRunner()), + judges: [ToolPredictionJudge()], + judgeThreshold: 0.6, }, - task: NoOpTaskRunner(), - scorers: [ToolPredictionScorer()], - threshold: 0.6, - timeout: 30000, -}); + (it) => defineEvalCases(it, cases, { timeout: 30000 }), +); diff --git a/packages/mcp-server-evals/src/evals/get-trace-details.eval.ts b/packages/mcp-server-evals/src/evals/get-trace-details.eval.ts index 5fa3f2a1b..599457033 100644 --- a/packages/mcp-server-evals/src/evals/get-trace-details.eval.ts +++ b/packages/mcp-server-evals/src/evals/get-trace-details.eval.ts @@ -1,37 +1,46 @@ -import { describeEval, ToolCallScorer } from "vitest-evals/legacy"; -import { FIXTURES, McpToolCallTaskRunner } from "./utils"; +import { describeEval } from "vitest-evals"; +import { + createTaskHarness, + defineEvalCases, + FIXTURES, + McpToolCallTaskRunner, + ToolCallJudge, + type EvalCase, +} from "./utils"; -describeEval("get-trace-details", { - data: async () => { - return [ +const cases: EvalCase[] = [ + { + input: `Show me trace ${FIXTURES.traceId} from Sentry in ${FIXTURES.organizationSlug}.`, + expectedTools: [ { - input: `Show me trace ${FIXTURES.traceId} from Sentry in ${FIXTURES.organizationSlug}.`, - expectedTools: [ - { - name: "get_trace_details", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - traceId: FIXTURES.traceId, - }, - }, - ], + name: "get_trace_details", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + traceId: FIXTURES.traceId, + }, }, + ], + }, + { + input: `Explain trace ${FIXTURES.traceId} in ${FIXTURES.organizationSlug}.`, + expectedTools: [ { - input: `Explain trace ${FIXTURES.traceId} in ${FIXTURES.organizationSlug}.`, - expectedTools: [ - { - name: "get_trace_details", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - traceId: FIXTURES.traceId, - }, - }, - ], + name: "get_trace_details", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + traceId: FIXTURES.traceId, + }, }, - ]; + ], + }, +]; + +describeEval( + "get-trace-details", + { + harness: createTaskHarness("get-trace-details", McpToolCallTaskRunner()), + judges: [ToolCallJudge({ ordered: true, params: "fuzzy" })], + judgeThreshold: 0.6, }, - task: McpToolCallTaskRunner(), - scorers: [ToolCallScorer({ ordered: true, params: "fuzzy" })], - threshold: 0.6, - timeout: 90000, -}); + (it) => defineEvalCases(it, cases, { timeout: 90000 }), +); diff --git a/packages/mcp-server-evals/src/evals/list-dsns.eval.ts b/packages/mcp-server-evals/src/evals/list-dsns.eval.ts index 2500148be..7c802e942 100644 --- a/packages/mcp-server-evals/src/evals/list-dsns.eval.ts +++ b/packages/mcp-server-evals/src/evals/list-dsns.eval.ts @@ -1,25 +1,34 @@ -import { describeEval } from "vitest-evals/legacy"; -import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils"; +import { describeEval } from "vitest-evals"; +import { + createTaskHarness, + defineEvalCases, + FIXTURES, + NoOpTaskRunner, + ToolPredictionJudge, + type EvalCase, +} from "./utils"; -describeEval("list-dsns", { - data: async () => { - return [ +const cases: EvalCase[] = [ + { + input: `What is the SENTRY_DSN for ${FIXTURES.organizationSlug}/${FIXTURES.projectSlug}?`, + expectedTools: [ { - input: `What is the SENTRY_DSN for ${FIXTURES.organizationSlug}/${FIXTURES.projectSlug}?`, - expectedTools: [ - { - name: "find_dsns", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - projectSlug: FIXTURES.projectSlug, - }, - }, - ], + name: "find_dsns", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + projectSlug: FIXTURES.projectSlug, + }, }, - ]; + ], }, - task: NoOpTaskRunner(), - scorers: [ToolPredictionScorer()], - threshold: 0.6, - timeout: 30000, -}); +]; + +describeEval( + "list-dsns", + { + harness: createTaskHarness("list-dsns", NoOpTaskRunner()), + judges: [ToolPredictionJudge()], + judgeThreshold: 0.6, + }, + (it) => defineEvalCases(it, cases, { timeout: 30000 }), +); diff --git a/packages/mcp-server-evals/src/evals/list-issues.eval.ts b/packages/mcp-server-evals/src/evals/list-issues.eval.ts index 373c9d4fe..7757611da 100644 --- a/packages/mcp-server-evals/src/evals/list-issues.eval.ts +++ b/packages/mcp-server-evals/src/evals/list-issues.eval.ts @@ -1,94 +1,103 @@ -import { describeEval } from "vitest-evals/legacy"; -import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils"; +import { describeEval } from "vitest-evals"; +import { + createTaskHarness, + defineEvalCases, + FIXTURES, + NoOpTaskRunner, + ToolPredictionJudge, + type EvalCase, +} from "./utils"; -describeEval("list-issues", { - data: async () => { - return [ +const cases: EvalCase[] = [ + { + input: `What are the most common production errors in ${FIXTURES.organizationSlug}?`, + expectedTools: [ { - input: `What are the most common production errors in ${FIXTURES.organizationSlug}?`, - expectedTools: [ - { - name: "find_organizations", - arguments: {}, - }, - { - name: "search_issues", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - query: "is:unresolved", - sort: "freq", - }, - }, - ], + name: "find_organizations", + arguments: {}, }, { - input: `Show me the top issues in ${FIXTURES.organizationSlug} organization`, - expectedTools: [ - { - name: "find_organizations", - arguments: {}, - }, - { - name: "search_issues", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - sort: "freq", - }, - }, - ], + name: "search_issues", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + query: "is:unresolved", + sort: "freq", + }, + }, + ], + }, + { + input: `Show me the top issues in ${FIXTURES.organizationSlug} organization`, + expectedTools: [ + { + name: "find_organizations", + arguments: {}, + }, + { + name: "search_issues", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + sort: "freq", + }, + }, + ], + }, + { + input: `What are the most recent issues in ${FIXTURES.organizationSlug}?`, + expectedTools: [ + { + name: "find_organizations", + arguments: {}, }, { - input: `What are the most recent issues in ${FIXTURES.organizationSlug}?`, - expectedTools: [ - { - name: "find_organizations", - arguments: {}, - }, - { - name: "search_issues", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - sort: "date", - }, - }, - ], + name: "search_issues", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + sort: "date", + }, }, + ], + }, + { + input: `Find the newest production issues in ${FIXTURES.organizationSlug}`, + expectedTools: [ + { + name: "find_organizations", + arguments: {}, + }, + { + name: "search_issues", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + sort: "new", + }, + }, + ], + }, + { + input: `What issues is david@sentry.io experiencing in ${FIXTURES.organizationSlug}?`, + expectedTools: [ { - input: `Find the newest production issues in ${FIXTURES.organizationSlug}`, - expectedTools: [ - { - name: "find_organizations", - arguments: {}, - }, - { - name: "search_issues", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - sort: "new", - }, - }, - ], + name: "find_organizations", + arguments: {}, }, { - input: `What issues is david@sentry.io experiencing in ${FIXTURES.organizationSlug}?`, - expectedTools: [ - { - name: "find_organizations", - arguments: {}, - }, - { - name: "search_issues", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - query: "user.email:david@sentry.io", - }, - }, - ], + name: "search_issues", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + query: "user.email:david@sentry.io", + }, }, - ]; + ], + }, +]; + +describeEval( + "list-issues", + { + harness: createTaskHarness("list-issues", NoOpTaskRunner()), + judges: [ToolPredictionJudge()], + judgeThreshold: 0.6, }, - task: NoOpTaskRunner(), - scorers: [ToolPredictionScorer()], - threshold: 0.6, - timeout: 30000, -}); + (it) => defineEvalCases(it, cases, { timeout: 30000 }), +); diff --git a/packages/mcp-server-evals/src/evals/list-organizations.eval.ts b/packages/mcp-server-evals/src/evals/list-organizations.eval.ts index cc549fd39..a57c5b51a 100644 --- a/packages/mcp-server-evals/src/evals/list-organizations.eval.ts +++ b/packages/mcp-server-evals/src/evals/list-organizations.eval.ts @@ -1,22 +1,30 @@ -import { describeEval } from "vitest-evals/legacy"; -import { NoOpTaskRunner, ToolPredictionScorer } from "./utils"; +import { describeEval } from "vitest-evals"; +import { + createTaskHarness, + defineEvalCases, + NoOpTaskRunner, + ToolPredictionJudge, + type EvalCase, +} from "./utils"; -describeEval("list-organizations", { - data: async () => { - return [ +const cases: EvalCase[] = [ + { + input: `What organizations do I have access to in Sentry`, + expectedTools: [ { - input: `What organizations do I have access to in Sentry`, - expectedTools: [ - { - name: "find_organizations", - arguments: {}, - }, - ], + name: "find_organizations", + arguments: {}, }, - ]; + ], }, - task: NoOpTaskRunner(), - scorers: [ToolPredictionScorer()], - threshold: 0.6, - timeout: 30000, -}); +]; + +describeEval( + "list-organizations", + { + harness: createTaskHarness("list-organizations", NoOpTaskRunner()), + judges: [ToolPredictionJudge()], + judgeThreshold: 0.6, + }, + (it) => defineEvalCases(it, cases, { timeout: 30000 }), +); diff --git a/packages/mcp-server-evals/src/evals/list-projects.eval.ts b/packages/mcp-server-evals/src/evals/list-projects.eval.ts index 1160d08ab..4995e4abf 100644 --- a/packages/mcp-server-evals/src/evals/list-projects.eval.ts +++ b/packages/mcp-server-evals/src/evals/list-projects.eval.ts @@ -1,29 +1,38 @@ -import { describeEval } from "vitest-evals/legacy"; -import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils"; +import { describeEval } from "vitest-evals"; +import { + createTaskHarness, + defineEvalCases, + FIXTURES, + NoOpTaskRunner, + ToolPredictionJudge, + type EvalCase, +} from "./utils"; -describeEval("list-projects", { - data: async () => { - return [ +const cases: EvalCase[] = [ + { + input: `What projects do I have access to in Sentry for '${FIXTURES.organizationSlug}'`, + expectedTools: [ { - input: `What projects do I have access to in Sentry for '${FIXTURES.organizationSlug}'`, - expectedTools: [ - { - name: "find_organizations", - arguments: {}, - }, - { - name: "find_projects", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - regionUrl: "https://us.sentry.io", - }, - }, - ], + name: "find_organizations", + arguments: {}, }, - ]; + { + name: "find_projects", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + regionUrl: "https://us.sentry.io", + }, + }, + ], + }, +]; + +describeEval( + "list-projects", + { + harness: createTaskHarness("list-projects", NoOpTaskRunner()), + judges: [ToolPredictionJudge()], + judgeThreshold: 0.6, }, - task: NoOpTaskRunner(), - scorers: [ToolPredictionScorer()], - threshold: 0.6, - timeout: 30000, -}); + (it) => defineEvalCases(it, cases, { timeout: 30000 }), +); diff --git a/packages/mcp-server-evals/src/evals/list-releases.eval.ts b/packages/mcp-server-evals/src/evals/list-releases.eval.ts index c0aef147a..b8d6d984f 100644 --- a/packages/mcp-server-evals/src/evals/list-releases.eval.ts +++ b/packages/mcp-server-evals/src/evals/list-releases.eval.ts @@ -1,53 +1,62 @@ -import { describeEval } from "vitest-evals/legacy"; -import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils"; +import { describeEval } from "vitest-evals"; +import { + createTaskHarness, + defineEvalCases, + FIXTURES, + NoOpTaskRunner, + ToolPredictionJudge, + type EvalCase, +} from "./utils"; -describeEval("list-releases", { - data: async () => { - return [ +const cases: EvalCase[] = [ + { + input: `Show me the releases in ${FIXTURES.organizationSlug}`, + expectedTools: [ { - input: `Show me the releases in ${FIXTURES.organizationSlug}`, - expectedTools: [ - { - name: "find_organizations", - arguments: {}, - }, - { - name: "find_releases", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - regionUrl: "https://us.sentry.io", - }, - }, - ], + name: "find_organizations", + arguments: {}, }, { - input: `Show me a list of versions in ${FIXTURES.organizationSlug}/${FIXTURES.projectSlug}`, - expectedTools: [ - { - name: "find_organizations", - arguments: {}, - }, - { - name: "find_projects", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - regionUrl: "https://us.sentry.io", - }, - }, - { - name: "find_releases", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - projectSlug: FIXTURES.projectSlug, - regionUrl: "https://us.sentry.io", - }, - }, - ], + name: "find_releases", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + regionUrl: "https://us.sentry.io", + }, }, - ]; + ], }, - task: NoOpTaskRunner(), - scorers: [ToolPredictionScorer()], - threshold: 0.6, - timeout: 30000, -}); + { + input: `Show me a list of versions in ${FIXTURES.organizationSlug}/${FIXTURES.projectSlug}`, + expectedTools: [ + { + name: "find_organizations", + arguments: {}, + }, + { + name: "find_projects", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + regionUrl: "https://us.sentry.io", + }, + }, + { + name: "find_releases", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + projectSlug: FIXTURES.projectSlug, + regionUrl: "https://us.sentry.io", + }, + }, + ], + }, +]; + +describeEval( + "list-releases", + { + harness: createTaskHarness("list-releases", NoOpTaskRunner()), + judges: [ToolPredictionJudge()], + judgeThreshold: 0.6, + }, + (it) => defineEvalCases(it, cases, { timeout: 30000 }), +); diff --git a/packages/mcp-server-evals/src/evals/list-tags.eval.ts b/packages/mcp-server-evals/src/evals/list-tags.eval.ts index 34981578f..07b9a82d0 100644 --- a/packages/mcp-server-evals/src/evals/list-tags.eval.ts +++ b/packages/mcp-server-evals/src/evals/list-tags.eval.ts @@ -1,29 +1,38 @@ -import { describeEval } from "vitest-evals/legacy"; -import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils"; +import { describeEval } from "vitest-evals"; +import { + createTaskHarness, + defineEvalCases, + FIXTURES, + NoOpTaskRunner, + ToolPredictionJudge, + type EvalCase, +} from "./utils"; -describeEval("list-tags", { - data: async () => { - return [ +const cases: EvalCase[] = [ + { + input: `What are common tags in ${FIXTURES.organizationSlug}`, + expectedTools: [ { - input: `What are common tags in ${FIXTURES.organizationSlug}`, - expectedTools: [ - { - name: "find_organizations", - arguments: {}, - }, - { - name: "find_tags", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - regionUrl: "https://us.sentry.io", - }, - }, - ], + name: "find_organizations", + arguments: {}, }, - ]; + { + name: "find_tags", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + regionUrl: "https://us.sentry.io", + }, + }, + ], + }, +]; + +describeEval( + "list-tags", + { + harness: createTaskHarness("list-tags", NoOpTaskRunner()), + judges: [ToolPredictionJudge()], + judgeThreshold: 0.6, }, - task: NoOpTaskRunner(), - scorers: [ToolPredictionScorer()], - threshold: 0.6, - timeout: 30000, -}); + (it) => defineEvalCases(it, cases, { timeout: 30000 }), +); diff --git a/packages/mcp-server-evals/src/evals/list-teams.eval.ts b/packages/mcp-server-evals/src/evals/list-teams.eval.ts index 0bc17940a..c6706b6aa 100644 --- a/packages/mcp-server-evals/src/evals/list-teams.eval.ts +++ b/packages/mcp-server-evals/src/evals/list-teams.eval.ts @@ -1,61 +1,70 @@ -import { describeEval } from "vitest-evals/legacy"; -import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils"; +import { describeEval } from "vitest-evals"; +import { + createTaskHarness, + defineEvalCases, + FIXTURES, + NoOpTaskRunner, + ToolPredictionJudge, + type EvalCase, +} from "./utils"; -describeEval("list-teams", { - data: async () => { - return [ +const cases: EvalCase[] = [ + { + input: `What teams do I have access to in Sentry for '${FIXTURES.organizationSlug}'`, + expectedTools: [ { - input: `What teams do I have access to in Sentry for '${FIXTURES.organizationSlug}'`, - expectedTools: [ - { - name: "find_organizations", - arguments: {}, - }, - { - name: "find_teams", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - regionUrl: "https://us.sentry.io", - }, - }, - ], + name: "find_organizations", + arguments: {}, }, { - input: `Do I have access to the team '${FIXTURES.teamSlug}' for '${FIXTURES.organizationSlug}'`, - expectedTools: [ - { - name: "find_organizations", - arguments: {}, - }, - { - name: "find_teams", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - regionUrl: "https://us.sentry.io", - }, - }, - ], + name: "find_teams", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + regionUrl: "https://us.sentry.io", + }, + }, + ], + }, + { + input: `Do I have access to the team '${FIXTURES.teamSlug}' for '${FIXTURES.organizationSlug}'`, + expectedTools: [ + { + name: "find_organizations", + arguments: {}, + }, + { + name: "find_teams", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + regionUrl: "https://us.sentry.io", + }, + }, + ], + }, + { + input: `Do I have access to the team 'an-imaginary-team' for '${FIXTURES.organizationSlug}'`, + expectedTools: [ + { + name: "find_organizations", + arguments: {}, }, { - input: `Do I have access to the team 'an-imaginary-team' for '${FIXTURES.organizationSlug}'`, - expectedTools: [ - { - name: "find_organizations", - arguments: {}, - }, - { - name: "find_teams", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - regionUrl: "https://us.sentry.io", - }, - }, - ], + name: "find_teams", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + regionUrl: "https://us.sentry.io", + }, }, - ]; + ], + }, +]; + +describeEval( + "list-teams", + { + harness: createTaskHarness("list-teams", NoOpTaskRunner()), + judges: [ToolPredictionJudge()], + judgeThreshold: 0.6, }, - task: NoOpTaskRunner(), - scorers: [ToolPredictionScorer()], - threshold: 0.6, - timeout: 30000, -}); + (it) => defineEvalCases(it, cases, { timeout: 30000 }), +); diff --git a/packages/mcp-server-evals/src/evals/natural-language-search.eval.ts b/packages/mcp-server-evals/src/evals/natural-language-search.eval.ts index 9ea93b06d..f35244fa9 100644 --- a/packages/mcp-server-evals/src/evals/natural-language-search.eval.ts +++ b/packages/mcp-server-evals/src/evals/natural-language-search.eval.ts @@ -1,9 +1,15 @@ import { SentryApiService } from "@sentry/mcp-core/api-client"; import { searchEventsAgent } from "@sentry/mcp-core/tools/search-events/agent"; import { searchIssuesAgent } from "@sentry/mcp-core/tools/search-issues/agent"; -import { describeEval, ToolCallScorer } from "vitest-evals/legacy"; import "../setup-env"; -import { StructuredOutputScorer } from "./utils/structuredOutputScorer"; +import { describeEval } from "vitest-evals"; +import { + createTaskHarness, + defineEvalCases, + StructuredOutputJudge, + ToolCallJudge, + type EvalCase, +} from "./utils"; function toToolArguments(args: unknown): Record { return args && typeof args === "object" && !Array.isArray(args) @@ -11,129 +17,144 @@ function toToolArguments(args: unknown): Record { : {}; } -describeEval("natural-language-search-events", { - data: async () => [ - { - input: - "Count unhandled error events by environment over the last 24 hours", - expectedTools: [], - expected: { - dataset: "errors", - query: /error\.handled:false|error\.unhandled:true/, - fields: (value: unknown) => - Array.isArray(value) && - value.includes("environment") && - value.includes("count()"), - sort: "-count()", - timeRange: { statsPeriod: "24h" }, - }, +const naturalLanguageSearchEventsCases: EvalCase[] = [ + { + input: "Count unhandled error events by environment over the last 24 hours", + expectedTools: [], + expected: { + dataset: "errors", + query: /error\.handled:false|error\.unhandled:true/, + fields: (value: unknown) => + Array.isArray(value) && + value.includes("environment") && + value.includes("count()"), + sort: "-count()", + timeRange: { statsPeriod: "24h" }, }, - { - input: "Show warning logs from production over the last 7 days", - expectedTools: [], - expected: { - dataset: "logs", - query: (value: unknown) => - typeof value === "string" && - /(?:severity|level):warn(?:ing)?/.test(value) && - value.includes("environment:production"), - sort: "-timestamp", - timeRange: { statsPeriod: "7d" }, - }, + }, + { + input: "Show warning logs from production over the last 7 days", + expectedTools: [], + expected: { + dataset: "logs", + query: (value: unknown) => + typeof value === "string" && + /(?:severity|level):warn(?:ing)?/.test(value) && + value.includes("environment:production"), + sort: "-timestamp", + timeRange: { statsPeriod: "7d" }, }, - { - input: - "In spans, show p95 span duration grouped by span.op over the last 7 days", - expectedTools: [], - expected: { - dataset: "spans", - query: "", - fields: (value: unknown) => - Array.isArray(value) && - value.includes("span.op") && - value.includes("p95(span.duration)"), - sort: "-p95(span.duration)", - timeRange: { statsPeriod: "7d" }, - }, + }, + { + input: + "In spans, show p95 span duration grouped by span.op over the last 7 days", + expectedTools: [], + expected: { + dataset: "spans", + query: "", + fields: (value: unknown) => + Array.isArray(value) && + value.includes("span.op") && + value.includes("p95(span.duration)"), + sort: "-p95(span.duration)", + timeRange: { statsPeriod: "7d" }, }, - { - input: - "In metrics, show p95 http.request.duration grouped by environment over the last 24 hours", - expectedTools: [ - { - name: "datasetAttributes", - }, - ], - expected: { - dataset: "metrics", - query: (value: unknown) => - typeof value === "string" && - value.includes("metric.name:http.request.duration") && - value.includes("metric.type:distribution"), - fields: (value: unknown) => - Array.isArray(value) && - value.includes("sentry.environment") && - value.includes( - "p95(value,http.request.duration,distribution,millisecond)", - ), - sort: "-p95(value,http.request.duration,distribution,millisecond)", - timeRange: { statsPeriod: "24h" }, + }, + { + input: + "In metrics, show p95 http.request.duration grouped by environment over the last 24 hours", + expectedTools: [ + { + name: "datasetAttributes", }, + ], + expected: { + dataset: "metrics", + query: (value: unknown) => + typeof value === "string" && + value.includes("metric.name:http.request.duration") && + value.includes("metric.type:distribution"), + fields: (value: unknown) => + Array.isArray(value) && + value.includes("sentry.environment") && + value.includes( + "p95(value,http.request.duration,distribution,millisecond)", + ), + sort: "-p95(value,http.request.duration,distribution,millisecond)", + timeRange: { statsPeriod: "24h" }, }, - ], - task: async (input) => { - const apiService = new SentryApiService({ accessToken: "test-token" }); - const agentResult = await searchEventsAgent({ - query: input, - organizationSlug: "sentry-mcp-evals", - apiService, - environmentNames: [], - }); - - return { - result: JSON.stringify(agentResult.result), - toolCalls: agentResult.toolCalls.map((call) => ({ - name: call.toolName, - arguments: toToolArguments(call.args), - })), - }; }, - scorers: [ToolCallScorer(), StructuredOutputScorer({ match: "fuzzy" })], - threshold: 0.6, - timeout: 30000, -}); +]; -describeEval("natural-language-search-issues", { - data: async () => [ - { - input: "Show resolved high-priority issues, newest first", - expectedTools: [], - expected: { - query: (value: unknown) => - typeof value === "string" && - value.includes("is:resolved") && - value.includes("issue.priority:high"), - sort: "new", +describeEval( + "natural-language-search-events", + { + harness: createTaskHarness( + "natural-language-search-events", + async (input) => { + const apiService = new SentryApiService({ accessToken: "test-token" }); + const agentResult = await searchEventsAgent({ + query: input, + organizationSlug: "sentry-mcp-evals", + apiService, + environmentNames: [], + }); + + return { + result: JSON.stringify(agentResult.result), + toolCalls: agentResult.toolCalls.map((call) => ({ + name: call.toolName, + arguments: toToolArguments(call.args), + })), + }; }, + ), + judges: [ToolCallJudge(), StructuredOutputJudge({ match: "fuzzy" })], + judgeThreshold: 0.6, + }, + (it) => + defineEvalCases(it, naturalLanguageSearchEventsCases, { timeout: 30000 }), +); + +const naturalLanguageSearchIssuesCases: EvalCase[] = [ + { + input: "Show resolved high-priority issues, newest first", + expectedTools: [], + expected: { + query: (value: unknown) => + typeof value === "string" && + value.includes("is:resolved") && + value.includes("issue.priority:high"), + sort: "new", }, - ], - task: async (input) => { - const apiService = new SentryApiService({ accessToken: "test-token" }); - const agentResult = await searchIssuesAgent({ - query: input, - organizationSlug: "sentry-mcp-evals", - apiService, - }); + }, +]; + +describeEval( + "natural-language-search-issues", + { + harness: createTaskHarness( + "natural-language-search-issues", + async (input) => { + const apiService = new SentryApiService({ accessToken: "test-token" }); + const agentResult = await searchIssuesAgent({ + query: input, + organizationSlug: "sentry-mcp-evals", + apiService, + }); - return { - result: JSON.stringify(agentResult.result), - toolCalls: agentResult.toolCalls.map((call) => ({ - name: call.toolName, - arguments: toToolArguments(call.args), - })), - }; + return { + result: JSON.stringify(agentResult.result), + toolCalls: agentResult.toolCalls.map((call) => ({ + name: call.toolName, + arguments: toToolArguments(call.args), + })), + }; + }, + ), + judges: [ToolCallJudge(), StructuredOutputJudge({ match: "fuzzy" })], + judgeThreshold: 0.6, }, - scorers: [ToolCallScorer(), StructuredOutputScorer({ match: "fuzzy" })], - threshold: 0.6, - timeout: 30000, -}); + (it) => + defineEvalCases(it, naturalLanguageSearchIssuesCases, { timeout: 30000 }), +); diff --git a/packages/mcp-server-evals/src/evals/search-docs.eval.ts b/packages/mcp-server-evals/src/evals/search-docs.eval.ts index 084c01bdc..a4f8179a6 100644 --- a/packages/mcp-server-evals/src/evals/search-docs.eval.ts +++ b/packages/mcp-server-evals/src/evals/search-docs.eval.ts @@ -1,51 +1,59 @@ -import { describeEval } from "vitest-evals/legacy"; -import { NoOpTaskRunner, ToolPredictionScorer } from "./utils"; +import { describeEval } from "vitest-evals"; +import { + createTaskHarness, + defineEvalCases, + NoOpTaskRunner, + ToolPredictionJudge, + type EvalCase, +} from "./utils"; -describeEval("search-docs", { - data: async () => { - return [ +const cases: EvalCase[] = [ + { + input: + "I need documentation on how to set up error tracking with Sentry in JavaScript", + expectedTools: [ { - input: - "I need documentation on how to set up error tracking with Sentry in JavaScript", - expectedTools: [ - { - name: "search_docs", - arguments: { - query: "set up error tracking JavaScript", - maxResults: 3, - }, - }, - ], + name: "search_docs", + arguments: { + query: "set up error tracking JavaScript", + maxResults: 3, + }, }, + ], + }, + { + input: + "I need help configuring Sentry with React components and error boundaries", + expectedTools: [ { - input: - "I need help configuring Sentry with React components and error boundaries", - expectedTools: [ - { - name: "search_docs", - arguments: { - query: "React components error boundaries", - maxResults: 3, - }, - }, - ], + name: "search_docs", + arguments: { + query: "React components error boundaries", + maxResults: 3, + }, }, + ], + }, + { + input: "What is Sentry's rate limiting and how does it work?", + expectedTools: [ { - input: "What is Sentry's rate limiting and how does it work?", - expectedTools: [ - { - name: "search_docs", - arguments: { - query: "rate limiting", - maxResults: 3, - }, - }, - ], + name: "search_docs", + arguments: { + query: "rate limiting", + maxResults: 3, + }, }, - ]; + ], + }, +]; + +describeEval( + "search-docs", + { + harness: createTaskHarness("search-docs", NoOpTaskRunner()), + judges: [ToolPredictionJudge()], + judgeThreshold: 0.6, }, - task: NoOpTaskRunner(), - scorers: [ToolPredictionScorer()], - threshold: 0.6, - timeout: 30000, -}); + (it) => defineEvalCases(it, cases, { timeout: 30000 }), +); diff --git a/packages/mcp-server-evals/src/evals/search-events-agent-attribute-context.eval.ts b/packages/mcp-server-evals/src/evals/search-events-agent-attribute-context.eval.ts index 38a5283ac..672e3ca8f 100644 --- a/packages/mcp-server-evals/src/evals/search-events-agent-attribute-context.eval.ts +++ b/packages/mcp-server-evals/src/evals/search-events-agent-attribute-context.eval.ts @@ -1,9 +1,14 @@ -import { describeEval } from "vitest-evals/legacy"; -import { ToolCallScorer } from "vitest-evals/legacy"; import { searchEventsAgent } from "@sentry/mcp-core/tools/search-events/agent"; import { SentryApiService } from "@sentry/mcp-core/api-client"; -import { StructuredOutputScorer } from "./utils/structuredOutputScorer"; import "../setup-env"; +import { describeEval } from "vitest-evals"; +import { + createTaskHarness, + defineEvalCases, + StructuredOutputJudge, + ToolCallJudge, + type EvalCase, +} from "./utils"; // Compares the attribute the agent picks when the trace-items attributes // endpoint is called with and without `expand=context`. The with-context @@ -26,464 +31,476 @@ class AttributeContextApiService extends SentryApiService { } } -describeEval("search-events-agent-attributes-without-context", { - data: async () => { - return [ +const withoutContextCases: EvalCase[] = [ + { + // http.method is also picked from the static span fields + input: "Count spans grouped by HTTP method over the last 7 days", + expectedTools: [ { - // http.method is also picked from the static span fields - input: "Count spans grouped by HTTP method over the last 7 days", - expectedTools: [ - { - name: "datasetAttributes", - arguments: { - dataset: "spans", - }, - }, - ], - expected: { + name: "datasetAttributes", + arguments: { dataset: "spans", - fields: (value: unknown) => - Array.isArray(value) && - value.includes("http.method") && - !value.includes("http.request.method"), }, }, + ], + expected: { + dataset: "spans", + fields: (value: unknown) => + Array.isArray(value) && + value.includes("http.method") && + !value.includes("http.request.method"), + }, + }, + { + // http.status_code is also picked from static span fields + input: "Show me spans with HTTP status code 503 in the last 24 hours", + expectedTools: [ { - // http.status_code is also picked from static span fields - input: "Show me spans with HTTP status code 503 in the last 24 hours", - expectedTools: [ - { - name: "datasetAttributes", - arguments: { - dataset: "spans", - }, - }, - ], - expected: { + name: "datasetAttributes", + arguments: { dataset: "spans", - query: (value: unknown) => - typeof value === "string" && - value.includes("http.status_code:503") && - !value.includes("http.response.status_code"), }, }, + ], + expected: { + dataset: "spans", + query: (value: unknown) => + typeof value === "string" && + value.includes("http.status_code:503") && + !value.includes("http.response.status_code"), + }, + }, + { + // transaction is picked from span fields + input: "total request duration by endpoint (transaction boundaries)", + expectedTools: [ { - // transaction is picked from span fields - input: "total request duration by endpoint (transaction boundaries)", - expectedTools: [ - { - name: "datasetAttributes", - arguments: { - dataset: "spans", - }, - }, - ], - expected: { + name: "datasetAttributes", + arguments: { dataset: "spans", - fields: (value: unknown) => - Array.isArray(value) && - value.includes("transaction") && - !value.includes("sentry.segment.name"), }, }, + ], + expected: { + dataset: "spans", + fields: (value: unknown) => + Array.isArray(value) && + value.includes("transaction") && + !value.includes("sentry.segment.name"), + }, + }, + { + // environment is picked from the common fields + input: + "Show me spans from non production environments in the last 24 hours", + expectedTools: [ { - // environment is picked from the common fields - input: - "Show me spans from non production environments in the last 24 hours", - expectedTools: [ - { - name: "datasetAttributes", - arguments: { - dataset: "spans", - }, - }, - ], - expected: { + name: "datasetAttributes", + arguments: { dataset: "spans", - query: (value: unknown) => - typeof value === "string" && - value.includes("!environment:production") && - !value.includes("sentry.environment"), }, }, + ], + expected: { + dataset: "spans", + query: (value: unknown) => + typeof value === "string" && + value.includes("!environment:production") && + !value.includes("sentry.environment"), + }, + }, + { + // kiosk.firmware and kiosk.firmware_revision are made-up attributes, so + // the agent has no prior knowledge of either. Without briefs it can + // only match on the name, and kiosk.firmware_revision matches + // "firmware revision" + input: + "Count spans grouped by the kiosk's installed firmware revision over the last 7 days", + expectedTools: [ { - // kiosk.firmware and kiosk.firmware_revision are made-up attributes, so - // the agent has no prior knowledge of either. Without briefs it can - // only match on the name, and kiosk.firmware_revision matches - // "firmware revision" - input: - "Count spans grouped by the kiosk's installed firmware revision over the last 7 days", - expectedTools: [ - { - name: "datasetAttributes", - arguments: { - dataset: "spans", - }, - }, - ], - expected: { + name: "datasetAttributes", + arguments: { dataset: "spans", - fields: (value: unknown) => - Array.isArray(value) && - value.includes("kiosk.firmware_revision") && - !value.includes("kiosk.firmware"), }, }, + ], + expected: { + dataset: "spans", + fields: (value: unknown) => + Array.isArray(value) && + value.includes("kiosk.firmware_revision") && + !value.includes("kiosk.firmware"), + }, + }, + { + // Without context the logs attributes only list code.filepath + input: + "Count error logs grouped by source code file path over the last 24 hours", + expectedTools: [ { - // Without context the logs attributes only list code.filepath - input: - "Count error logs grouped by source code file path over the last 24 hours", - expectedTools: [ - { - name: "datasetAttributes", - arguments: { - dataset: "logs", - }, - }, - ], - expected: { + name: "datasetAttributes", + arguments: { dataset: "logs", - fields: (value: unknown) => - Array.isArray(value) && - value.includes("code.filepath") && - !value.includes("code.file.path"), }, }, + ], + expected: { + dataset: "logs", + fields: (value: unknown) => + Array.isArray(value) && + value.includes("code.filepath") && + !value.includes("code.file.path"), + }, + }, + { + // Without context the metrics attributes only list http.method + input: + "In metrics, show p95 http.request.duration grouped by HTTP method over the last 24 hours", + expectedTools: [ { - // Without context the metrics attributes only list http.method - input: - "In metrics, show p95 http.request.duration grouped by HTTP method over the last 24 hours", - expectedTools: [ - { - name: "datasetAttributes", - arguments: { - dataset: "metrics", - }, - }, - ], - expected: { + name: "datasetAttributes", + arguments: { dataset: "metrics", - fields: (value: unknown) => - Array.isArray(value) && - value.includes("http.method") && - !value.includes("http.request.method"), }, }, + ], + expected: { + dataset: "metrics", + fields: (value: unknown) => + Array.isArray(value) && + value.includes("http.method") && + !value.includes("http.request.method"), + }, + }, + { + // worker.queue_wait and worker.queue_wait_time are made-up attributes. + // Without briefs the agent can only match on the name, and + // worker.queue_wait_time matches "worker queue wait time" + input: + "From logs, what is the average worker queue wait time jobs actually experienced over the last 24 hours?", + expectedTools: [ { - // worker.queue_wait and worker.queue_wait_time are made-up attributes. - // Without briefs the agent can only match on the name, and - // worker.queue_wait_time matches "worker queue wait time" - input: - "From logs, what is the average worker queue wait time jobs actually experienced over the last 24 hours?", - expectedTools: [ - { - name: "datasetAttributes", - arguments: { - dataset: "logs", - }, - }, - ], - expected: { + name: "datasetAttributes", + arguments: { dataset: "logs", - fields: (value: unknown) => - Array.isArray(value) && - value.includes("avg(worker.queue_wait_time)") && - !value.includes("avg(worker.queue_wait)"), }, }, + ], + expected: { + dataset: "logs", + fields: (value: unknown) => + Array.isArray(value) && + value.includes("avg(worker.queue_wait_time)") && + !value.includes("avg(worker.queue_wait)"), + }, + }, + { + // fleet.region and fleet.deployment_region are made-up attributes. + // Without briefs the agent can only match on the name, and + // fleet.deployment_region matches "fleet deployment region" + input: + "In metrics, show p95 http.request.duration grouped by the fleet deployment region each server is currently running in over the last 24 hours", + expectedTools: [ { - // fleet.region and fleet.deployment_region are made-up attributes. - // Without briefs the agent can only match on the name, and - // fleet.deployment_region matches "fleet deployment region" - input: - "In metrics, show p95 http.request.duration grouped by the fleet deployment region each server is currently running in over the last 24 hours", - expectedTools: [ - { - name: "datasetAttributes", - arguments: { - dataset: "metrics", - }, - }, - ], - expected: { + name: "datasetAttributes", + arguments: { dataset: "metrics", - fields: (value: unknown) => - Array.isArray(value) && - value.includes("fleet.deployment_region") && - !value.includes("fleet.region"), }, }, - ]; + ], + expected: { + dataset: "metrics", + fields: (value: unknown) => + Array.isArray(value) && + value.includes("fleet.deployment_region") && + !value.includes("fleet.region"), + }, }, - task: async (input) => { - // Create an API service that never requests attribute context - const apiService = new AttributeContextApiService(false); +]; + +describeEval( + "search-events-agent-attributes-without-context", + { + harness: createTaskHarness( + "search-events-agent-attributes-without-context", + async (input) => { + // Create an API service that never requests attribute context + const apiService = new AttributeContextApiService(false); - const agentResult = await searchEventsAgent({ - query: input, - organizationSlug: "sentry-mcp-evals", - apiService, - }); + const agentResult = await searchEventsAgent({ + query: input, + organizationSlug: "sentry-mcp-evals", + apiService, + }); - return { - result: JSON.stringify(agentResult.result), - toolCalls: agentResult.toolCalls.map((call: any) => ({ - name: call.toolName, - arguments: call.args, - })), - }; + return { + result: JSON.stringify(agentResult.result), + toolCalls: agentResult.toolCalls.map((call: any) => ({ + name: call.toolName, + arguments: call.args, + })), + }; + }, + ), + judges: [ + ToolCallJudge({ params: "fuzzy" }), // Validates tool calls + StructuredOutputJudge({ match: "fuzzy" }), // Validates the structured query output with flexible matching + ], }, - scorers: [ - ToolCallScorer({ params: "fuzzy" }), // Validates tool calls - StructuredOutputScorer({ match: "fuzzy" }), // Validates the structured query output with flexible matching - ], -}); + (it) => defineEvalCases(it, withoutContextCases), +); -describeEval("search-events-agent-attributes-with-context", { - data: async () => { - return [ +const withContextCases: EvalCase[] = [ + { + // Context marks http.method as deprecated in favor of http.request.method + input: "Count spans grouped by HTTP method over the last 7 days", + expectedTools: [ { - // Context marks http.method as deprecated in favor of http.request.method - input: "Count spans grouped by HTTP method over the last 7 days", - expectedTools: [ - { - name: "datasetAttributes", - arguments: { - dataset: "spans", - }, - }, - ], - expected: { + name: "datasetAttributes", + arguments: { dataset: "spans", - fields: (value: unknown) => - Array.isArray(value) && - value.includes("http.request.method") && - !value.includes("http.method"), }, }, + ], + expected: { + dataset: "spans", + fields: (value: unknown) => + Array.isArray(value) && + value.includes("http.request.method") && + !value.includes("http.method"), + }, + }, + { + // Context marks http.status_code as deprecated in favor of + // http.response.status_code + input: "Show me spans with HTTP status code 503 in the last 24 hours", + expectedTools: [ { - // Context marks http.status_code as deprecated in favor of - // http.response.status_code - input: "Show me spans with HTTP status code 503 in the last 24 hours", - expectedTools: [ - { - name: "datasetAttributes", - arguments: { - dataset: "spans", - }, - }, - ], - expected: { + name: "datasetAttributes", + arguments: { dataset: "spans", - query: (value: unknown) => - typeof value === "string" && - value.includes("http.response.status_code:503") && - !value.includes("http.status_code"), }, }, + ], + expected: { + dataset: "spans", + query: (value: unknown) => + typeof value === "string" && + value.includes("http.response.status_code:503") && + !value.includes("http.status_code"), + }, + }, + { + // sentry.segment.name is picked from span fields due to transaction deprecation and validation endpoint passing + // even though transaction is shown in example queries + input: "total request duration by endpoint (transaction boundaries)", + expectedTools: [ { - // sentry.segment.name is picked from span fields due to transaction deprecation and validation endpoint passing - // even though transaction is shown in example queries - input: "total request duration by endpoint (transaction boundaries)", - expectedTools: [ - { - name: "datasetAttributes", - arguments: { - dataset: "spans", - }, - }, - { - name: "validateSearch", - arguments: { - dataset: "spans", - fields: ["sentry.segment.name"], - }, - }, - ], - expected: { + name: "datasetAttributes", + arguments: { dataset: "spans", - fields: (value: unknown) => - Array.isArray(value) && - !value.includes("transaction") && - value.includes("sentry.segment.name"), }, }, { - // Context marks environment as deprecated in favor of - // sentry.environment - input: - "Show me spans from non production environments in the last 24 hours", - expectedTools: [ - { - name: "datasetAttributes", - arguments: { - dataset: "spans", - }, - }, - ], - expected: { + name: "validateSearch", + arguments: { dataset: "spans", - // sentry.environment:production contains environment:production, so - // look for a bare environment key not preceded by "sentry." - query: (value: unknown) => - typeof value === "string" && - value.includes("!sentry.environment:production") && - !/(? + Array.isArray(value) && + !value.includes("transaction") && + value.includes("sentry.segment.name"), + }, + }, + { + // Context marks environment as deprecated in favor of + // sentry.environment + input: + "Show me spans from non production environments in the last 24 hours", + expectedTools: [ { - // deviceMemory and device.memory both look like a match by name, but - // only device.memory has a brief describing the device's total memory - input: - "What is the average estimated total memory capacity of devices across spans in the last 7 days?", - expectedTools: [ - { - name: "datasetAttributes", - arguments: { - dataset: "spans", - }, - }, - ], - expected: { + name: "datasetAttributes", + arguments: { dataset: "spans", - fields: (value: unknown) => - Array.isArray(value) && - value.includes("avg(device.memory)") && - !value.some( - (field) => - typeof field === "string" && field.includes("deviceMemory"), - ), }, }, + ], + expected: { + dataset: "spans", + // sentry.environment:production contains environment:production, so + // look for a bare environment key not preceded by "sentry." + query: (value: unknown) => + typeof value === "string" && + value.includes("!sentry.environment:production") && + !/(? + Array.isArray(value) && + value.includes("avg(device.memory)") && + !value.some( + (field) => + typeof field === "string" && field.includes("deviceMemory"), + ), + }, + }, + { + // kiosk.firmware_revision matches "firmware revision" by name, but its + // brief says it is the staged revision. Only kiosk.firmware's brief + // describes the installed firmware + input: + "Count spans grouped by the kiosk's installed firmware revision over the last 7 days", + expectedTools: [ { - // kiosk.firmware_revision matches "firmware revision" by name, but its - // brief says it is the staged revision. Only kiosk.firmware's brief - // describes the installed firmware - input: - "Count spans grouped by the kiosk's installed firmware revision over the last 7 days", - expectedTools: [ - { - name: "datasetAttributes", - arguments: { - dataset: "spans", - }, - }, - ], - expected: { + name: "datasetAttributes", + arguments: { dataset: "spans", - fields: (value: unknown) => - Array.isArray(value) && - value.includes("kiosk.firmware") && - !value.includes("kiosk.firmware_revision"), }, }, + ], + expected: { + dataset: "spans", + fields: (value: unknown) => + Array.isArray(value) && + value.includes("kiosk.firmware") && + !value.includes("kiosk.firmware_revision"), + }, + }, + { + // Context marks code.filepath as deprecated in favor of code.file.path + input: + "Count error logs grouped by source code file path over the last 24 hours", + expectedTools: [ { - // Context marks code.filepath as deprecated in favor of code.file.path - input: - "Count error logs grouped by source code file path over the last 24 hours", - expectedTools: [ - { - name: "datasetAttributes", - arguments: { - dataset: "logs", - }, - }, - ], - expected: { + name: "datasetAttributes", + arguments: { dataset: "logs", - fields: (value: unknown) => - Array.isArray(value) && - value.includes("code.file.path") && - !value.includes("code.filepath"), }, }, + ], + expected: { + dataset: "logs", + fields: (value: unknown) => + Array.isArray(value) && + value.includes("code.file.path") && + !value.includes("code.filepath"), + }, + }, + { + // Context marks http.method as deprecated in favor of + // http.request.method + input: + "In metrics, show p95 http.request.duration grouped by HTTP method over the last 24 hours", + expectedTools: [ { - // Context marks http.method as deprecated in favor of - // http.request.method - input: - "In metrics, show p95 http.request.duration grouped by HTTP method over the last 24 hours", - expectedTools: [ - { - name: "datasetAttributes", - arguments: { - dataset: "metrics", - }, - }, - ], - expected: { + name: "datasetAttributes", + arguments: { dataset: "metrics", - fields: (value: unknown) => - Array.isArray(value) && - value.includes("http.request.method") && - !value.includes("http.method"), }, }, + ], + expected: { + dataset: "metrics", + fields: (value: unknown) => + Array.isArray(value) && + value.includes("http.request.method") && + !value.includes("http.method"), + }, + }, + { + // worker.queue_wait_time matches "worker queue wait time" by name, but + // its brief says it is a configured limit. Only worker.queue_wait's + // brief describes the measured wait + input: + "From logs, what is the average worker queue wait time jobs actually experienced over the last 24 hours?", + expectedTools: [ { - // worker.queue_wait_time matches "worker queue wait time" by name, but - // its brief says it is a configured limit. Only worker.queue_wait's - // brief describes the measured wait - input: - "From logs, what is the average worker queue wait time jobs actually experienced over the last 24 hours?", - expectedTools: [ - { - name: "datasetAttributes", - arguments: { - dataset: "logs", - }, - }, - ], - expected: { + name: "datasetAttributes", + arguments: { dataset: "logs", - fields: (value: unknown) => - Array.isArray(value) && - value.includes("avg(worker.queue_wait)") && - !value.includes("avg(worker.queue_wait_time)"), }, }, + ], + expected: { + dataset: "logs", + fields: (value: unknown) => + Array.isArray(value) && + value.includes("avg(worker.queue_wait)") && + !value.includes("avg(worker.queue_wait_time)"), + }, + }, + { + // fleet.deployment_region matches "fleet deployment region" by name, + // but its brief says it is the region the server moves to next. Only + // fleet.region's brief describes where the server runs now + input: + "In metrics, show p95 http.request.duration grouped by the fleet deployment region each server is currently running in over the last 24 hours", + expectedTools: [ { - // fleet.deployment_region matches "fleet deployment region" by name, - // but its brief says it is the region the server moves to next. Only - // fleet.region's brief describes where the server runs now - input: - "In metrics, show p95 http.request.duration grouped by the fleet deployment region each server is currently running in over the last 24 hours", - expectedTools: [ - { - name: "datasetAttributes", - arguments: { - dataset: "metrics", - }, - }, - ], - expected: { + name: "datasetAttributes", + arguments: { dataset: "metrics", - fields: (value: unknown) => - Array.isArray(value) && - value.includes("fleet.region") && - !value.includes("fleet.deployment_region"), }, }, - ]; + ], + expected: { + dataset: "metrics", + fields: (value: unknown) => + Array.isArray(value) && + value.includes("fleet.region") && + !value.includes("fleet.deployment_region"), + }, }, - task: async (input) => { - // Create an API service that always requests attribute context - const apiService = new AttributeContextApiService(true); +]; + +describeEval( + "search-events-agent-attributes-with-context", + { + harness: createTaskHarness( + "search-events-agent-attributes-with-context", + async (input) => { + // Create an API service that always requests attribute context + const apiService = new AttributeContextApiService(true); - const agentResult = await searchEventsAgent({ - query: input, - organizationSlug: "sentry-mcp-evals", - apiService, - }); + const agentResult = await searchEventsAgent({ + query: input, + organizationSlug: "sentry-mcp-evals", + apiService, + }); - return { - result: JSON.stringify(agentResult.result), - toolCalls: agentResult.toolCalls.map((call: any) => ({ - name: call.toolName, - arguments: call.args, - })), - }; + return { + result: JSON.stringify(agentResult.result), + toolCalls: agentResult.toolCalls.map((call: any) => ({ + name: call.toolName, + arguments: call.args, + })), + }; + }, + ), + judges: [ + ToolCallJudge({ params: "fuzzy" }), // Validates tool calls + StructuredOutputJudge({ match: "fuzzy" }), // Validates the structured query output with flexible matching + ], }, - scorers: [ - ToolCallScorer({ params: "fuzzy" }), // Validates tool calls - StructuredOutputScorer({ match: "fuzzy" }), // Validates the structured query output with flexible matching - ], -}); + (it) => defineEvalCases(it, withContextCases), +); diff --git a/packages/mcp-server-evals/src/evals/search-events-agent-regex.eval.ts b/packages/mcp-server-evals/src/evals/search-events-agent-regex.eval.ts index 245e4453c..de49e5568 100644 --- a/packages/mcp-server-evals/src/evals/search-events-agent-regex.eval.ts +++ b/packages/mcp-server-evals/src/evals/search-events-agent-regex.eval.ts @@ -1,8 +1,14 @@ import { SentryApiService } from "@sentry/mcp-core/api-client"; import { searchEventsAgent } from "@sentry/mcp-core/tools/search-events/agent"; -import { describeEval, ToolCallScorer } from "vitest-evals/legacy"; -import { StructuredOutputScorer } from "./utils/structuredOutputScorer"; import "../setup-env"; +import { describeEval } from "vitest-evals"; +import { + createTaskHarness, + defineEvalCases, + StructuredOutputJudge, + ToolCallJudge, + type EvalCase, +} from "./utils"; function messageRegexPattern(query: unknown): string | undefined { if (typeof query !== "string") { @@ -35,192 +41,196 @@ function hasRegexFilter(query: unknown): boolean { return typeof query === "string" && /:\/\/.+\/\/(?=[\s)]|$)/.test(query); } -describeEval("search-events-agent-regex", { - data: async () => [ - { - input: - "Show logs from the last day whose message matches the regex `timeout after \\d+ms`", - expectedTools: [], - expected: { - dataset: "logs", - query: (value: unknown) => { - const pattern = messageRegexPattern(value); - return ( - pattern !== undefined && - pattern.includes("timeout after") && - pattern.includes("\\d") - ); - }, - timeRange: { statsPeriod: "24h" }, +const cases: EvalCase[] = [ + { + input: + "Show logs from the last day whose message matches the regex `timeout after \\d+ms`", + expectedTools: [], + expected: { + dataset: "logs", + query: (value: unknown) => { + const pattern = messageRegexPattern(value); + return ( + pattern !== undefined && + pattern.includes("timeout after") && + pattern.includes("\\d") + ); }, + timeRange: { statsPeriod: "24h" }, }, - { - input: "Find logs whose message contains an IPv4 address", - expectedTools: [], - expected: { - dataset: "logs", - query: (value: unknown) => { - const pattern = messageRegexPattern(value); - return ( - pattern !== undefined && - /\\d|\[0-9\]/.test(pattern) && - pattern.includes("\\.") - ); - }, + }, + { + input: "Find logs whose message contains an IPv4 address", + expectedTools: [], + expected: { + dataset: "logs", + query: (value: unknown) => { + const pattern = messageRegexPattern(value); + return ( + pattern !== undefined && + /\\d|\[0-9\]/.test(pattern) && + pattern.includes("\\.") + ); }, }, - { - input: - "Find error logs whose message ends with a 5xx status code, like 'upstream returned status=503'", - expectedTools: [], - expected: { - dataset: "logs", - query: (value: unknown) => { - const pattern = messageRegexPattern(value); - return ( - /\bseverity:error\b/.test(String(value)) && - pattern !== undefined && - pattern.includes("5") && - pattern.endsWith("$") - ); - }, + }, + { + input: + "Find error logs whose message ends with a 5xx status code, like 'upstream returned status=503'", + expectedTools: [], + expected: { + dataset: "logs", + query: (value: unknown) => { + const pattern = messageRegexPattern(value); + return ( + /\bseverity:error\b/.test(String(value)) && + pattern !== undefined && + pattern.includes("5") && + pattern.endsWith("$") + ); }, }, - { - input: - "Show logs whose message does not look like 'job completed'", - expectedTools: [], - expected: { - dataset: "logs", - query: (value: unknown) => - typeof value === "string" && - /!(?:message|log\.body):\/\//.test(value) && - messageRegexPattern(value)?.includes("completed") === true, - }, + }, + { + input: + "Show logs whose message does not look like 'job completed'", + expectedTools: [], + expected: { + dataset: "logs", + query: (value: unknown) => + typeof value === "string" && + /!(?:message|log\.body):\/\//.test(value) && + messageRegexPattern(value)?.includes("completed") === true, }, - { - input: - "Find logs whose message looks like 'connection refused on port ', ignoring case", - expectedTools: [], - expected: { - dataset: "logs", - query: (value: unknown) => { - const pattern = messageRegexPattern(value); - return ( - pattern !== undefined && - /^\(\?i[):]/.test(pattern) && - pattern - .toLowerCase() - .replace(/\\s[+*]?/g, " ") - .includes("connection refused") && - /\\d|\[0-9\]/.test(pattern) - ); - }, + }, + { + input: + "Find logs whose message looks like 'connection refused on port ', ignoring case", + expectedTools: [], + expected: { + dataset: "logs", + query: (value: unknown) => { + const pattern = messageRegexPattern(value); + return ( + pattern !== undefined && + /^\(\?i[):]/.test(pattern) && + pattern + .toLowerCase() + .replace(/\\s[+*]?/g, " ") + .includes("connection refused") && + /\\d|\[0-9\]/.test(pattern) + ); }, }, - { - input: "Find logs whose message contains a UUID", - expectedTools: [], - expected: { - dataset: "logs", - query: (value: unknown) => { - const pattern = messageRegexPattern(value); - return ( - pattern !== undefined && - pattern.includes("-") && - /\{(4|8|12|36)\}/.test(pattern) - ); - }, + }, + { + input: "Find logs whose message contains a UUID", + expectedTools: [], + expected: { + dataset: "logs", + query: (value: unknown) => { + const pattern = messageRegexPattern(value); + return ( + pattern !== undefined && + pattern.includes("-") && + /\{(4|8|12|36)\}/.test(pattern) + ); }, }, - { - input: "Find logs whose message starts with 'Worker shutting down'", - expectedTools: [], - expected: { - dataset: "logs", - query: (value: unknown) => - typeof value === "string" && - value.includes("Worker shutting down*") && - !value.includes("*Worker shutting down") && - !hasRegexFilter(value), - }, + }, + { + input: "Find logs whose message starts with 'Worker shutting down'", + expectedTools: [], + expected: { + dataset: "logs", + query: (value: unknown) => + typeof value === "string" && + value.includes("Worker shutting down*") && + !value.includes("*Worker shutting down") && + !hasRegexFilter(value), }, - { - input: "Find logs that mention either 'cache miss' or 'cache evicted'", - expectedTools: [], - expected: { - dataset: "logs", - query: (value: unknown) => - typeof value === "string" && - value.includes("*cache miss*") && - value.includes("*cache evicted*") && - !hasRegexFilter(value), - }, + }, + { + input: "Find logs that mention either 'cache miss' or 'cache evicted'", + expectedTools: [], + expected: { + dataset: "logs", + query: (value: unknown) => + typeof value === "string" && + value.includes("*cache miss*") && + value.includes("*cache evicted*") && + !hasRegexFilter(value), }, - { - input: - "Find spans whose description looks like 'GET /api/users/'", - expectedTools: [], - expected: { - dataset: "spans", - query: (value: unknown) => - typeof value === "string" && - value.includes("/api/users/") && - !hasRegexFilter(value), - }, + }, + { + input: "Find spans whose description looks like 'GET /api/users/'", + expectedTools: [], + expected: { + dataset: "spans", + query: (value: unknown) => + typeof value === "string" && + value.includes("/api/users/") && + !hasRegexFilter(value), }, - { - input: "Find errors whose message looks like 'timeout after ms'", - expectedTools: [], - expected: { - dataset: "errors", - query: (value: unknown) => - typeof value === "string" && - value.includes("timeout after") && - !hasRegexFilter(value), - }, + }, + { + input: "Find errors whose message looks like 'timeout after ms'", + expectedTools: [], + expected: { + dataset: "errors", + query: (value: unknown) => + typeof value === "string" && + value.includes("timeout after") && + !hasRegexFilter(value), }, - { - input: "Show me error logs about payments", - expectedTools: [], - expected: { - dataset: "logs", - query: (value: unknown) => - typeof value === "string" && - value.includes("*payment") && - !hasRegexFilter(value), - }, + }, + { + input: "Show me error logs about payments", + expectedTools: [], + expected: { + dataset: "logs", + query: (value: unknown) => + typeof value === "string" && + value.includes("*payment") && + !hasRegexFilter(value), }, - { - input: "Find warning logs that mention disk space", - expectedTools: [], - expected: { - dataset: "logs", - query: (value: unknown) => - typeof value === "string" && - value.includes("*disk") && - !hasRegexFilter(value), - }, + }, + { + input: "Find warning logs that mention disk space", + expectedTools: [], + expected: { + dataset: "logs", + query: (value: unknown) => + typeof value === "string" && + value.includes("*disk") && + !hasRegexFilter(value), }, - ], - task: async (input) => { - const apiService = new SentryApiService({ accessToken: "test-token" }); - const agentResult = await searchEventsAgent({ - query: input, - organizationSlug: "sentry-mcp-evals", - apiService, - }); + }, +]; + +describeEval( + "search-events-agent-regex", + { + harness: createTaskHarness("search-events-agent-regex", async (input) => { + const apiService = new SentryApiService({ accessToken: "test-token" }); + const agentResult = await searchEventsAgent({ + query: input, + organizationSlug: "sentry-mcp-evals", + apiService, + }); - return { - result: JSON.stringify(agentResult.result), - toolCalls: agentResult.toolCalls.map((call) => ({ - name: call.toolName, - arguments: - typeof call.args === "object" && call.args !== null - ? { ...call.args } - : {}, - })), - }; + return { + result: JSON.stringify(agentResult.result), + toolCalls: agentResult.toolCalls.map((call) => ({ + name: call.toolName, + arguments: + typeof call.args === "object" && call.args !== null + ? { ...call.args } + : {}, + })), + }; + }), + judges: [ToolCallJudge(), StructuredOutputJudge({ match: "fuzzy" })], }, - scorers: [ToolCallScorer(), StructuredOutputScorer({ match: "fuzzy" })], -}); + (it) => defineEvalCases(it, cases), +); diff --git a/packages/mcp-server-evals/src/evals/search-events-agent.eval.ts b/packages/mcp-server-evals/src/evals/search-events-agent.eval.ts index 08683a53e..6d1a86fcc 100644 --- a/packages/mcp-server-evals/src/evals/search-events-agent.eval.ts +++ b/packages/mcp-server-evals/src/evals/search-events-agent.eval.ts @@ -1,281 +1,289 @@ -import { describeEval } from "vitest-evals/legacy"; -import { ToolCallScorer } from "vitest-evals/legacy"; import { searchEventsAgent } from "@sentry/mcp-core/tools/search-events/agent"; import { SentryApiService } from "@sentry/mcp-core/api-client"; -import { StructuredOutputScorer } from "./utils/structuredOutputScorer"; import "../setup-env"; +import { describeEval } from "vitest-evals"; +import { + createTaskHarness, + defineEvalCases, + StructuredOutputJudge, + ToolCallJudge, + type EvalCase, +} from "./utils"; // The shared MSW server is already started in setup-env.ts -describeEval("search-events-agent", { - data: async () => { - return [ - { - // Simple query with common fields - should NOT require tool calls - input: "Show me all errors from today", - expectedTools: [], - expected: { - dataset: "errors", - query: "", // No filters, just time range - sort: "-timestamp", - timeRange: { statsPeriod: "24h" }, - }, - }, +const cases: EvalCase[] = [ + { + // Simple query with common fields - should NOT require tool calls + input: "Show me all errors from today", + expectedTools: [], + expected: { + dataset: "errors", + query: "", // No filters, just time range + sort: "-timestamp", + timeRange: { statsPeriod: "24h" }, + }, + }, + { + // Query with "me" reference - should only require whoami + input: "Show me my errors from last week", + expectedTools: [ { - // Query with "me" reference - should only require whoami - input: "Show me my errors from last week", - expectedTools: [ - { - name: "whoami", - arguments: {}, - }, - ], - expected: { - dataset: "errors", - query: /user\.email:test@example\.com|user\.id:123456/, // Can be either - sort: "-timestamp", - timeRange: { statsPeriod: "7d" }, - }, + name: "whoami", + arguments: {}, }, + ], + expected: { + dataset: "errors", + query: /user\.email:test@example\.com|user\.id:123456/, // Can be either + sort: "-timestamp", + timeRange: { statsPeriod: "7d" }, + }, + }, + { + // Common performance query - should NOT require tool calls + input: "Show me slow API calls taking more than 1 second", + expectedTools: [], + expected: { + dataset: "spans", + query: /span\.duration:>1000|span\.duration:>1s/, // Can express as ms or seconds + sort: "-span.duration", + }, + }, + { + // Query with OpenTelemetry attributes that need discovery + input: "Show me LLM calls where temperature setting is above 0.7", + expectedTools: [ { - // Common performance query - should NOT require tool calls - input: "Show me slow API calls taking more than 1 second", - expectedTools: [], - expected: { + name: "datasetAttributes", + arguments: { dataset: "spans", - query: /span\.duration:>1000|span\.duration:>1s/, // Can express as ms or seconds - sort: "-span.duration", }, }, { - // Query with OpenTelemetry attributes that need discovery - input: "Show me LLM calls where temperature setting is above 0.7", - expectedTools: [ - { - name: "datasetAttributes", - arguments: { - dataset: "spans", - }, - }, - { - name: "otelSemantics", - arguments: { - namespace: "gen_ai", - dataset: "spans", - }, - }, - ], - expected: { + name: "otelSemantics", + arguments: { + namespace: "gen_ai", dataset: "spans", - query: "gen_ai.request.temperature:>0.7", - sort: "-span.duration", }, }, + ], + expected: { + dataset: "spans", + query: "gen_ai.request.temperature:>0.7", + sort: "-span.duration", + }, + }, + { + // Query with custom field requiring discovery + input: "Find errors with custom.payment.processor field", + expectedTools: [ { - // Query with custom field requiring discovery - input: "Find errors with custom.payment.processor field", - expectedTools: [ - { - name: "datasetAttributes", - arguments: { - dataset: "errors", - }, - }, - ], - expected: { + name: "datasetAttributes", + arguments: { dataset: "errors", - query: "has:custom.payment.processor", - sort: "-timestamp", }, }, + ], + expected: { + dataset: "errors", + query: "has:custom.payment.processor", + sort: "-timestamp", + }, + }, + { + // Query with custom field requiring discovery + input: "Show me spans where custom.db.pool_size is greater than 10", + expectedTools: [ { - // Query with custom field requiring discovery - input: "Show me spans where custom.db.pool_size is greater than 10", - expectedTools: [ - { - name: "datasetAttributes", - arguments: { - dataset: "spans", - }, - }, - ], - expected: { + name: "datasetAttributes", + arguments: { dataset: "spans", - query: "custom.db.pool_size:>10", - sort: "-span.duration", }, }, + ], + expected: { + dataset: "spans", + query: "custom.db.pool_size:>10", + sort: "-span.duration", + }, + }, + { + // User-supplied Sentry syntax should remain authoritative. The agent + // can validate fields, but it should not rewrite or drop explicit + // filters/fields while translating the request. + input: + 'In spans, search for transaction:"VPN connections" tags[type]:Unified tags[country]:CN over the last 7 days. Return tags[type], tags[sequence], and count(), sorted by count descending.', + expectedTools: [ { - // User-supplied Sentry syntax should remain authoritative. The agent - // can validate fields, but it should not rewrite or drop explicit - // filters/fields while translating the request. - input: - 'In spans, search for transaction:"VPN connections" tags[type]:Unified tags[country]:CN over the last 7 days. Return tags[type], tags[sequence], and count(), sorted by count descending.', - expectedTools: [ - { - name: "datasetAttributes", - }, - ], - expected: { - dataset: "spans", - query: (value: unknown) => - typeof value === "string" && - [ - 'sentry.segment.name:"VPN connections"', - "tags[type]:Unified", - "tags[country]:CN", - ].every((token) => value.includes(token)), - fields: (value: unknown) => - Array.isArray(value) && - ["tags[type]", "tags[sequence]", "count()"].every((field) => - value.includes(field), - ), - sort: "-count()", - timeRange: { statsPeriod: "7d" }, - }, + name: "datasetAttributes", }, + ], + expected: { + dataset: "spans", + query: (value: unknown) => + typeof value === "string" && + [ + 'sentry.segment.name:"VPN connections"', + "tags[type]:Unified", + "tags[country]:CN", + ].every((token) => value.includes(token)), + fields: (value: unknown) => + Array.isArray(value) && + ["tags[type]", "tags[sequence]", "count()"].every((field) => + value.includes(field), + ), + sort: "-count()", + timeRange: { statsPeriod: "7d" }, + }, + }, + { + // Structured filters must stay structured. Never rewrite a field:value + // filter into message/log.body full-text (tweet false-success path). + input: + "In logs, search for conv_id:ZYGC-86ZR from the last 24 hours. Return timestamp, message, and trace.", + expectedTools: [ { - // Structured filters must stay structured. Never rewrite a field:value - // filter into message/log.body full-text (tweet false-success path). - input: - "In logs, search for conv_id:ZYGC-86ZR from the last 24 hours. Return timestamp, message, and trace.", - expectedTools: [ - { - name: "datasetAttributes", - arguments: { - dataset: "logs", - }, - }, - ], - expected: { + name: "datasetAttributes", + arguments: { dataset: "logs", - query: (value: unknown) => - typeof value === "string" && - value.includes("conv_id:ZYGC-86ZR") && - !/\bmessage\s*:/i.test(value) && - !/\blog\.body\s*:/i.test(value), - fields: (value: unknown) => - Array.isArray(value) && - ["timestamp", "message", "trace"].every((field) => - value.includes(field), - ), - sort: "-timestamp", - timeRange: { statsPeriod: "24h" }, }, }, + ], + expected: { + dataset: "logs", + query: (value: unknown) => + typeof value === "string" && + value.includes("conv_id:ZYGC-86ZR") && + !/\bmessage\s*:/i.test(value) && + !/\blog\.body\s*:/i.test(value), + fields: (value: unknown) => + Array.isArray(value) && + ["timestamp", "message", "trace"].every((field) => + value.includes(field), + ), + sort: "-timestamp", + timeRange: { statsPeriod: "24h" }, + }, + }, + { + // Real attribute renames are still allowed when the structured value + // stays on a non-full-text field. + input: + "In spans, search for spon.duration:>100 over the last day. Return span.duration sorted descending.", + expectedTools: [ { - // Real attribute renames are still allowed when the structured value - // stays on a non-full-text field. - input: - "In spans, search for spon.duration:>100 over the last day. Return span.duration sorted descending.", - expectedTools: [ - { - name: "datasetAttributes", - arguments: { - dataset: "spans", - }, - }, - ], - expected: { + name: "datasetAttributes", + arguments: { dataset: "spans", - query: (value: unknown) => - typeof value === "string" && - value.includes("span.duration:>100") && - !/\bmessage\s*:/i.test(value), - fields: (value: unknown) => - Array.isArray(value) && value.includes("span.duration"), - sort: "-span.duration", - timeRange: { statsPeriod: "24h" }, }, }, + ], + expected: { + dataset: "spans", + query: (value: unknown) => + typeof value === "string" && + value.includes("span.duration:>100") && + !/\bmessage\s*:/i.test(value), + fields: (value: unknown) => + Array.isArray(value) && value.includes("span.duration"), + sort: "-span.duration", + timeRange: { statsPeriod: "24h" }, + }, + }, + { + // Query requiring equation field calculation + input: "How many total tokens did we consume yesterday", + expectedTools: [ { - // Query requiring equation field calculation - input: "How many total tokens did we consume yesterday", - expectedTools: [ - { - name: "datasetAttributes", - arguments: { - dataset: "spans", - }, - }, - // Agent may find gen_ai fields and use them for calculation - ], - expected: { + name: "datasetAttributes", + arguments: { dataset: "spans", - // For aggregations, query filter is optional - empty query gets all spans - query: /^$|has:gen_ai\.usage\.(input_tokens|output_tokens)/, - // Equation to sum both token types - fields: [ - "equation|sum(gen_ai.usage.input_tokens) + sum(gen_ai.usage.output_tokens)", - ], - // Sort by the equation result in descending order - sort: "-equation|sum(gen_ai.usage.input_tokens) + sum(gen_ai.usage.output_tokens)", - timeRange: { statsPeriod: "24h" }, - }, - }, - { - // Query that tests sort field self-correction - // Agent should self-correct by adding count() to fields when sorting by it - input: "Show me the top 10 most frequent error types", - expectedTools: [], - expected: { - dataset: "errors", - query: "", // No specific filter, just aggregate all errors - // Agent should include count() in fields since we're sorting by it - fields: ["error.type", "count()"], - // Sort by count in descending order to get "most frequent" - sort: "-count()", - // timeRange can be null or have a default period }, }, + // Agent may find gen_ai fields and use them for calculation + ], + expected: { + dataset: "spans", + // For aggregations, query filter is optional - empty query gets all spans + query: /^$|has:gen_ai\.usage\.(input_tokens|output_tokens)/, + // Equation to sum both token types + fields: [ + "equation|sum(gen_ai.usage.input_tokens) + sum(gen_ai.usage.output_tokens)", + ], + // Sort by the equation result in descending order + sort: "-equation|sum(gen_ai.usage.input_tokens) + sum(gen_ai.usage.output_tokens)", + timeRange: { statsPeriod: "24h" }, + }, + }, + { + // Query that tests sort field self-correction + // Agent should self-correct by adding count() to fields when sorting by it + input: "Show me the top 10 most frequent error types", + expectedTools: [], + expected: { + dataset: "errors", + query: "", // No specific filter, just aggregate all errors + // Agent should include count() in fields since we're sorting by it + fields: ["error.type", "count()"], + // Sort by count in descending order to get "most frequent" + sort: "-count()", + // timeRange can be null or have a default period + }, + }, + { + // Complex aggregate query that tests sort field self-correction + // Agent should self-correct by including avg(span.duration) in fields + input: + "Show me database operations grouped by type, sorted by average duration", + expectedTools: [ { - // Complex aggregate query that tests sort field self-correction - // Agent should self-correct by including avg(span.duration) in fields - input: - "Show me database operations grouped by type, sorted by average duration", - expectedTools: [ - { - name: "datasetAttributes", - arguments: { - dataset: "spans", - }, - }, - ], - expected: { + name: "datasetAttributes", + arguments: { dataset: "spans", - query: "has:db.operation", - // Agent must include avg(span.duration) since we're sorting by it - // Use db.operation as the grouping field (span.op is deprecated) - fields: ["db.operation", "avg(span.duration)"], - // Sort by average duration - sort: "-avg(span.duration)", - // timeRange is optional }, }, - ]; + ], + expected: { + dataset: "spans", + query: "has:db.operation", + // Agent must include avg(span.duration) since we're sorting by it + // Use db.operation as the grouping field (span.op is deprecated) + fields: ["db.operation", "avg(span.duration)"], + // Sort by average duration + sort: "-avg(span.duration)", + // timeRange is optional + }, }, - task: async (input) => { - // Create a real API service that will use MSW mocks - const apiService = new SentryApiService({ - accessToken: "test-token", - }); +]; + +describeEval( + "search-events-agent", + { + harness: createTaskHarness("search-events-agent", async (input) => { + // Create a real API service that will use MSW mocks + const apiService = new SentryApiService({ + accessToken: "test-token", + }); - const agentResult = await searchEventsAgent({ - query: input, - organizationSlug: "sentry-mcp-evals", - apiService, - }); + const agentResult = await searchEventsAgent({ + query: input, + organizationSlug: "sentry-mcp-evals", + apiService, + }); - return { - result: JSON.stringify(agentResult.result), - toolCalls: agentResult.toolCalls.map((call: any) => ({ - name: call.toolName, - arguments: call.args, - })), - }; + return { + result: JSON.stringify(agentResult.result), + toolCalls: agentResult.toolCalls.map((call: any) => ({ + name: call.toolName, + arguments: call.args, + })), + }; + }), + judges: [ + ToolCallJudge(), // Validates tool calls + StructuredOutputJudge({ match: "fuzzy" }), // Validates the structured query output with flexible matching + ], }, - scorers: [ - ToolCallScorer(), // Validates tool calls - StructuredOutputScorer({ match: "fuzzy" }), // Validates the structured query output with flexible matching - ], -}); + (it) => defineEvalCases(it, cases), +); diff --git a/packages/mcp-server-evals/src/evals/search-events-environments.eval.ts b/packages/mcp-server-evals/src/evals/search-events-environments.eval.ts index c0fa9a9b1..973ad966e 100644 --- a/packages/mcp-server-evals/src/evals/search-events-environments.eval.ts +++ b/packages/mcp-server-evals/src/evals/search-events-environments.eval.ts @@ -1,94 +1,106 @@ import { SentryApiService } from "@sentry/mcp-core/api-client"; import { searchEventsAgent } from "@sentry/mcp-core/tools/search-events/agent"; import { expect } from "vitest"; -import { describeEval } from "vitest-evals/legacy"; import "../setup-env"; -import { StructuredOutputScorer } from "./utils/structuredOutputScorer"; +import { describeEval } from "vitest-evals"; +import { + createTaskHarness, + defineEvalCases, + StructuredOutputJudge, + type EvalCase, +} from "./utils"; -describeEval("search-events-environment-grounding", { - data: async () => - ["message:*decoder*", "message:*decoder* environment:production"].map( - (query) => ({ - // Sanitized request shape from a real failure: a structured errors - // aggregate exhausted all five steps on invented environment values, - // despite the prompt already listing the only real environment. - // Keep the handler's request wrapper, including environment: null. - input: [ - "Translate this Sentry event search request.", - "The query may be natural language or already-valid Sentry search syntax.", - "Preserve valid explicit parameters, but correct dataset, query syntax, fields, sort, and time range when they conflict or would fail.", - "If the user query already uses Sentry search syntax, treat its filters as authoritative unless validateSearch proves a field is invalid.", - "Never replace a structured field filter with message/log.body/full-text matching. If no valid attribute exists for an explicit field:value filter, keep the field and let validation fail.", - "For spans, logs, and metrics, use datasetAttributes to discover likely fields with substringMatch, query, and attributeTypes before dropping or renaming explicit fields.", - "A broad datasetAttributes result may be truncated, so absence from that preview does not prove an explicit field is invalid.", - "For non-replay datasets, call validateSearch on the candidate request and fix failures in this same pass before returning.", - "For non-replay datasets, convert environment parameters into query filters. For replays, keep environment in the separate environment parameter.", - "", - `User query: ${query}`, - "Current parameters:", - JSON.stringify( - { - dataset: "errors", - fields: ["count()"], - sort: "-count()", - statsPeriod: "90d", - environment: null, - }, - null, - 2, - ), - ].join("\n"), - expected: { - dataset: "errors", - fields: ["count()"], - sort: "-count()", - environment: null, - timeRange: { statsPeriod: "90d" }, - }, - }), +const cases: EvalCase[] = [ + "message:*decoder*", + "message:*decoder* environment:production", +].map((query) => ({ + // Sanitized request shape from a real failure: a structured errors + // aggregate exhausted all five steps on invented environment values, + // despite the prompt already listing the only real environment. + // Keep the handler's request wrapper, including environment: null. + input: [ + "Translate this Sentry event search request.", + "The query may be natural language or already-valid Sentry search syntax.", + "Preserve valid explicit parameters, but correct dataset, query syntax, fields, sort, and time range when they conflict or would fail.", + "If the user query already uses Sentry search syntax, treat its filters as authoritative unless validateSearch proves a field is invalid.", + "Never replace a structured field filter with message/log.body/full-text matching. If no valid attribute exists for an explicit field:value filter, keep the field and let validation fail.", + "For spans, logs, and metrics, use datasetAttributes to discover likely fields with substringMatch, query, and attributeTypes before dropping or renaming explicit fields.", + "A broad datasetAttributes result may be truncated, so absence from that preview does not prove an explicit field is invalid.", + "For non-replay datasets, call validateSearch on the candidate request and fix failures in this same pass before returning.", + "For non-replay datasets, convert environment parameters into query filters. For replays, keep environment in the separate environment parameter.", + "", + `User query: ${query}`, + "Current parameters:", + JSON.stringify( + { + dataset: "errors", + fields: ["count()"], + sort: "-count()", + statsPeriod: "90d", + environment: null, + }, + null, + 2, ), - task: async (input) => { - const agentResult = await searchEventsAgent({ - query: input, - organizationSlug: "sentry-mcp-evals", - apiService: new SentryApiService({ accessToken: "test-token" }), - environmentNames: ["production"], - }); - - const validationCalls = agentResult.toolCalls.filter( - (call) => call.toolName === "validateSearch", - ); - expect(validationCalls.length).toBeGreaterThan(0); + ].join("\n"), + expected: { + dataset: "errors", + fields: ["count()"], + sort: "-count()", + environment: null, + timeRange: { statsPeriod: "90d" }, + }, +})); - const queries = [agentResult.result.query]; - for (const call of validationCalls) { - expect(call.args).toBeTypeOf("object"); - expect(call.args).not.toBeNull(); - expect(call.args).not.toHaveProperty("environment"); - const args = call.args as Record; - expect(args.query).toBeTypeOf("string"); - queries.push(args.query as string); - } +describeEval( + "search-events-environment-grounding", + { + harness: createTaskHarness( + "search-events-environment-grounding", + async (input) => { + const agentResult = await searchEventsAgent({ + query: input, + organizationSlug: "sentry-mcp-evals", + apiService: new SentryApiService({ accessToken: "test-token" }), + environmentNames: ["production"], + }); - // Check every attempted validation, not just the final successful query: - // repeated invalid tool calls can consume the budget before any output. - const requestedEnvironment = input.includes("environment:production"); - for (const query of queries) { - expect(query).toContain("message:*decoder*"); - if (requestedEnvironment) { - expect(query).toMatch( - /(?:^|\s)environment:(?:production|"production")(?=\s|$)/, + const validationCalls = agentResult.toolCalls.filter( + (call) => call.toolName === "validateSearch", ); - expect(query.match(/\benvironment\s*:/g)).toHaveLength(1); - } else { - expect(query).not.toMatch(/\benvironment\s*:/); - } - } - expect(agentResult.result.environment).toBeNull(); + expect(validationCalls.length).toBeGreaterThan(0); + + const queries = [agentResult.result.query]; + for (const call of validationCalls) { + expect(call.args).toBeTypeOf("object"); + expect(call.args).not.toBeNull(); + expect(call.args).not.toHaveProperty("environment"); + const args = call.args as Record; + expect(args.query).toBeTypeOf("string"); + queries.push(args.query as string); + } - return { result: JSON.stringify(agentResult.result) }; + // Check every attempted validation, not just the final successful query: + // repeated invalid tool calls can consume the budget before any output. + const requestedEnvironment = input.includes("environment:production"); + for (const query of queries) { + expect(query).toContain("message:*decoder*"); + if (requestedEnvironment) { + expect(query).toMatch( + /(?:^|\s)environment:(?:production|"production")(?=\s|$)/, + ); + expect(query.match(/\benvironment\s*:/g)).toHaveLength(1); + } else { + expect(query).not.toMatch(/\benvironment\s*:/); + } + } + expect(agentResult.result.environment).toBeNull(); + + return { result: JSON.stringify(agentResult.result) }; + }, + ), + judges: [StructuredOutputJudge()], + judgeThreshold: 1, }, - scorers: [StructuredOutputScorer()], - threshold: 1, - timeout: 60000, -}); + (it) => defineEvalCases(it, cases, { timeout: 60000 }), +); diff --git a/packages/mcp-server-evals/src/evals/search-events.eval.ts b/packages/mcp-server-evals/src/evals/search-events.eval.ts index 891c53952..271c628fb 100644 --- a/packages/mcp-server-evals/src/evals/search-events.eval.ts +++ b/packages/mcp-server-evals/src/evals/search-events.eval.ts @@ -1,105 +1,114 @@ -import { describeEval } from "vitest-evals/legacy"; -import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils"; +import { describeEval } from "vitest-evals"; +import { + createTaskHarness, + defineEvalCases, + FIXTURES, + NoOpTaskRunner, + ToolPredictionJudge, + type EvalCase, +} from "./utils"; // Note: This eval requires OPENROUTER_API_KEY to be set in the environment // The dataset search tools (search_errors, search_traces, search_logs, ...) use the AI SDK to translate natural language queries -describeEval("search-events", { - data: async () => { - return [ - // Core test: Basic error event search +const cases: EvalCase[] = [ + // Core test: Basic error event search + { + input: `Find database timeouts in ${FIXTURES.organizationSlug} from the last week`, + expectedTools: [ { - input: `Find database timeouts in ${FIXTURES.organizationSlug} from the last week`, - expectedTools: [ - { - name: "find_organizations", - arguments: {}, - }, - { - name: "search_errors", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - query: "database timeouts from the last week", - }, - }, - ], + name: "find_organizations", + arguments: {}, }, - // Core test: Performance spans search { - input: `Find slow API calls taking over 5 seconds in ${FIXTURES.organizationSlug}`, - expectedTools: [ - { - name: "find_organizations", - arguments: {}, - }, - { - name: "search_traces", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - query: "slow API calls taking over 5 seconds", - }, - }, - ], + name: "search_errors", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + query: "database timeouts from the last week", + }, }, - // Core test: Logs search + ], + }, + // Core test: Performance spans search + { + input: `Find slow API calls taking over 5 seconds in ${FIXTURES.organizationSlug}`, + expectedTools: [ + { + name: "find_organizations", + arguments: {}, + }, + { + name: "search_traces", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + query: "slow API calls taking over 5 seconds", + }, + }, + ], + }, + // Core test: Logs search + { + input: `Show me error logs from the last hour in ${FIXTURES.organizationSlug}`, + expectedTools: [ + { + name: "find_organizations", + arguments: {}, + }, + { + name: "search_logs", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + query: "error logs from the last hour", + }, + }, + ], + }, + // Core test: Project-specific search + { + input: `Show me authentication errors in ${FIXTURES.organizationSlug}/${FIXTURES.projectSlug}`, + expectedTools: [ { - input: `Show me error logs from the last hour in ${FIXTURES.organizationSlug}`, - expectedTools: [ - { - name: "find_organizations", - arguments: {}, - }, - { - name: "search_logs", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - query: "error logs from the last hour", - }, - }, - ], + name: "find_organizations", + arguments: {}, }, - // Core test: Project-specific search { - input: `Show me authentication errors in ${FIXTURES.organizationSlug}/${FIXTURES.projectSlug}`, - expectedTools: [ - { - name: "find_organizations", - arguments: {}, - }, - { - name: "search_errors", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - projectSlug: FIXTURES.projectSlug, - query: "authentication errors", - }, - }, - ], + name: "search_errors", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + projectSlug: FIXTURES.projectSlug, + query: "authentication errors", + }, + }, + ], + }, + // Core test: Search with 'me' reference + { + input: `Show me errors affecting me in ${FIXTURES.organizationSlug}`, + expectedTools: [ + { + name: "find_organizations", + arguments: {}, }, - // Core test: Search with 'me' reference { - input: `Show me errors affecting me in ${FIXTURES.organizationSlug}`, - expectedTools: [ - { - name: "find_organizations", - arguments: {}, - }, - { - name: "whoami", - arguments: {}, - }, - { - name: "search_errors", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - query: "errors affecting user.id:12345", - }, - }, - ], + name: "whoami", + arguments: {}, }, - ]; + { + name: "search_errors", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + query: "errors affecting user.id:12345", + }, + }, + ], + }, +]; + +describeEval( + "search-events", + { + harness: createTaskHarness("search-events", NoOpTaskRunner()), + judges: [ToolPredictionJudge()], + judgeThreshold: 0.6, }, - task: NoOpTaskRunner(), - scorers: [ToolPredictionScorer()], - threshold: 0.6, - timeout: 30000, -}); + (it) => defineEvalCases(it, cases, { timeout: 30000 }), +); diff --git a/packages/mcp-server-evals/src/evals/search-issue-events-agent.eval.ts b/packages/mcp-server-evals/src/evals/search-issue-events-agent.eval.ts index b195cd320..4c7da0066 100644 --- a/packages/mcp-server-evals/src/evals/search-issue-events-agent.eval.ts +++ b/packages/mcp-server-evals/src/evals/search-issue-events-agent.eval.ts @@ -1,128 +1,135 @@ -import { describeEval } from "vitest-evals/legacy"; -import { ToolCallScorer } from "vitest-evals/legacy"; import { searchIssueEventsAgent } from "@sentry/mcp-core/tools/search-issue-events/agent"; import { SentryApiService } from "@sentry/mcp-core/api-client"; -import { StructuredOutputScorer } from "./utils/structuredOutputScorer"; import "../setup-env"; +import { describeEval } from "vitest-evals"; +import { + createTaskHarness, + defineEvalCases, + StructuredOutputJudge, + ToolCallJudge, + type EvalCase, +} from "./utils"; // The shared MSW server is already started in setup-env.ts -describeEval("search-issue-events-agent", { - data: async () => { - return [ - { - // Simple time-based query - should NOT require tool calls - input: "Show me events from the last hour", - expectedTools: [], - expected: { - query: "", // No additional filters beyond issue constraint - sort: "-timestamp", - timeRange: { statsPeriod: "1h" }, - }, - }, - { - // Environment and release filtering - should NOT require tool calls - input: "Find production events with release v1.0.5", - expectedTools: [], - expected: { - query: - /environment:production.*release:v1\.0\.5|release:v1\.0\.5.*environment:production/, - sort: "-timestamp", - }, - }, - { - // User-specific filtering - may require whoami if query uses "me" - input: "Show me events affecting user alice@example.com", - expectedTools: [], - expected: { - query: "user.email:alice@example.com", - sort: "-timestamp", - }, - }, - { - // Query with "me" reference - should require whoami - input: "Show me events from my user", - expectedTools: [ - { - name: "whoami", - arguments: {}, - }, - ], - expected: { - query: /user\.email:test@example\.com|user:test@example\.com/, // Various valid forms - sort: "-timestamp", - }, - }, - { - // Trace ID filtering - should NOT require tool calls - input: "Find events with trace ID abc123def456", - expectedTools: [], - expected: { - query: "trace:abc123def456", - sort: "-timestamp", - }, - }, - { - // URL pattern filtering - should NOT require tool calls - input: "Show me events from the /checkout/ page", - expectedTools: [], - expected: { - query: /"url:.*\/checkout\/.*"|url:".*checkout.*"/, // URL pattern with wildcard - sort: "-timestamp", - }, - }, +const cases: EvalCase[] = [ + { + // Simple time-based query - should NOT require tool calls + input: "Show me events from the last hour", + expectedTools: [], + expected: { + query: "", // No additional filters beyond issue constraint + sort: "-timestamp", + timeRange: { statsPeriod: "1h" }, + }, + }, + { + // Environment and release filtering - should NOT require tool calls + input: "Find production events with release v1.0.5", + expectedTools: [], + expected: { + query: + /environment:production.*release:v1\.0\.5|release:v1\.0\.5.*environment:production/, + sort: "-timestamp", + }, + }, + { + // User-specific filtering - may require whoami if query uses "me" + input: "Show me events affecting user alice@example.com", + expectedTools: [], + expected: { + query: "user.email:alice@example.com", + sort: "-timestamp", + }, + }, + { + // Query with "me" reference - should require whoami + input: "Show me events from my user", + expectedTools: [ { - // Combined filters with time range - input: "Production events from yesterday with the 5.1 release", - expectedTools: [], - expected: { - query: - /environment:production.*release:|release:.*environment:production/, - sort: "-timestamp", - timeRange: { statsPeriod: "24h" }, - }, + name: "whoami", + arguments: {}, }, + ], + expected: { + query: /user\.email:test@example\.com|user:test@example\.com/, // Various valid forms + sort: "-timestamp", + }, + }, + { + // Trace ID filtering - should NOT require tool calls + input: "Find events with trace ID abc123def456", + expectedTools: [], + expected: { + query: "trace:abc123def456", + sort: "-timestamp", + }, + }, + { + // URL pattern filtering - should NOT require tool calls + input: "Show me events from the /checkout/ page", + expectedTools: [], + expected: { + query: /"url:.*\/checkout\/.*"|url:".*checkout.*"/, // URL pattern with wildcard + sort: "-timestamp", + }, + }, + { + // Combined filters with time range + input: "Production events from yesterday with the 5.1 release", + expectedTools: [], + expected: { + query: + /environment:production.*release:|release:.*environment:production/, + sort: "-timestamp", + timeRange: { statsPeriod: "24h" }, + }, + }, + { + // Query that might need field discovery for uncommon tags + input: "Events where device family is mobile", + expectedTools: [ { - // Query that might need field discovery for uncommon tags - input: "Events where device family is mobile", - expectedTools: [ - { - name: "issueEventFields", - arguments: {}, - }, - ], - expected: { - query: /device\.family:mobile|device:mobile/, - sort: "-timestamp", - }, + name: "issueEventFields", + arguments: {}, }, - ]; + ], + expected: { + query: /device\.family:mobile|device:mobile/, + sort: "-timestamp", + }, }, - task: async (input) => { - // Create a real API service that will use MSW mocks - const apiService = new SentryApiService({ - accessToken: "test-token", - }); +]; + +describeEval( + "search-issue-events-agent", + { + harness: createTaskHarness("search-issue-events-agent", async (input) => { + // Create a real API service that will use MSW mocks + const apiService = new SentryApiService({ + accessToken: "test-token", + }); - const agentResult = await searchIssueEventsAgent({ - query: input, - organizationSlug: "sentry-mcp-evals", - apiService, - }); + const agentResult = await searchIssueEventsAgent({ + query: input, + organizationSlug: "sentry-mcp-evals", + apiService, + }); - // Return in the format expected by ToolCallScorer - return { - result: JSON.stringify(agentResult.result), - toolCalls: agentResult.toolCalls.map((call: any) => ({ - name: call.toolName, - arguments: call.args, - })), - }; + // Return in the format expected by ToolCallJudge + return { + result: JSON.stringify(agentResult.result), + toolCalls: agentResult.toolCalls.map((call: any) => ({ + name: call.toolName, + arguments: call.args, + })), + }; + }), + judges: [ + ToolCallJudge(), // Validates tool calls + StructuredOutputJudge({ match: "fuzzy" }), // Validates the structured query output with flexible matching + ], + judgeThreshold: 0.6, }, - scorers: [ - ToolCallScorer(), // Validates tool calls - StructuredOutputScorer({ match: "fuzzy" }), // Validates the structured query output with flexible matching - ], - threshold: 0.6, - timeout: 30000, -}); + (it) => defineEvalCases(it, cases, { timeout: 30000 }), +); diff --git a/packages/mcp-server-evals/src/evals/search-issue-events.eval.ts b/packages/mcp-server-evals/src/evals/search-issue-events.eval.ts index de90883c0..39646ebda 100644 --- a/packages/mcp-server-evals/src/evals/search-issue-events.eval.ts +++ b/packages/mcp-server-evals/src/evals/search-issue-events.eval.ts @@ -1,87 +1,96 @@ -import { describeEval } from "vitest-evals/legacy"; -import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils"; +import { describeEval } from "vitest-evals"; +import { + createTaskHarness, + defineEvalCases, + FIXTURES, + NoOpTaskRunner, + ToolPredictionJudge, + type EvalCase, +} from "./utils"; // Note: This eval requires OPENROUTER_API_KEY to be set in the environment // The search_issue_events tool uses the AI SDK to translate natural language queries -describeEval("search-issue-events", { - data: async () => { - return [ - // Core test: Basic time-based filtering within an issue +const cases: EvalCase[] = [ + // Core test: Basic time-based filtering within an issue + { + input: `Show me events from the last hour in issue ${FIXTURES.issueId} in ${FIXTURES.organizationSlug}`, + expectedTools: [ { - input: `Show me events from the last hour in issue ${FIXTURES.issueId} in ${FIXTURES.organizationSlug}`, - expectedTools: [ - { - name: "find_organizations", - arguments: {}, - }, - { - name: "search_issue_events", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - issueId: FIXTURES.issueId, - query: "from the last hour", - }, - }, - ], + name: "find_organizations", + arguments: {}, }, - // Core test: Environment and release filtering { - input: `Find production events with release v1.0 in issue ${FIXTURES.issueId} in ${FIXTURES.organizationSlug}`, - expectedTools: [ - { - name: "find_organizations", - arguments: {}, - }, - { - name: "search_issue_events", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - issueId: FIXTURES.issueId, - query: "production events with release v1.0", - }, - }, - ], + name: "search_issue_events", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + issueId: FIXTURES.issueId, + query: "from the last hour", + }, + }, + ], + }, + // Core test: Environment and release filtering + { + input: `Find production events with release v1.0 in issue ${FIXTURES.issueId} in ${FIXTURES.organizationSlug}`, + expectedTools: [ + { + name: "find_organizations", + arguments: {}, }, - // Core test: User-specific filtering { - input: `Show me events affecting user alice@example.com in issue ${FIXTURES.issueId} in ${FIXTURES.organizationSlug}`, - expectedTools: [ - { - name: "find_organizations", - arguments: {}, - }, - { - name: "search_issue_events", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - issueId: FIXTURES.issueId, - query: "affecting user alice@example.com", - }, - }, - ], + name: "search_issue_events", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + issueId: FIXTURES.issueId, + query: "production events with release v1.0", + }, }, - // Core test: Trace ID filtering + ], + }, + // Core test: User-specific filtering + { + input: `Show me events affecting user alice@example.com in issue ${FIXTURES.issueId} in ${FIXTURES.organizationSlug}`, + expectedTools: [ { - input: `Find events with trace ID abc123 in issue ${FIXTURES.issueId} in ${FIXTURES.organizationSlug}`, - expectedTools: [ - { - name: "find_organizations", - arguments: {}, - }, - { - name: "search_issue_events", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - issueId: FIXTURES.issueId, - query: "with trace ID abc123", - }, - }, - ], + name: "find_organizations", + arguments: {}, }, - ]; + { + name: "search_issue_events", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + issueId: FIXTURES.issueId, + query: "affecting user alice@example.com", + }, + }, + ], + }, + // Core test: Trace ID filtering + { + input: `Find events with trace ID abc123 in issue ${FIXTURES.issueId} in ${FIXTURES.organizationSlug}`, + expectedTools: [ + { + name: "find_organizations", + arguments: {}, + }, + { + name: "search_issue_events", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + issueId: FIXTURES.issueId, + query: "with trace ID abc123", + }, + }, + ], + }, +]; + +describeEval( + "search-issue-events", + { + harness: createTaskHarness("search-issue-events", NoOpTaskRunner()), + judges: [ToolPredictionJudge()], + judgeThreshold: 0.6, }, - task: NoOpTaskRunner(), - scorers: [ToolPredictionScorer()], - threshold: 0.6, - timeout: 30000, -}); + (it) => defineEvalCases(it, cases, { timeout: 30000 }), +); diff --git a/packages/mcp-server-evals/src/evals/search-issues-agent.eval.ts b/packages/mcp-server-evals/src/evals/search-issues-agent.eval.ts index d5bb58af8..2d837b77f 100644 --- a/packages/mcp-server-evals/src/evals/search-issues-agent.eval.ts +++ b/packages/mcp-server-evals/src/evals/search-issues-agent.eval.ts @@ -1,155 +1,162 @@ import { SentryApiService } from "@sentry/mcp-core/api-client"; import { searchIssuesAgent } from "@sentry/mcp-core/tools/search-issues/agent"; -import { describeEval } from "vitest-evals/legacy"; -import { ToolCallScorer } from "vitest-evals/legacy"; -import { StructuredOutputScorer } from "./utils/structuredOutputScorer"; import "../setup-env"; +import { describeEval } from "vitest-evals"; +import { + createTaskHarness, + defineEvalCases, + StructuredOutputJudge, + ToolCallJudge, + type EvalCase, +} from "./utils"; // The shared MSW server is already started in setup-env.ts -describeEval("search-issues-agent", { - data: async () => { - return [ - { - // Simple query with common fields - should NOT require tool calls - input: "Show me unresolved issues", - expectedTools: [], - expected: { - query: "is:unresolved", - sort: "date", // Agent uses "date" as default - }, - }, - { - // Natural-language "me" reference should resolve through whoami. - input: "Show me issues assigned to me", - expectedTools: [ - { - name: "whoami", - arguments: {}, - }, - ], - expected: { - query: - /assigned_or_suggested:test@example\.com|assigned:test@example\.com|assigned:me/, // Various valid forms - sort: "date", - }, - }, - { - // Explicit "me" is valid Sentry syntax and should not be resolved. - input: "assigned:me is:unresolved", - expectedTools: [], - expected: { - query: /(?=.*assigned:me)(?=.*is:unresolved)/, - sort: "date", - }, - }, - { - // Complex query but with common fields - should NOT require tool calls - input: "Show me critical unhandled errors from the last 24 hours", - expectedTools: [], - expected: { - query: - /(?=.*is:unresolved)(?=.*error\.handled:false)(?=.*lastSeen:-24h)/, - sort: /date|user/, - }, - }, - { - // Tag-presence query can be expressed directly with has: - input: "Show me issues with custom.payment.failed tag", - expectedTools: [], - expected: { - query: - /has:custom\.payment\.failed|custom\.payment\.failed|tags\[custom\.payment\.failed\]/, // All are valid tag forms - sort: "date", // Agent should always return a sort value - }, - }, - { - // Another query requiring field discovery - input: "Find issues where the kafka.consumer.group is orders-processor", - expectedTools: [ - { - name: "issueFields", - arguments: {}, // No arguments needed anymore - }, - ], - expected: { - query: - /kafka\.consumer\.group:orders-processor|tags\[kafka\.consumer\.group\]:orders-processor/, - sort: "date", // Agent should always return a sort value - }, - }, - { - // Easy to fix issues - should use seer_actionability filter - input: "Show me easy to fix bugs", - expectedTools: [], - expected: { - query: /issue\.seer_actionability/, - sort: "date", - }, - }, - { - // Quick wins query - should combine actionability with unresolved - input: "Show me quick wins in production", - expectedTools: [], - expected: { - query: - /issue\.seer_actionability.*environment:production|environment:production.*issue\.seer_actionability/, - sort: /date|user/, - }, - }, - { - // Explicit issue-search syntax should be preserved, not broadened. - input: "is:for_review release:latest assigned:me issue.priority:high", - expectedTools: [], - expected: { - query: - /(?=.*is:for_review)(?=.*release:latest)(?=.*assigned:me)(?=.*issue\.priority:high)/, - sort: "date", - }, - }, +const cases: EvalCase[] = [ + { + // Simple query with common fields - should NOT require tool calls + input: "Show me unresolved issues", + expectedTools: [], + expected: { + query: "is:unresolved", + sort: "date", // Agent uses "date" as default + }, + }, + { + // Natural-language "me" reference should resolve through whoami. + input: "Show me issues assigned to me", + expectedTools: [ { - // Mixed natural language may set sort, but explicit filters stay intact. - input: "sort by users is:for_review release:latest", - expectedTools: [], - expected: { - query: /^(?!.*sort:)(?=.*is:for_review)(?=.*release:latest)/, - sort: "user", - }, + name: "whoami", + arguments: {}, }, + ], + expected: { + query: + /assigned_or_suggested:test@example\.com|assigned:test@example\.com|assigned:me/, // Various valid forms + sort: "date", + }, + }, + { + // Explicit "me" is valid Sentry syntax and should not be resolved. + input: "assigned:me is:unresolved", + expectedTools: [], + expected: { + query: /(?=.*assigned:me)(?=.*is:unresolved)/, + sort: "date", + }, + }, + { + // Complex query but with common fields - should NOT require tool calls + input: "Show me critical unhandled errors from the last 24 hours", + expectedTools: [], + expected: { + query: /(?=.*is:unresolved)(?=.*error\.handled:false)(?=.*lastSeen:-24h)/, + sort: /date|user/, + }, + }, + { + // Tag-presence query can be expressed directly with has: + input: "Show me issues with custom.payment.failed tag", + expectedTools: [], + expected: { + query: + /has:custom\.payment\.failed|custom\.payment\.failed|tags\[custom\.payment\.failed\]/, // All are valid tag forms + sort: "date", // Agent should always return a sort value + }, + }, + { + // Another query requiring field discovery + input: "Find issues where the kafka.consumer.group is orders-processor", + expectedTools: [ { - // Valid inbox/substatus filters should not be generalized. - input: "is:new is:regressed", - expectedTools: [], - expected: { - query: /^(?!.*is:unresolved)(?=.*is:new)(?=.*is:regressed)/, - sort: "date", - }, + name: "issueFields", + arguments: {}, // No arguments needed anymore }, - ]; + ], + expected: { + query: + /kafka\.consumer\.group:orders-processor|tags\[kafka\.consumer\.group\]:orders-processor/, + sort: "date", // Agent should always return a sort value + }, + }, + { + // Easy to fix issues - should use seer_actionability filter + input: "Show me easy to fix bugs", + expectedTools: [], + expected: { + query: /issue\.seer_actionability/, + sort: "date", + }, + }, + { + // Quick wins query - should combine actionability with unresolved + input: "Show me quick wins in production", + expectedTools: [], + expected: { + query: + /issue\.seer_actionability.*environment:production|environment:production.*issue\.seer_actionability/, + sort: /date|user/, + }, + }, + { + // Explicit issue-search syntax should be preserved, not broadened. + input: "is:for_review release:latest assigned:me issue.priority:high", + expectedTools: [], + expected: { + query: + /(?=.*is:for_review)(?=.*release:latest)(?=.*assigned:me)(?=.*issue\.priority:high)/, + sort: "date", + }, }, - task: async (input) => { - // Create a real API service that will use MSW mocks - const apiService = new SentryApiService({ - accessToken: "test-token", - }); + { + // Mixed natural language may set sort, but explicit filters stay intact. + input: "sort by users is:for_review release:latest", + expectedTools: [], + expected: { + query: /^(?!.*sort:)(?=.*is:for_review)(?=.*release:latest)/, + sort: "user", + }, + }, + { + // Valid inbox/substatus filters should not be generalized. + input: "is:new is:regressed", + expectedTools: [], + expected: { + query: /^(?!.*is:unresolved)(?=.*is:new)(?=.*is:regressed)/, + sort: "date", + }, + }, +]; + +describeEval( + "search-issues-agent", + { + harness: createTaskHarness("search-issues-agent", async (input) => { + // Create a real API service that will use MSW mocks + const apiService = new SentryApiService({ + accessToken: "test-token", + }); - const agentResult = await searchIssuesAgent({ - query: input, - organizationSlug: "sentry-mcp-evals", - apiService, - }); + const agentResult = await searchIssuesAgent({ + query: input, + organizationSlug: "sentry-mcp-evals", + apiService, + }); - // Return in the format expected by ToolCallScorer - return { - result: JSON.stringify(agentResult.result), - toolCalls: agentResult.toolCalls.map((call: any) => ({ - name: call.toolName, - arguments: call.args, - })), - }; + // Return in the format expected by ToolCallJudge + return { + result: JSON.stringify(agentResult.result), + toolCalls: agentResult.toolCalls.map((call: any) => ({ + name: call.toolName, + arguments: call.args, + })), + }; + }), + judges: [ + ToolCallJudge(), // Validates tool calls + StructuredOutputJudge({ match: "fuzzy" }), // Validates the structured query output with flexible matching + ], }, - scorers: [ - ToolCallScorer(), // Validates tool calls - StructuredOutputScorer({ match: "fuzzy" }), // Validates the structured query output with flexible matching - ], -}); + (it) => defineEvalCases(it, cases), +); diff --git a/packages/mcp-server-evals/src/evals/search-issues.eval.ts b/packages/mcp-server-evals/src/evals/search-issues.eval.ts index d0da1de57..b9222d440 100644 --- a/packages/mcp-server-evals/src/evals/search-issues.eval.ts +++ b/packages/mcp-server-evals/src/evals/search-issues.eval.ts @@ -1,88 +1,97 @@ -import { describeEval } from "vitest-evals/legacy"; -import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils"; +import { describeEval } from "vitest-evals"; +import { + createTaskHarness, + defineEvalCases, + FIXTURES, + NoOpTaskRunner, + ToolPredictionJudge, + type EvalCase, +} from "./utils"; // Note: This eval requires OPENROUTER_API_KEY to be set in the environment // The search_issues tool uses the AI SDK to translate natural language queries -describeEval("search-issues", { - data: async () => { - return [ - // Core test: Basic issue search +const cases: EvalCase[] = [ + // Core test: Basic issue search + { + input: `Show me unresolved issues in ${FIXTURES.organizationSlug}`, + expectedTools: [ { - input: `Show me unresolved issues in ${FIXTURES.organizationSlug}`, - expectedTools: [ - { - name: "find_organizations", - arguments: {}, - }, - { - name: "search_issues", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - query: "unresolved issues", - }, - }, - ], + name: "find_organizations", + arguments: {}, }, - // Core test: Search with 'me' reference (tests whoami integration) { - input: `Find issues assigned to me in ${FIXTURES.organizationSlug}`, - expectedTools: [ - { - name: "find_organizations", - arguments: {}, - }, - { - name: "whoami", - arguments: {}, - }, - { - name: "search_issues", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - query: "issues assigned to me", - }, - }, - ], + name: "search_issues", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + query: "unresolved issues", + }, }, - // Core test: Project-specific search + ], + }, + // Core test: Search with 'me' reference (tests whoami integration) + { + input: `Find issues assigned to me in ${FIXTURES.organizationSlug}`, + expectedTools: [ + { + name: "find_organizations", + arguments: {}, + }, + { + name: "whoami", + arguments: {}, + }, + { + name: "search_issues", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + query: "issues assigned to me", + }, + }, + ], + }, + // Core test: Project-specific search + { + input: `Search for database errors in ${FIXTURES.organizationSlug}/${FIXTURES.projectSlug}`, + expectedTools: [ { - input: `Search for database errors in ${FIXTURES.organizationSlug}/${FIXTURES.projectSlug}`, - expectedTools: [ - { - name: "find_organizations", - arguments: {}, - }, - { - name: "search_issues", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - projectSlugOrId: FIXTURES.projectSlug, - query: "database errors", - }, - }, - ], + name: "find_organizations", + arguments: {}, }, - // Core test: Complex natural language query { - input: `Find critical production errors affecting more than 100 users in ${FIXTURES.organizationSlug}`, - expectedTools: [ - { - name: "find_organizations", - arguments: {}, - }, - { - name: "search_issues", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - query: "critical production errors affecting more than 100 users", - }, - }, - ], + name: "search_issues", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + projectSlugOrId: FIXTURES.projectSlug, + query: "database errors", + }, }, - ]; + ], + }, + // Core test: Complex natural language query + { + input: `Find critical production errors affecting more than 100 users in ${FIXTURES.organizationSlug}`, + expectedTools: [ + { + name: "find_organizations", + arguments: {}, + }, + { + name: "search_issues", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + query: "critical production errors affecting more than 100 users", + }, + }, + ], + }, +]; + +describeEval( + "search-issues", + { + harness: createTaskHarness("search-issues", NoOpTaskRunner()), + judges: [ToolPredictionJudge()], + judgeThreshold: 0.6, }, - task: NoOpTaskRunner(), - scorers: [ToolPredictionScorer()], - threshold: 0.6, - timeout: 30000, -}); + (it) => defineEvalCases(it, cases, { timeout: 30000 }), +); diff --git a/packages/mcp-server-evals/src/evals/update-issue.eval.ts b/packages/mcp-server-evals/src/evals/update-issue.eval.ts index 8ff652e3e..01333a996 100644 --- a/packages/mcp-server-evals/src/evals/update-issue.eval.ts +++ b/packages/mcp-server-evals/src/evals/update-issue.eval.ts @@ -1,125 +1,134 @@ -import { describeEval } from "vitest-evals/legacy"; -import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils"; +import { describeEval } from "vitest-evals"; +import { + createTaskHarness, + defineEvalCases, + FIXTURES, + NoOpTaskRunner, + ToolPredictionJudge, + type EvalCase, +} from "./utils"; -describeEval("update-issue", { - data: async () => { - return [ - // Core use case: Resolve an issue +const cases: EvalCase[] = [ + // Core use case: Resolve an issue + { + input: `Resolve the issue ${FIXTURES.issueId} in organization ${FIXTURES.organizationSlug}. Output only the new status as a single word.`, + expectedTools: [ { - input: `Resolve the issue ${FIXTURES.issueId} in organization ${FIXTURES.organizationSlug}. Output only the new status as a single word.`, - expectedTools: [ - { - name: "find_organizations", - arguments: {}, - }, - { - name: "update_issue", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - issueId: FIXTURES.issueId, - status: "resolved", - regionUrl: "https://us.sentry.io", - }, - }, - ], + name: "find_organizations", + arguments: {}, }, - // Core use case: Assign an issue { - input: `Assign the issue ${FIXTURES.issueId} in organization ${FIXTURES.organizationSlug} to 'john.doe'. Output only the assigned username.`, - expectedTools: [ - { - name: "find_organizations", - arguments: {}, - }, - { - name: "update_issue", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - issueId: FIXTURES.issueId, - assignedTo: "john.doe", - regionUrl: "https://us.sentry.io", - }, - }, - ], + name: "update_issue", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + issueId: FIXTURES.issueId, + status: "resolved", + regionUrl: "https://us.sentry.io", + }, }, - // Core use case: Using issue URL (alternative input method) + ], + }, + // Core use case: Assign an issue + { + input: `Assign the issue ${FIXTURES.issueId} in organization ${FIXTURES.organizationSlug} to 'john.doe'. Output only the assigned username.`, + expectedTools: [ { - input: `Resolve the issue at ${FIXTURES.issueUrl}. Output only the new status as a single word.`, - expectedTools: [ - { - name: "update_issue", - arguments: { - issueUrl: FIXTURES.issueUrl, - status: "resolved", - }, - }, - ], + name: "find_organizations", + arguments: {}, }, - // Regression: default ignored status should map to "until escalating" { - input: `Ignore the issue ${FIXTURES.issueId} in organization ${FIXTURES.organizationSlug} until it escalates. Output only the new status as a single word.`, - expectedTools: [ - { - name: "find_organizations", - arguments: {}, - }, - { - name: "update_issue", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - issueId: FIXTURES.issueId, - status: "ignored", - regionUrl: "https://us.sentry.io", - }, - }, - ], + name: "update_issue", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + issueId: FIXTURES.issueId, + assignedTo: "john.doe", + regionUrl: "https://us.sentry.io", + }, }, - // Regression: permanent ignores need the explicit forever mode + ], + }, + // Core use case: Using issue URL (alternative input method) + { + input: `Resolve the issue at ${FIXTURES.issueUrl}. Output only the new status as a single word.`, + expectedTools: [ + { + name: "update_issue", + arguments: { + issueUrl: FIXTURES.issueUrl, + status: "resolved", + }, + }, + ], + }, + // Regression: default ignored status should map to "until escalating" + { + input: `Ignore the issue ${FIXTURES.issueId} in organization ${FIXTURES.organizationSlug} until it escalates. Output only the new status as a single word.`, + expectedTools: [ { - input: `Ignore the issue ${FIXTURES.issueId} in organization ${FIXTURES.organizationSlug} forever. Output only the new status as a single word.`, - expectedTools: [ - { - name: "find_organizations", - arguments: {}, - }, - { - name: "update_issue", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - issueId: FIXTURES.issueId, - status: "ignored", - ignoreMode: "forever", - regionUrl: "https://us.sentry.io", - }, - }, - ], + name: "find_organizations", + arguments: {}, }, - // Regression: count-based ignores should use the structured ignore fields { - input: `Ignore the issue ${FIXTURES.issueId} in organization ${FIXTURES.organizationSlug} until it happens 100 times in 60 minutes. Output only the new status as a single word.`, - expectedTools: [ - { - name: "find_organizations", - arguments: {}, - }, - { - name: "update_issue", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - issueId: FIXTURES.issueId, - status: "ignored", - ignoreMode: "untilOccurrenceCount", - ignoreCount: 100, - ignoreWindowMinutes: 60, - regionUrl: "https://us.sentry.io", - }, - }, - ], + name: "update_issue", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + issueId: FIXTURES.issueId, + status: "ignored", + regionUrl: "https://us.sentry.io", + }, }, - ]; + ], + }, + // Regression: permanent ignores need the explicit forever mode + { + input: `Ignore the issue ${FIXTURES.issueId} in organization ${FIXTURES.organizationSlug} forever. Output only the new status as a single word.`, + expectedTools: [ + { + name: "find_organizations", + arguments: {}, + }, + { + name: "update_issue", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + issueId: FIXTURES.issueId, + status: "ignored", + ignoreMode: "forever", + regionUrl: "https://us.sentry.io", + }, + }, + ], + }, + // Regression: count-based ignores should use the structured ignore fields + { + input: `Ignore the issue ${FIXTURES.issueId} in organization ${FIXTURES.organizationSlug} until it happens 100 times in 60 minutes. Output only the new status as a single word.`, + expectedTools: [ + { + name: "find_organizations", + arguments: {}, + }, + { + name: "update_issue", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + issueId: FIXTURES.issueId, + status: "ignored", + ignoreMode: "untilOccurrenceCount", + ignoreCount: 100, + ignoreWindowMinutes: 60, + regionUrl: "https://us.sentry.io", + }, + }, + ], + }, +]; + +describeEval( + "update-issue", + { + harness: createTaskHarness("update-issue", NoOpTaskRunner()), + judges: [ToolPredictionJudge()], + judgeThreshold: 0.6, }, - task: NoOpTaskRunner(), - scorers: [ToolPredictionScorer()], - threshold: 0.6, - timeout: 30000, -}); + (it) => defineEvalCases(it, cases, { timeout: 30000 }), +); diff --git a/packages/mcp-server-evals/src/evals/update-project.eval.ts b/packages/mcp-server-evals/src/evals/update-project.eval.ts index a2b9e435b..0cfed26d9 100644 --- a/packages/mcp-server-evals/src/evals/update-project.eval.ts +++ b/packages/mcp-server-evals/src/evals/update-project.eval.ts @@ -1,40 +1,49 @@ -import { describeEval } from "vitest-evals/legacy"; -import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils"; +import { describeEval } from "vitest-evals"; +import { + createTaskHarness, + defineEvalCases, + FIXTURES, + NoOpTaskRunner, + ToolPredictionJudge, + type EvalCase, +} from "./utils"; -describeEval("update-project", { - data: async () => { - return [ +const cases: EvalCase[] = [ + { + input: `Update the project '${FIXTURES.projectSlug}' in organization '${FIXTURES.organizationSlug}' to change its name to 'Updated Project Name' and slug to 'updated-project-slug'. Output only the new project slug as plain text without any formatting:\nupdated-project-slug`, + expectedTools: [ { - input: `Update the project '${FIXTURES.projectSlug}' in organization '${FIXTURES.organizationSlug}' to change its name to 'Updated Project Name' and slug to 'updated-project-slug'. Output only the new project slug as plain text without any formatting:\nupdated-project-slug`, - expectedTools: [ - { - name: "update_project", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - projectSlug: FIXTURES.projectSlug, - name: "Updated Project Name", - slug: "updated-project-slug", - }, - }, - ], + name: "update_project", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + projectSlug: FIXTURES.projectSlug, + name: "Updated Project Name", + slug: "updated-project-slug", + }, }, + ], + }, + { + input: `Grant the team '${FIXTURES.teamSlug}' access to the project '${FIXTURES.projectSlug}' in organization '${FIXTURES.organizationSlug}'. Output only the team slug as plain text without any formatting:\nthe-goats`, + expectedTools: [ { - input: `Grant the team '${FIXTURES.teamSlug}' access to the project '${FIXTURES.projectSlug}' in organization '${FIXTURES.organizationSlug}'. Output only the team slug as plain text without any formatting:\nthe-goats`, - expectedTools: [ - { - name: "add_team_to_project", - arguments: { - organizationSlug: FIXTURES.organizationSlug, - projectSlug: FIXTURES.projectSlug, - teamSlug: FIXTURES.teamSlug, - }, - }, - ], + name: "add_team_to_project", + arguments: { + organizationSlug: FIXTURES.organizationSlug, + projectSlug: FIXTURES.projectSlug, + teamSlug: FIXTURES.teamSlug, + }, }, - ]; + ], + }, +]; + +describeEval( + "update-project", + { + harness: createTaskHarness("update-project", NoOpTaskRunner()), + judges: [ToolPredictionJudge()], + judgeThreshold: 0.6, }, - task: NoOpTaskRunner(), - scorers: [ToolPredictionScorer()], - threshold: 0.6, - timeout: 30000, -}); + (it) => defineEvalCases(it, cases, { timeout: 30000 }), +); diff --git a/packages/mcp-server-evals/src/evals/utils/harness.ts b/packages/mcp-server-evals/src/evals/utils/harness.ts new file mode 100644 index 000000000..4583c9d2e --- /dev/null +++ b/packages/mcp-server-evals/src/evals/utils/harness.ts @@ -0,0 +1,95 @@ +import { + attachHarnessRunToError, + createFailedHarnessRun, + createHarness, + toJsonValue, + type EvalTestAPI, + type Harness, + type JsonValue, + type TranscriptEvent, +} from "vitest-evals"; + +export type ExpectedToolCall = { + name: string; + arguments?: Record; +}; + +/** + * One eval scenario. The task only sees `input`; judges read the expectations + * from the case, because suite-level judges receive the harness input. + */ +export type EvalCase = { + input: string; + name?: string; + expectedTools?: ExpectedToolCall[]; + expected?: Record; +}; + +type EvalTaskResult = { + result: string; + toolCalls?: Array<{ name: string; arguments?: unknown }>; +}; + +type EvalTaskRunner = (input: string) => Promise; + +type EvalHarness = Harness; + +function toJsonObject(value: unknown): Record | undefined { + const normalized = toJsonValue(value); + return normalized && + typeof normalized === "object" && + !Array.isArray(normalized) + ? normalized + : undefined; +} + +/** Adapts a task runner into a harness with a normalized transcript. */ +export function createTaskHarness( + name: string, + task: EvalTaskRunner, +): EvalHarness { + return createHarness({ + name, + run: async ({ input: { input } }) => { + let result: EvalTaskResult; + try { + result = await task(input); + } catch (error) { + // Record the prompt, not the whole case with its expectations. + throw attachHarnessRunToError( + error, + createFailedHarnessRun(input, error), + ); + } + + const events: TranscriptEvent[] = [ + { type: "message", role: "user", content: input }, + ...(result.toolCalls ?? []).map((call, index): TranscriptEvent => { + const args = toJsonObject(call.arguments); + return { + type: "tool_call", + id: `tool-${index}`, + name: call.name, + ...(args ? { arguments: args } : {}), + }; + }), + { type: "message", role: "assistant", content: result.result }, + ]; + + return { output: result.result, events }; + }, + }); +} + +/** Registers one test per case; each runs the suite harness and judges once. */ +export function defineEvalCases( + it: EvalTestAPI, + cases: EvalCase[], + { timeout = 60000 }: { timeout?: number } = {}, +) { + for (const testCase of cases) { + it(testCase.name ?? testCase.input, { timeout }, async ({ run }) => { + await run(testCase); + }); + } +} diff --git a/packages/mcp-server-evals/src/evals/utils/index.ts b/packages/mcp-server-evals/src/evals/utils/index.ts index 0316b2a61..538266071 100644 --- a/packages/mcp-server-evals/src/evals/utils/index.ts +++ b/packages/mcp-server-evals/src/evals/utils/index.ts @@ -1,7 +1,12 @@ export { FIXTURES } from "./fixtures"; -export { McpToolCallTaskRunner } from "./mcpToolCallRunner"; -export { NoOpTaskRunner } from "./runner"; export { - ToolPredictionScorer, + createTaskHarness, + defineEvalCases, + type EvalCase, type ExpectedToolCall, -} from "./toolPredictionScorer"; +} from "./harness"; +export { McpToolCallTaskRunner } from "./mcpToolCallRunner"; +export { NoOpTaskRunner } from "./runner"; +export { StructuredOutputJudge } from "./structuredOutputJudge"; +export { ToolCallJudge } from "./toolCallJudge"; +export { ToolPredictionJudge } from "./toolPredictionJudge"; diff --git a/packages/mcp-server-evals/src/evals/utils/runner.ts b/packages/mcp-server-evals/src/evals/utils/runner.ts index 7a8e6d105..18cda3402 100644 --- a/packages/mcp-server-evals/src/evals/utils/runner.ts +++ b/packages/mcp-server-evals/src/evals/utils/runner.ts @@ -1,6 +1,6 @@ /** * A no-op task runner that doesn't execute tools, just returns the input - * for use with ToolPredictionScorer. This allows tests to focus on predicting + * for use with ToolPredictionJudge. This allows tests to focus on predicting * which tools would be called without actually executing them. */ export function NoOpTaskRunner() { diff --git a/packages/mcp-server-evals/src/evals/utils/structuredOutputScorer.ts b/packages/mcp-server-evals/src/evals/utils/structuredOutputJudge.ts similarity index 51% rename from packages/mcp-server-evals/src/evals/utils/structuredOutputScorer.ts rename to packages/mcp-server-evals/src/evals/utils/structuredOutputJudge.ts index 087a4a2dc..c3e2a2378 100644 --- a/packages/mcp-server-evals/src/evals/utils/structuredOutputScorer.ts +++ b/packages/mcp-server-evals/src/evals/utils/structuredOutputJudge.ts @@ -1,10 +1,7 @@ -import type { Score, ScoreFn, BaseScorerOptions } from "vitest-evals/legacy"; +import { createJudge, type Judge, type JudgeContext } from "vitest-evals"; +import type { EvalCase } from "./harness"; -interface StructuredOutputScorerOptions extends BaseScorerOptions { - expected?: Record; -} - -interface StructuredOutputScorerConfig { +interface StructuredOutputJudgeConfig { /** * How to match field values * - "strict": Exact equality required (default) @@ -38,50 +35,39 @@ interface StructuredOutputScorerConfig { } /** - * A configurable scorer for evaluating structured outputs (e.g., JSON) from LLM responses. + * A configurable judge for evaluating structured outputs (e.g., JSON) from LLM responses. * - * Similar to ToolCallScorer but for validating structured data outputs like API queries. + * Reads the expected fields from the eval case (`input.expected`). This stays + * local instead of using the built-in vitest-evals `StructuredOutputJudge`, + * whose fuzzy matching (substring, case-insensitive) would change scores. * - * @param config - Configuration options for the scorer + * @param config - Configuration options for the judge * @param config.match - How to match field values: "strict", "fuzzy", or custom function * @param config.requireAll - Require all expected fields (vs partial credit) * @param config.allowExtras - Allow additional fields in output * @param config.debug - Enable debug logging * * @example - * // Default: strict matching - * describeEval("query generation", { - * data: async () => [{ - * input: "Show me errors from today", - * expected: { - * dataset: "errors", - * query: "", - * sort: "-timestamp", - * timeRange: { statsPeriod: "24h" } - * } - * }], - * task: myTask, - * scorers: [StructuredOutputScorer()] - * }); - * - * @example - * // Fuzzy matching with regex patterns - * describeEval("flexible query matching", { - * data: async () => [{ - * input: "Find slow API calls", - * expected: { - * dataset: "spans", - * query: /span\.duration:>1000|span\.duration:>1s/, - * sort: "-span.duration" - * } - * }], - * task: myTask, - * scorers: [StructuredOutputScorer({ match: "fuzzy" })] - * }); + * describeEval( + * "query generation", + * { harness, judges: [StructuredOutputJudge({ match: "fuzzy" })] }, + * (it) => { + * defineEvalCases(it, [ + * { + * input: "Find slow API calls", + * expected: { + * dataset: "spans", + * query: /span\.duration:>1000|span\.duration:>1s/, + * sort: "-span.duration", + * }, + * }, + * ]); + * }, + * ); */ -export function StructuredOutputScorer( - config: StructuredOutputScorerConfig = {}, -): ScoreFn { +export function StructuredOutputJudge( + config: StructuredOutputJudgeConfig = {}, +): Judge> { const { match = "strict", requireAll = true, @@ -89,104 +75,105 @@ export function StructuredOutputScorer( debug = false, } = config; - return async (opts: StructuredOutputScorerOptions): Promise => { - const { output, expected } = opts; + return createJudge>( + "StructuredOutputJudge", + ({ input: { expected }, output }) => { + // If no expected output provided, just check if we got valid JSON + if (!expected) { + try { + JSON.parse(output); + return { score: 1, metadata: { rationale: "Valid JSON output" } }; + } catch { + return { score: 0, metadata: { rationale: "Invalid JSON output" } }; + } + } - // If no expected output provided, just check if we got valid JSON - if (!expected) { + let parsed: Record; try { - JSON.parse(output); - return { score: 1, metadata: { rationale: "Valid JSON output" } }; - } catch { - return { score: 0, metadata: { rationale: "Invalid JSON output" } }; + parsed = JSON.parse(output); + } catch (error) { + return { + score: 0, + metadata: { + rationale: `Failed to parse output as JSON: ${String(error)}`, + }, + }; } - } - - let parsed: Record; - try { - parsed = JSON.parse(output); - } catch (error) { - return { - score: 0, - metadata: { - rationale: `Failed to parse output as JSON: ${String(error)}`, - }, - }; - } - - // Check for error field in output - if (parsed.error && parsed.error !== "" && parsed.error !== null) { - return { - score: 0, - metadata: { rationale: `Output contains error: ${parsed.error}` }, - }; - } - const matchFn = getMatchFunction(match); - const { matches, mismatches, extras } = compareObjects( - expected, - parsed, - matchFn, - ); + // Check for error field in output + if (parsed.error && parsed.error !== "" && parsed.error !== null) { + return { + score: 0, + metadata: { rationale: `Output contains error: ${parsed.error}` }, + }; + } - if (debug) { - console.log("StructuredOutputScorer debug:"); - console.log("Expected:", expected); - console.log("Actual:", parsed); - console.log("Matches:", matches); - console.log("Mismatches:", mismatches); - console.log("Extras:", extras); - } + const matchFn = getMatchFunction(match); + const { matches, mismatches, extras } = compareObjects( + expected, + parsed, + matchFn, + ); + + if (debug) { + console.log("StructuredOutputJudge debug:"); + console.log("Expected:", expected); + console.log("Actual:", parsed); + console.log("Matches:", matches); + console.log("Mismatches:", mismatches); + console.log("Extras:", extras); + } - // Calculate score - const totalExpected = Object.keys(expected).length; - const totalMatched = matches.length; - const hasExtras = extras.length > 0; - - let score: number; - let rationale: string; - - if (requireAll && mismatches.length > 0) { - score = 0; - rationale = `Missing required fields: ${mismatches.map((m) => m.key).join(", ")}`; - } else if (!allowExtras && hasExtras) { - score = 0; - rationale = `Unexpected extra fields: ${extras.join(", ")}`; - } else if (totalExpected === 0) { - score = 1; - rationale = "No expected fields to match"; - } else { - score = totalMatched / totalExpected; - if (score === 1) { - rationale = "All expected fields match"; + // Calculate score + const totalExpected = Object.keys(expected).length; + const totalMatched = matches.length; + const hasExtras = extras.length > 0; + + let score: number; + let rationale: string; + + if (requireAll && mismatches.length > 0) { + score = 0; + rationale = `Missing required fields: ${mismatches.map((m) => m.key).join(", ")}`; + } else if (!allowExtras && hasExtras) { + score = 0; + rationale = `Unexpected extra fields: ${extras.join(", ")}`; + } else if (totalExpected === 0) { + score = 1; + rationale = "No expected fields to match"; } else { - rationale = `Matched ${totalMatched}/${totalExpected} fields`; + score = totalMatched / totalExpected; + if (score === 1) { + rationale = "All expected fields match"; + } else { + rationale = `Matched ${totalMatched}/${totalExpected} fields`; + } } - } - // Add mismatch details to rationale - if (mismatches.length > 0 && score < 1) { - const details = mismatches - .map( - (m) => - `${m.key}: expected ${formatValue(m.expected)}, got ${formatValue(m.actual)}`, - ) - .join("; "); - rationale += ` - ${details}`; - } + // Add mismatch details to rationale + if (mismatches.length > 0 && score < 1) { + const details = mismatches + .map( + (m) => + `${m.key}: expected ${formatValue(m.expected)}, got ${formatValue(m.actual)}`, + ) + .join("; "); + rationale += ` - ${details}`; + } - return { - score, - metadata: { - rationale, - output, - }, - }; - }; + return { + score, + metadata: { + rationale, + output, + }, + }; + }, + ); } function getMatchFunction( - match: StructuredOutputScorerConfig["match"], + match: StructuredOutputJudgeConfig["match"], ): (expected: any, actual: any, key: string) => boolean { if (typeof match === "function") { return match; diff --git a/packages/mcp-server-evals/src/evals/utils/toolCallJudge.ts b/packages/mcp-server-evals/src/evals/utils/toolCallJudge.ts new file mode 100644 index 000000000..4b6a5579a --- /dev/null +++ b/packages/mcp-server-evals/src/evals/utils/toolCallJudge.ts @@ -0,0 +1,22 @@ +import { + createJudge, + ToolCallJudge as BaseToolCallJudge, + type Judge, + type JudgeContext, + type ToolCallJudgeConfig, +} from "vitest-evals"; +import type { EvalCase } from "./harness"; + +/** Checks the run's tool calls against the eval case's `expectedTools`. */ +export function ToolCallJudge( + config: ToolCallJudgeConfig = {}, +): Judge> { + const judge = BaseToolCallJudge(config); + return createJudge>("ToolCallJudge", (ctx) => + judge.assess({ + ...ctx, + input: ctx.input.input, + expectedTools: ctx.input.expectedTools, + }), + ); +} diff --git a/packages/mcp-server-evals/src/evals/utils/toolPredictionScorer.ts b/packages/mcp-server-evals/src/evals/utils/toolPredictionJudge.ts similarity index 69% rename from packages/mcp-server-evals/src/evals/utils/toolPredictionScorer.ts rename to packages/mcp-server-evals/src/evals/utils/toolPredictionJudge.ts index 4379d84b9..21a5f611e 100644 --- a/packages/mcp-server-evals/src/evals/utils/toolPredictionScorer.ts +++ b/packages/mcp-server-evals/src/evals/utils/toolPredictionJudge.ts @@ -3,10 +3,17 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { generateObject, type LanguageModel } from "ai"; +import { + createJudge, + toJsonValue, + type Judge, + type JudgeContext, +} from "vitest-evals"; import { z } from "zod"; import { experimental_createMCPClient } from "@ai-sdk/mcp"; import { Experimental_StdioMCPTransport } from "@ai-sdk/mcp/mcp-stdio"; import { getOpenRouterModel } from "@sentry/mcp-core/internal/agents/openrouter-provider"; +import type { EvalCase } from "./harness"; const CLI_DIR = fileURLToPath(new URL("../../../../cli", import.meta.url)); @@ -66,18 +73,6 @@ async function getAvailableTools(): Promise { return cachedTools; } -export interface ExpectedToolCall { - name: string; - arguments: Record; -} - -interface ToolPredictionScorerOptions { - input: string; - output: string; - expectedTools?: ExpectedToolCall[]; - result?: any; -} - const predictionSchema = z.object({ score: z.number().min(0).max(1).describe("Score from 0 to 1"), rationale: z.string().describe("Explanation of the score"), @@ -138,38 +133,48 @@ CRITICAL: The expected tools represent the actual realistic behavior for this sp } /** - * A scorer that uses AI to predict what tools would be called without executing them. + * A judge that uses AI to predict what tools would be called without executing them. * This is much faster than actually running the tools and checking what was called. * * @param model - Optional language model to use for predictions (defaults to OpenRouter) - * @returns A scorer function that compares predicted vs expected tool calls + * @returns A judge that compares predicted vs expected tool calls * * @example * ```typescript - * import { ToolPredictionScorer } from './utils/toolPredictionScorer'; - * import { NoOpTaskRunner } from './utils/runner'; - * import { describeEval } from 'vitest-evals/legacy'; + * import { describeEval } from "vitest-evals"; + * import { + * createTaskHarness, + * defineEvalCases, + * NoOpTaskRunner, + * ToolPredictionJudge, + * } from "./utils"; * - * describeEval("Sentry issue search", { - * data: async () => [ - * { - * input: "Find the newest issues in my-org", - * expectedTools: [ - * { name: "find_organizations", arguments: {} }, - * { name: "find_issues", arguments: { organizationSlug: "my-org", sortBy: "first_seen" } } - * ] - * } - * ], - * task: NoOpTaskRunner(), // Don't execute tools, just predict them - * scorers: [ToolPredictionScorer()], - * threshold: 0.8 - * }); + * describeEval( + * "Sentry issue search", + * { + * // Don't execute tools, just predict them + * harness: createTaskHarness("Sentry issue search", NoOpTaskRunner()), + * judges: [ToolPredictionJudge()], + * judgeThreshold: 0.8, + * }, + * (it) => { + * defineEvalCases(it, [ + * { + * input: "Find the newest issues in my-org", + * expectedTools: [ + * { name: "find_organizations", arguments: {} }, + * { name: "find_issues", arguments: { organizationSlug: "my-org", sortBy: "first_seen" } }, + * ], + * }, + * ]); + * }, + * ); * ``` * - * The scorer works by: + * The judge works by: * 1. Connecting to the MCP server to get available tools and their descriptions * 2. Using AI to predict what tools would be called for the given task - * 3. Comparing predictions against the expectedTools array + * 3. Comparing predictions against the case's `expectedTools` array * 4. Returning a score from 0.0 to 1.0 based on accuracy * * Scoring criteria: @@ -179,65 +184,64 @@ CRITICAL: The expected tools represent the actual realistic behavior for this sp * - 0.3: Some expected tools predicted but significant issues * - 0.0: Wrong tools or critical tools missing * - * If `expectedTools` is not provided in test data, the scorer is automatically skipped - * and returns `{ score: null }` to allow other scorers to run without interference. + * If a case has no `expectedTools`, the judge is skipped and returns + * `{ score: null }`. */ -export function ToolPredictionScorer( +export function ToolPredictionJudge( model: LanguageModel = getOpenRouterModel(), -) { - return async function ToolPredictionScorer( - opts: ToolPredictionScorerOptions, - ) { - // If expectedTools is not defined, skip this scorer - if (!opts.expectedTools) { +): Judge> { + return createJudge>( + "ToolPredictionJudge", + async ({ input: { input, expectedTools } }) => { + // If expectedTools is not defined, skip this judge + if (!expectedTools) { + return { + score: null, + metadata: { + rationale: "Skipped: No expectedTools defined for this test case", + }, + }; + } + + // Get available tools from the MCP server + const AVAILABLE_TOOLS = await getAvailableTools(); + + // Generate a description of the expected tools for the prompt + const expectedDescription = expectedTools + .map( + (tool) => + `- ${tool.name} with arguments: ${JSON.stringify(tool.arguments)}`, + ) + .join("\n"); + + const { object } = await generateObject({ + model, + prompt: generateSystemPrompt( + AVAILABLE_TOOLS, + input, + expectedDescription, + ), + schema: predictionSchema, + providerOptions: { + openai: { + structuredOutputs: false, + strictJsonSchema: false, + }, + }, + experimental_telemetry: { + isEnabled: true, + functionId: "tool_prediction_scorer", + }, + }); + return { - score: null, + score: object.score, metadata: { - rationale: "Skipped: No expectedTools defined for this test case", + rationale: object.rationale, + predictedTools: object.predictedTools, + expectedTools: toJsonValue(expectedTools), }, }; - } - - const expectedTools = opts.expectedTools; - - // Get available tools from the MCP server - const AVAILABLE_TOOLS = await getAvailableTools(); - - // Generate a description of the expected tools for the prompt - const expectedDescription = expectedTools - .map( - (tool) => - `- ${tool.name} with arguments: ${JSON.stringify(tool.arguments)}`, - ) - .join("\n"); - - const { object } = await generateObject({ - model, - prompt: generateSystemPrompt( - AVAILABLE_TOOLS, - opts.input, - expectedDescription, - ), - schema: predictionSchema, - providerOptions: { - openai: { - structuredOutputs: false, - strictJsonSchema: false, - }, - }, - experimental_telemetry: { - isEnabled: true, - functionId: "tool_prediction_scorer", - }, - }); - - return { - score: object.score, - metadata: { - rationale: object.rationale, - predictedTools: object.predictedTools, - expectedTools: expectedTools, - }, - }; - }; + }, + ); }