sentry-junior · Nov 5, 2026
diff --git a/docs/contributing/adding-tools.md b/docs/contributing/adding-tools.md
@@ -296,18 +296,22 @@ See [api-patterns.md](api-patterns.md#mock-patterns) for validation examples.
 **⚠️ Each eval costs time and API credits. Only test core functionality!**
 
 ```typescript
-describeEval("your-tool", {
-  data: async () => [
-    {
-      input: `Primary use case in ${FIXTURES.organizationSlug}`,
-      expected: "Expected response",
-    },
-    // Maximum 2-3 scenarios!
-  ],
-  task: TaskRunner(),
-  scorers: [Factuality()],
-  threshold: 0.6,
-});
+describeEval(
+  "your-tool",
+  {
+    harness: createTaskHarness("your-tool", NoOpTaskRunner()),
+    judges: [ToolPredictionJudge()],
+    judgeThreshold: 0.6,
+  },
+  (it) =>
+    defineEvalCases(it, [
+      {
+        input: `Primary use case in ${FIXTURES.organizationSlug}`,
+        expectedTools: [{ name: "your_tool", arguments: {} }],
+      },
+      // Maximum 2-3 scenarios!
+    ]),
+);
 ```
 
 ## Testing Workflow

diff --git a/docs/testing/overview.md b/docs/testing/overview.md
@@ -275,23 +275,45 @@ expect(result.timestamp).toMatchInlineSnapshot(); // ❌
 ### Eval Test Structure
 
 ```typescript
-import { describeEval } from "vitest-evals/legacy";
-import { TaskRunner, Factuality } from "./utils";
-
-describeEval("tool-name", {
-  data: async () => [
-    {
-      input: "Natural language request",
-      expected: "Expected response content",
-    },
-  ],
-  task: TaskRunner(), // Uses AI to call tools
-  scorers: [Factuality()], // Validates output
-  threshold: 0.6,
-  timeout: 30000,
-});
+import { describeEval } from "vitest-evals";
+import {
+  createTaskHarness,
+  defineEvalCases,
+  FIXTURES,
+  NoOpTaskRunner,
+  ToolPredictionJudge,
+  type EvalCase,
+} from "./utils";
+
+const cases: EvalCase[] = [
+  {
+    input: `Natural language request in ${FIXTURES.organizationSlug}`,
+    expectedTools: [
+      {
+        name: "tool_name",
+        arguments: { organizationSlug: FIXTURES.organizationSlug },
+      },
+    ],
+  },
+];
+
+describeEval(
+  "tool-name",
+  {
+    // NoOpTaskRunner skips tool execution; the judge predicts the tool calls
+    harness: createTaskHarness("tool-name", NoOpTaskRunner()),
+    judges: [ToolPredictionJudge()],
+    judgeThreshold: 0.6,
+  },
+  (it) => defineEvalCases(it, cases, { timeout: 30000 }),
+);
 ```
 
+Each case is the harness input. The task receives only `input`; judges read
+`expectedTools` and `expected` from the case. Embedded-agent evals pass their
+own task to `createTaskHarness` and use `ToolCallJudge` and
+`StructuredOutputJudge`.
+
 ### Running Evals
 
 ```bash

diff --git a/packages/mcp-server-evals/src/evals/autofix.eval.ts b/packages/mcp-server-evals/src/evals/autofix.eval.ts
@@ -1,35 +1,44 @@
-import { describeEval } from "vitest-evals/legacy";
-import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils";
+import { describeEval } from "vitest-evals";
+import {
+  createTaskHarness,
+  defineEvalCases,
+  FIXTURES,
+  NoOpTaskRunner,
+  ToolPredictionJudge,
+  type EvalCase,
+} from "./utils";
 
-describeEval("begin-issue-fix", {
-  data: async () => {
-    return [
+const cases: EvalCase[] = [
+  {
+    input: `Whats the status on root causing this issue in Sentry?\n${FIXTURES.testIssueUrl}`,
+    expectedTools: [
       {
-        input: `Whats the status on root causing this issue in Sentry?\n${FIXTURES.testIssueUrl}`,
-        expectedTools: [
-          {
-            name: "analyze_issue_with_seer",
-            arguments: {
-              issueUrl: FIXTURES.testIssueUrl,
-            },
-          },
-        ],
+        name: "analyze_issue_with_seer",
+        arguments: {
+          issueUrl: FIXTURES.testIssueUrl,
+        },
       },
+    ],
+  },
+  {
+    input: `Can you root cause this issue and retrieve the analysis?\n${FIXTURES.testIssueUrl}`,
+    expectedTools: [
       {
-        input: `Can you root cause this issue and retrieve the analysis?\n${FIXTURES.testIssueUrl}`,
-        expectedTools: [
-          {
-            name: "analyze_issue_with_seer",
-            arguments: {
-              issueUrl: FIXTURES.testIssueUrl,
-            },
-          },
-        ],
+        name: "analyze_issue_with_seer",
+        arguments: {
+          issueUrl: FIXTURES.testIssueUrl,
+        },
       },
-    ];
+    ],
+  },
+];
+
+describeEval(
+  "begin-issue-fix",
+  {
+    harness: createTaskHarness("begin-issue-fix", NoOpTaskRunner()),
+    judges: [ToolPredictionJudge()],
+    judgeThreshold: 0.6,
   },
-  task: NoOpTaskRunner(),
-  scorers: [ToolPredictionScorer()],
-  threshold: 0.6,
-  timeout: 30000,
-});
+  (it) => defineEvalCases(it, cases, { timeout: 30000 }),
+);
diff --git a/packages/mcp-server-evals/src/evals/create-dsn.eval.ts b/packages/mcp-server-evals/src/evals/create-dsn.eval.ts
@@ -1,26 +1,35 @@
-import { describeEval } from "vitest-evals/legacy";
-import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils";
+import { describeEval } from "vitest-evals";
+import {
+  createTaskHarness,
+  defineEvalCases,
+  FIXTURES,
+  NoOpTaskRunner,
+  ToolPredictionJudge,
+  type EvalCase,
+} from "./utils";
 
-describeEval("create-dsn", {
-  data: async () => {
-    return [
+const cases: EvalCase[] = [
+  {
+    input: `Create a new DSN named "Production" for '${FIXTURES.organizationSlug}/${FIXTURES.projectSlug}'`,
+    expectedTools: [
       {
-        input: `Create a new DSN named "Production" for '${FIXTURES.organizationSlug}/${FIXTURES.projectSlug}'`,
-        expectedTools: [
-          {
-            name: "create_dsn",
-            arguments: {
-              organizationSlug: FIXTURES.organizationSlug,
-              projectSlug: FIXTURES.projectSlug,
-              name: "Production",
-            },
-          },
-        ],
+        name: "create_dsn",
+        arguments: {
+          organizationSlug: FIXTURES.organizationSlug,
+          projectSlug: FIXTURES.projectSlug,
+          name: "Production",
+        },
       },
-    ];
+    ],
   },
-  task: NoOpTaskRunner(),
-  scorers: [ToolPredictionScorer()],
-  threshold: 0.6,
-  timeout: 30000,
-});
+];
+
+describeEval(
+  "create-dsn",
+  {
+    harness: createTaskHarness("create-dsn", NoOpTaskRunner()),
+    judges: [ToolPredictionJudge()],
+    judgeThreshold: 0.6,
+  },
+  (it) => defineEvalCases(it, cases, { timeout: 30000 }),
+);
diff --git a/packages/mcp-server-evals/src/evals/create-project.eval.ts b/packages/mcp-server-evals/src/evals/create-project.eval.ts
@@ -1,38 +1,47 @@
-import { describeEval } from "vitest-evals/legacy";
-import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils";
+import { describeEval } from "vitest-evals";
+import {
+  createTaskHarness,
+  defineEvalCases,
+  FIXTURES,
+  NoOpTaskRunner,
+  ToolPredictionJudge,
+  type EvalCase,
+} from "./utils";
 
-describeEval("create-project", {
-  data: async () => {
-    return [
+const cases: EvalCase[] = [
+  {
+    input: `Create a new project in Sentry for '${FIXTURES.organizationSlug}' called '${FIXTURES.projectSlug}' with the '${FIXTURES.teamSlug}' team. Output **only** the project slug and the SENTRY_DSN in the format of:\n<PROJECT_SLUG>\n<SENTRY_DSN>`,
+    expectedTools: [
       {
-        input: `Create a new project in Sentry for '${FIXTURES.organizationSlug}' called '${FIXTURES.projectSlug}' with the '${FIXTURES.teamSlug}' team. Output **only** the project slug and the SENTRY_DSN in the format of:\n<PROJECT_SLUG>\n<SENTRY_DSN>`,
-        expectedTools: [
-          {
-            name: "find_organizations",
-            arguments: {},
-          },
-          {
-            name: "find_teams",
-            arguments: {
-              organizationSlug: FIXTURES.organizationSlug,
-              regionUrl: "https://us.sentry.io",
-            },
-          },
-          {
-            name: "create_project",
-            arguments: {
-              organizationSlug: FIXTURES.organizationSlug,
-              regionUrl: "https://us.sentry.io",
-              teamSlug: FIXTURES.teamSlug,
-              name: FIXTURES.projectSlug,
-            },
-          },
-        ],
+        name: "find_organizations",
+        arguments: {},
       },
-    ];
+      {
+        name: "find_teams",
+        arguments: {
+          organizationSlug: FIXTURES.organizationSlug,
+          regionUrl: "https://us.sentry.io",
+        },
+      },
+      {
+        name: "create_project",
+        arguments: {
+          organizationSlug: FIXTURES.organizationSlug,
+          regionUrl: "https://us.sentry.io",
+          teamSlug: FIXTURES.teamSlug,
+          name: FIXTURES.projectSlug,
+        },
+      },
+    ],
+  },
+];
+
+describeEval(
+  "create-project",
+  {
+    harness: createTaskHarness("create-project", NoOpTaskRunner()),
+    judges: [ToolPredictionJudge()],
+    judgeThreshold: 0.6,
   },
-  task: NoOpTaskRunner(),
-  scorers: [ToolPredictionScorer()],
-  threshold: 0.6,
-  timeout: 30000,
-});
+  (it) => defineEvalCases(it, cases, { timeout: 30000 }),
+);
diff --git a/packages/mcp-server-evals/src/evals/create-team.eval.ts b/packages/mcp-server-evals/src/evals/create-team.eval.ts
@@ -1,30 +1,39 @@
-import { describeEval } from "vitest-evals/legacy";
-import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils";
+import { describeEval } from "vitest-evals";
+import {
+  createTaskHarness,
+  defineEvalCases,
+  FIXTURES,
+  NoOpTaskRunner,
+  ToolPredictionJudge,
+  type EvalCase,
+} from "./utils";
 
-describeEval("create-team", {
-  data: async () => {
-    return [
+const cases: EvalCase[] = [
+  {
+    input: `Create a new team in Sentry for '${FIXTURES.organizationSlug}' called 'the-goats' response with **only** the team slug and no other text.`,
+    expectedTools: [
       {
-        input: `Create a new team in Sentry for '${FIXTURES.organizationSlug}' called 'the-goats' response with **only** the team slug and no other text.`,
-        expectedTools: [
-          {
-            name: "find_organizations",
-            arguments: {},
-          },
-          {
-            name: "create_team",
-            arguments: {
-              organizationSlug: FIXTURES.organizationSlug,
-              name: "the-goats",
-              regionUrl: "https://us.sentry.io",
-            },
-          },
-        ],
+        name: "find_organizations",
+        arguments: {},
       },
-    ];
+      {
+        name: "create_team",
+        arguments: {
+          organizationSlug: FIXTURES.organizationSlug,
+          name: "the-goats",
+          regionUrl: "https://us.sentry.io",
+        },
+      },
+    ],
+  },
+];
+
+describeEval(
+  "create-team",
+  {
+    harness: createTaskHarness("create-team", NoOpTaskRunner()),
+    judges: [ToolPredictionJudge()],
+    judgeThreshold: 0.6,
   },
-  task: NoOpTaskRunner(),
-  scorers: [ToolPredictionScorer()],
-  threshold: 0.6,
-  timeout: 30000,
-});
+  (it) => defineEvalCases(it, cases, { timeout: 30000 }),
+);