From 2dd3bbfaf6a9c408ed94b5b3006d9d30eba3cad3 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 04:38:26 +0000 Subject: [PATCH 1/6] fix(evals): Report blocking attendees and give delegation evals time for child research findMeetingTimes returned empty slots with no reason when the requester, who is required by default, was busy for the whole window. The agent then retried day by day and hit the 60-second idle limit. The result now names the required attendees who are never free in the window (blockedBy). The delegation case waits for three child agents' research and the parent's summary. In live runs one child Turn alone can take more than 60 seconds, so the case gets a 120-second idle budget through a new idleTimeoutMs call option. policies/evals.md names this exception. Refs #2413 Co-Authored-By: David Cramer --- packages/junior-evals/README.md | 2 + .../evals/agent/delegation.eval.ts | 38 ++++++++------ packages/junior-evals/src/fixture/agent.ts | 11 ++-- packages/junior-google/README.md | 5 +- .../src/tools/find-meeting-times.ts | 50 +++++++++++++++---- packages/junior-google/tests/calendar.test.ts | 45 ++++++++++++++++- policies/evals.md | 4 ++ 7 files changed, 122 insertions(+), 33 deletions(-) diff --git a/packages/junior-evals/README.md b/packages/junior-evals/README.md index d893e4fcd4..976c4b9d46 100644 --- a/packages/junior-evals/README.md +++ b/packages/junior-evals/README.md @@ -99,6 +99,8 @@ describe("Thread Continuity", () => { Conversation again after 15 seconds. The fixture starts a delayed delivery at once when nothing else runs. The 60 seconds start when the last delivery is due. + A call that waits for child agents can set `idleTimeoutMs`; see the + exceptions in `policies/evals.md`. - A failed call keeps its partial transcript and original error in the eval report. This includes calls that exceed the idle limit. - A channel `slackMention()` arrives as Slack sends it: an `app_mention` event diff --git a/packages/junior-evals/evals/agent/delegation.eval.ts b/packages/junior-evals/evals/agent/delegation.eval.ts index bd65fb4843..bfcd4eddf8 100644 --- a/packages/junior-evals/evals/agent/delegation.eval.ts +++ b/packages/junior-evals/evals/agent/delegation.eval.ts @@ -5,22 +5,32 @@ import { test } from "@junior-evals/fixture/test"; // The user never names spawnAgent here. These cases measure whether Junior // decides on its own when delegation pays for itself. +// The call waits for the child agents' research and the parent's summary. +// In live runs, one child Turn or the summary alone can take more than 60 +// seconds. Recorded runs finish in about 10 seconds. The test timeout leaves +// room for cleanup after the idle budget. +const CHILD_RESEARCH_IDLE_MS = 120_000; +const CHILD_RESEARCH_TEST_TIMEOUT_MS = 240_000; + describe("Implicit Delegation", () => { - test("when a request splits into independent research tracks, start a child agent for each track", async ({ - agent, - }) => { - // A small budget keeps each child short. The case measures the parent's - // choice to delegate, not the quality of the children's research. - const { run } = await agent({ limits: { maxToolCallsPerTurn: 6 } }); - const conversation = await run( - slackMention( - "I need to pick a database for a new service. Research how PostgreSQL, MySQL, and SQLite each handle adding a NOT NULL column with a default to a very large table: locking, rewrite cost, and replication impact. Go to each project's own docs, then give me the tradeoffs.", - ), - ); + test( + "when a request splits into independent research tracks, start a child agent for each track", + async ({ agent }) => { + // A small budget keeps each child short. The case measures the parent's + // choice to delegate, not the quality of the children's research. + const { run } = await agent({ limits: { maxToolCallsPerTurn: 6 } }); + const conversation = await run( + slackMention( + "I need to pick a database for a new service. Research how PostgreSQL, MySQL, and SQLite each handle adding a NOT NULL column with a default to a very large table: locking, rewrite cost, and replication impact. Go to each project's own docs, then give me the tradeoffs.", + ), + { idleTimeoutMs: CHILD_RESEARCH_IDLE_MS }, + ); - const spawned = completedToolCalls("spawnAgent", conversation); - expect(spawned.length).toBeGreaterThanOrEqual(2); - }); + const spawned = completedToolCalls("spawnAgent", conversation); + expect(spawned.length).toBeGreaterThanOrEqual(2); + }, + CHILD_RESEARCH_TEST_TIMEOUT_MS, + ); test("when a request needs only a few direct tool calls, do the work without a child agent", async ({ run, diff --git a/packages/junior-evals/src/fixture/agent.ts b/packages/junior-evals/src/fixture/agent.ts index 3f8a594522..fe2e3a6121 100644 --- a/packages/junior-evals/src/fixture/agent.ts +++ b/packages/junior-evals/src/fixture/agent.ts @@ -108,6 +108,8 @@ export interface CallOptions { progress: TurnProgress, actions: { send(input: Input): Promise }, ) => void | Promise; + /** Idle budget in place of 60 seconds. See `policies/evals.md`. */ + idleTimeoutMs?: number; } /** @@ -266,14 +268,13 @@ export async function createFixtureAgent( }); const startedAtMs = Date.now(); - const waitForIdle = async (): Promise => { + const waitForIdle = async (limitMs = IDLE_TIMEOUT_MS): Promise => { const startedAtMs = Date.now(); for (;;) { // The product delays some deliveries. For example, the worker checks // a busy Conversation again after 15 seconds. The budget starts when // the last delivery is due to start. - const deadline = - Math.max(startedAtMs, queue.latestStartAtMs()) + IDLE_TIMEOUT_MS; + const deadline = Math.max(startedAtMs, queue.latestStartAtMs()) + limitMs; const pending = [...queue.pending(), ...background]; if (pending.length === 0) { // Let work that a finished delivery started reach the queue. @@ -286,7 +287,7 @@ export async function createFixtureAgent( const remainingMs = deadline - Date.now(); if (remainingMs <= 0) { throw new Error( - `The agent was not idle within ${IDLE_TIMEOUT_MS / 1000} seconds`, + `The agent was not idle within ${limitMs / 1000} seconds`, ); } let timer: ReturnType | undefined; @@ -598,7 +599,7 @@ export async function createFixtureAgent( } try { await send(); - await waitForIdle(); + await waitForIdle(options.idleTimeoutMs); } catch (error) { // Keep the partial transcript before cleanup removes the test identities. // A timeout must show the work that was still running, not an empty run. diff --git a/packages/junior-google/README.md b/packages/junior-google/README.md index 1d88cb3d1e..a2f6a55705 100644 --- a/packages/junior-google/README.md +++ b/packages/junior-google/README.md @@ -16,7 +16,10 @@ Gmail and Drive writes are out of scope. reads event titles or details. Optional attendees do not block a slot. Slots where they are free come first, and each slot names the optional attendees who are busy or off hours. Slots are spread across days and - across each day, so the requester gets a real choice. + across each day, so the requester gets a real choice. When no slot fits, + `blockedBy` names the required attendees who are busy or off hours for the + whole window. The requester is required by default, so their own holds can + be the cause. - Every Calendar tool takes an optional `timeZone` for the times it reads and shows. Without it, the tool uses the requester's timezone from `ctx.users.resolveTimezone()`, so Junior does not have to ask for one. diff --git a/packages/junior-google/src/tools/find-meeting-times.ts b/packages/junior-google/src/tools/find-meeting-times.ts index 90b9496f82..9c4048781c 100644 --- a/packages/junior-google/src/tools/find-meeting-times.ts +++ b/packages/junior-google/src/tools/find-meeting-times.ts @@ -95,6 +95,12 @@ const outputSchema = pluginToolOutputSchema.extend({ }), ), slots: z.array(slotSchema), + blockedBy: z + .array(z.string()) + .optional() + .describe( + "Set only when no slot fits: required attendees who are busy or outside their working hours for the whole window.", + ), timeZone: z.string(), unavailable: z.array(z.object({ email: z.string(), reason: z.string() })), }); @@ -228,6 +234,9 @@ function spreadAcrossDays(candidates: Candidate[], max: number): Candidate[] { * Slots without such conflicts come first. Options are spread across the days * of `timeZone` and across each day. * + * When no slot fits, `blockedBy` names the required attendees who are never + * available in the window, so the caller can say who blocks the search. + * * Exported for tests. Busy intervals come from Google free/busy only, so * Junior never sees event titles or details. */ @@ -240,7 +249,10 @@ export function findFreeSlots(input: { timeZone: string; workdayEnd: string; workdayStart: string; -}): Array { +}): { + blockedBy: string[]; + slots: Array; +} { const durationMs = input.durationMinutes * 60_000; const stepMs = (input.durationMinutes < 30 ? 15 : 30) * 60_000; const dayStart = clockMinutes(input.workdayStart); @@ -273,11 +285,16 @@ export function findFreeSlots(input: { const required = input.attendees.filter((attendee) => attendee.required); const optional = input.attendees.filter((attendee) => !attendee.required); const candidates: Candidate[] = []; + const freeOnce = new Set(); let startMs = Math.ceil(input.timeMinMs / stepMs) * stepMs; while (startMs + durationMs <= input.timeMaxMs) { const endMs = startMs + durationMs; - if (required.every((attendee) => available(attendee, startMs, endMs))) { + const free = required.filter((attendee) => + available(attendee, startMs, endMs), + ); + for (const attendee of free) freeOnce.add(attendee.email); + if (free.length === required.length) { candidates.push({ date: partsIn(startMs, input.timeZone).date, endMs, @@ -303,13 +320,21 @@ export function findFreeSlots(input: { input.maxResults - picked.length, ), ); - return picked - .sort((a, b) => a.startMs - b.startMs) - .map(({ endMs, optionalConflicts, startMs }) => ({ - endMs, - optionalConflicts, - startMs, - })); + return { + // A found slot means every required attendee was free at least once. + blockedBy: candidates.length + ? [] + : required + .filter((attendee) => !freeOnce.has(attendee.email)) + .map((attendee) => attendee.email), + slots: picked + .sort((a, b) => a.startMs - b.startMs) + .map(({ endMs, optionalConflicts, startMs }) => ({ + endMs, + optionalConflicts, + startMs, + })), + }; } /** @@ -434,8 +459,10 @@ export function createFindMeetingTimesTool(ctx: GoogleToolContext) { }); }); - const slots = !slotAttendees.some((attendee) => attendee.required) - ? [] + const { blockedBy, slots } = !slotAttendees.some( + (attendee) => attendee.required, + ) + ? { blockedBy: [], slots: [] } : findFreeSlots({ attendees: slotAttendees, durationMinutes: input.durationMinutes, @@ -467,6 +494,7 @@ export function createFindMeetingTimesTool(ctx: GoogleToolContext) { : undefined), start: new Date(slot.startMs).toISOString(), })), + ...(blockedBy.length ? { blockedBy } : undefined), timeZone, unavailable, }; diff --git a/packages/junior-google/tests/calendar.test.ts b/packages/junior-google/tests/calendar.test.ts index 45e2e4d573..42b09a4864 100644 --- a/packages/junior-google/tests/calendar.test.ts +++ b/packages/junior-google/tests/calendar.test.ts @@ -30,7 +30,7 @@ const LA = "America/Los_Angeles"; describe("findFreeSlots", () => { it("spreads non-overlapping weekday slots across days inside local working hours", () => { // Friday 2026-10-09 through Monday 2026-10-12, Los Angeles (UTC-7). - const slots = findFreeSlots({ + const { slots } = findFreeSlots({ attendees: [ { // Busy Friday 09:00-12:00 local. @@ -67,7 +67,7 @@ describe("findFreeSlots", () => { it("keeps slots inside each attendee's own working hours", () => { // Monday 2026-10-12. With 08:00-17:00 hours, Los Angeles (UTC-7) and // London (UTC+1) share only 08:00-09:00 in Los Angeles. - const slots = findFreeSlots({ + const { slots } = findFreeSlots({ attendees: [ { busy: [], email: REQUESTER, required: true, timeZone: LA }, { @@ -101,6 +101,47 @@ describe("findFreeSlots", () => { }, ]); }); + + it("names the required attendee who is busy for the whole window when no slot fits", () => { + // Monday 2026-10-12, 12:00-17:00 in Los Angeles (UTC-7). The requester + // holds the whole afternoon; Bob is free from 13:30. + const { blockedBy, slots } = findFreeSlots({ + attendees: [ + { + busy: [ + { + startMs: Date.parse("2026-10-12T19:00:00Z"), + endMs: Date.parse("2026-10-13T00:00:00Z"), + }, + ], + email: REQUESTER, + required: true, + timeZone: LA, + }, + { + busy: [ + { + startMs: Date.parse("2026-10-12T19:00:00Z"), + endMs: Date.parse("2026-10-12T20:30:00Z"), + }, + ], + email: "bob@example.com", + required: true, + timeZone: LA, + }, + ], + durationMinutes: 90, + maxResults: 3, + timeMaxMs: Date.parse("2026-10-13T00:00:00Z"), + timeMinMs: Date.parse("2026-10-12T19:00:00Z"), + timeZone: LA, + workdayEnd: "17:00", + workdayStart: "12:00", + }); + + expect(slots).toEqual([]); + expect(blockedBy).toEqual([REQUESTER]); + }); }); describe("recurrenceRule", () => { diff --git a/policies/evals.md b/policies/evals.md index edb0470431..f9fb00b24c 100644 --- a/policies/evals.md +++ b/policies/evals.md @@ -81,3 +81,7 @@ Suite policy: value of a Junior-owned module. - Global setup may give the product the Sandbox snapshot ids of an earlier run, so that a run does not build the snapshots again. +- A case that measures delegation to child agents may give its call a longer + idle budget with `idleTimeoutMs`. The call waits for the research of the + child agents, and one child Turn can take more than 60 seconds. Name the + reason beside the value. From db253e9d1ec24abbb27c78c6dc3b378be67919fd Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 04:41:37 +0000 Subject: [PATCH 2/6] fix(evals): Assert the deployment watch, not the lookup step The deployment watch case required a github_getDeployment call before watchEvents. In live runs the agent often finds the deployment_source identifier through searchEventTypes and creates the exact correct watch. The user-visible outcome is the same, so the case now asserts the watch and the reply, not the step that produced the identifier. Refs #2413 Co-Authored-By: David Cramer --- .../junior-evals/evals/coding/subscriptions.eval.ts | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/packages/junior-evals/evals/coding/subscriptions.eval.ts b/packages/junior-evals/evals/coding/subscriptions.eval.ts index 1ab51dd402..0c5745df44 100644 --- a/packages/junior-evals/evals/coding/subscriptions.eval.ts +++ b/packages/junior-evals/evals/coding/subscriptions.eval.ts @@ -70,7 +70,7 @@ function watchedEvents(conversation: { toolCalls: ToolCall[] }) { } describe("Watches", () => { - test("looks up and subscribes to an exact deployment before GitHub creates it", async ({ + test("subscribes to an exact deployment before GitHub creates it", async ({ run, }) => { const commitSha = "c610b5d6a88c9da5d65627a1cdb3829b05c14f75"; @@ -93,15 +93,6 @@ describe("Watches", () => { }), ); - expect( - completedToolCalls("github_getDeployment", conversation).map( - (call) => call.input, - ), - ).toContainEqual({ - commitSha, - environment: "Production", - repo: "getsentry/junior-prod", - }); expect( completedToolCalls("watchEvents", conversation).map((call) => call.input), ).toContainEqual( From 295953dd0dd6da89e6923b868e865ad79e9703d5 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 05:01:48 +0000 Subject: [PATCH 3/6] fix(google): Drop unproven blockedBy result from findMeetingTimes The live run returned blockedBy correctly, but the recurring-slot case still timed out. No run shows that the field changes what the agent does, so it does not belong in this eval-fix PR. The recurring-slot case needs a separate change to how findMeetingTimes treats the requester's holds. Refs #2413 Co-Authored-By: David Cramer --- packages/junior-google/README.md | 5 +- .../src/tools/find-meeting-times.ts | 50 ++++--------------- packages/junior-google/tests/calendar.test.ts | 45 +---------------- 3 files changed, 14 insertions(+), 86 deletions(-) diff --git a/packages/junior-google/README.md b/packages/junior-google/README.md index a2f6a55705..1d88cb3d1e 100644 --- a/packages/junior-google/README.md +++ b/packages/junior-google/README.md @@ -16,10 +16,7 @@ Gmail and Drive writes are out of scope. reads event titles or details. Optional attendees do not block a slot. Slots where they are free come first, and each slot names the optional attendees who are busy or off hours. Slots are spread across days and - across each day, so the requester gets a real choice. When no slot fits, - `blockedBy` names the required attendees who are busy or off hours for the - whole window. The requester is required by default, so their own holds can - be the cause. + across each day, so the requester gets a real choice. - Every Calendar tool takes an optional `timeZone` for the times it reads and shows. Without it, the tool uses the requester's timezone from `ctx.users.resolveTimezone()`, so Junior does not have to ask for one. diff --git a/packages/junior-google/src/tools/find-meeting-times.ts b/packages/junior-google/src/tools/find-meeting-times.ts index 9c4048781c..90b9496f82 100644 --- a/packages/junior-google/src/tools/find-meeting-times.ts +++ b/packages/junior-google/src/tools/find-meeting-times.ts @@ -95,12 +95,6 @@ const outputSchema = pluginToolOutputSchema.extend({ }), ), slots: z.array(slotSchema), - blockedBy: z - .array(z.string()) - .optional() - .describe( - "Set only when no slot fits: required attendees who are busy or outside their working hours for the whole window.", - ), timeZone: z.string(), unavailable: z.array(z.object({ email: z.string(), reason: z.string() })), }); @@ -234,9 +228,6 @@ function spreadAcrossDays(candidates: Candidate[], max: number): Candidate[] { * Slots without such conflicts come first. Options are spread across the days * of `timeZone` and across each day. * - * When no slot fits, `blockedBy` names the required attendees who are never - * available in the window, so the caller can say who blocks the search. - * * Exported for tests. Busy intervals come from Google free/busy only, so * Junior never sees event titles or details. */ @@ -249,10 +240,7 @@ export function findFreeSlots(input: { timeZone: string; workdayEnd: string; workdayStart: string; -}): { - blockedBy: string[]; - slots: Array; -} { +}): Array { const durationMs = input.durationMinutes * 60_000; const stepMs = (input.durationMinutes < 30 ? 15 : 30) * 60_000; const dayStart = clockMinutes(input.workdayStart); @@ -285,16 +273,11 @@ export function findFreeSlots(input: { const required = input.attendees.filter((attendee) => attendee.required); const optional = input.attendees.filter((attendee) => !attendee.required); const candidates: Candidate[] = []; - const freeOnce = new Set(); let startMs = Math.ceil(input.timeMinMs / stepMs) * stepMs; while (startMs + durationMs <= input.timeMaxMs) { const endMs = startMs + durationMs; - const free = required.filter((attendee) => - available(attendee, startMs, endMs), - ); - for (const attendee of free) freeOnce.add(attendee.email); - if (free.length === required.length) { + if (required.every((attendee) => available(attendee, startMs, endMs))) { candidates.push({ date: partsIn(startMs, input.timeZone).date, endMs, @@ -320,21 +303,13 @@ export function findFreeSlots(input: { input.maxResults - picked.length, ), ); - return { - // A found slot means every required attendee was free at least once. - blockedBy: candidates.length - ? [] - : required - .filter((attendee) => !freeOnce.has(attendee.email)) - .map((attendee) => attendee.email), - slots: picked - .sort((a, b) => a.startMs - b.startMs) - .map(({ endMs, optionalConflicts, startMs }) => ({ - endMs, - optionalConflicts, - startMs, - })), - }; + return picked + .sort((a, b) => a.startMs - b.startMs) + .map(({ endMs, optionalConflicts, startMs }) => ({ + endMs, + optionalConflicts, + startMs, + })); } /** @@ -459,10 +434,8 @@ export function createFindMeetingTimesTool(ctx: GoogleToolContext) { }); }); - const { blockedBy, slots } = !slotAttendees.some( - (attendee) => attendee.required, - ) - ? { blockedBy: [], slots: [] } + const slots = !slotAttendees.some((attendee) => attendee.required) + ? [] : findFreeSlots({ attendees: slotAttendees, durationMinutes: input.durationMinutes, @@ -494,7 +467,6 @@ export function createFindMeetingTimesTool(ctx: GoogleToolContext) { : undefined), start: new Date(slot.startMs).toISOString(), })), - ...(blockedBy.length ? { blockedBy } : undefined), timeZone, unavailable, }; diff --git a/packages/junior-google/tests/calendar.test.ts b/packages/junior-google/tests/calendar.test.ts index 42b09a4864..45e2e4d573 100644 --- a/packages/junior-google/tests/calendar.test.ts +++ b/packages/junior-google/tests/calendar.test.ts @@ -30,7 +30,7 @@ const LA = "America/Los_Angeles"; describe("findFreeSlots", () => { it("spreads non-overlapping weekday slots across days inside local working hours", () => { // Friday 2026-10-09 through Monday 2026-10-12, Los Angeles (UTC-7). - const { slots } = findFreeSlots({ + const slots = findFreeSlots({ attendees: [ { // Busy Friday 09:00-12:00 local. @@ -67,7 +67,7 @@ describe("findFreeSlots", () => { it("keeps slots inside each attendee's own working hours", () => { // Monday 2026-10-12. With 08:00-17:00 hours, Los Angeles (UTC-7) and // London (UTC+1) share only 08:00-09:00 in Los Angeles. - const { slots } = findFreeSlots({ + const slots = findFreeSlots({ attendees: [ { busy: [], email: REQUESTER, required: true, timeZone: LA }, { @@ -101,47 +101,6 @@ describe("findFreeSlots", () => { }, ]); }); - - it("names the required attendee who is busy for the whole window when no slot fits", () => { - // Monday 2026-10-12, 12:00-17:00 in Los Angeles (UTC-7). The requester - // holds the whole afternoon; Bob is free from 13:30. - const { blockedBy, slots } = findFreeSlots({ - attendees: [ - { - busy: [ - { - startMs: Date.parse("2026-10-12T19:00:00Z"), - endMs: Date.parse("2026-10-13T00:00:00Z"), - }, - ], - email: REQUESTER, - required: true, - timeZone: LA, - }, - { - busy: [ - { - startMs: Date.parse("2026-10-12T19:00:00Z"), - endMs: Date.parse("2026-10-12T20:30:00Z"), - }, - ], - email: "bob@example.com", - required: true, - timeZone: LA, - }, - ], - durationMinutes: 90, - maxResults: 3, - timeMaxMs: Date.parse("2026-10-13T00:00:00Z"), - timeMinMs: Date.parse("2026-10-12T19:00:00Z"), - timeZone: LA, - workdayEnd: "17:00", - workdayStart: "12:00", - }); - - expect(slots).toEqual([]); - expect(blockedBy).toEqual([REQUESTER]); - }); }); describe("recurrenceRule", () => { From f3313c7b4b2ea3395797c569d4ddbb2fbfac2c3a Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 05:10:49 +0000 Subject: [PATCH 4/6] fix(evals): Stop child research after the delegation decision The delegation case measures the parent's choice to start child agents in its first Turn. It waited for the children's live research and the parent's summary, which take more than 60 seconds, so the earlier fix raised the idle budget to 120 seconds. Instead, the person says "stop" after the parent's first reply. The stop ends the child agents, so the call no longer waits for research that the case does not check. A child waits at its first tool request until the stop is sent, so replays make the same requests as the recording run. This removes the idleTimeoutMs call option and its policy exception. Refs #2413 --- packages/junior-evals/README.md | 2 - .../evals/agent/delegation.eval.ts | 75 ++++++++++++------- packages/junior-evals/src/fixture/agent.ts | 11 ++- policies/evals.md | 4 - 4 files changed, 55 insertions(+), 37 deletions(-) diff --git a/packages/junior-evals/README.md b/packages/junior-evals/README.md index 976c4b9d46..d893e4fcd4 100644 --- a/packages/junior-evals/README.md +++ b/packages/junior-evals/README.md @@ -99,8 +99,6 @@ describe("Thread Continuity", () => { Conversation again after 15 seconds. The fixture starts a delayed delivery at once when nothing else runs. The 60 seconds start when the last delivery is due. - A call that waits for child agents can set `idleTimeoutMs`; see the - exceptions in `policies/evals.md`. - A failed call keeps its partial transcript and original error in the eval report. This includes calls that exceed the idle limit. - A channel `slackMention()` arrives as Slack sends it: an `app_mention` event diff --git a/packages/junior-evals/evals/agent/delegation.eval.ts b/packages/junior-evals/evals/agent/delegation.eval.ts index bfcd4eddf8..750bbd940c 100644 --- a/packages/junior-evals/evals/agent/delegation.eval.ts +++ b/packages/junior-evals/evals/agent/delegation.eval.ts @@ -1,36 +1,61 @@ import { describe, expect } from "vitest"; -import { slackMention } from "@junior-evals/fixture/inputs"; +import { slackMention, slackThreadMessage } from "@junior-evals/fixture/inputs"; import { completedToolCalls } from "@junior-evals/fixture/results"; import { test } from "@junior-evals/fixture/test"; +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + // The user never names spawnAgent here. These cases measure whether Junior // decides on its own when delegation pays for itself. -// The call waits for the child agents' research and the parent's summary. -// In live runs, one child Turn or the summary alone can take more than 60 -// seconds. Recorded runs finish in about 10 seconds. The test timeout leaves -// room for cleanup after the idle budget. -const CHILD_RESEARCH_IDLE_MS = 120_000; -const CHILD_RESEARCH_TEST_TIMEOUT_MS = 240_000; - describe("Implicit Delegation", () => { - test( - "when a request splits into independent research tracks, start a child agent for each track", - async ({ agent }) => { - // A small budget keeps each child short. The case measures the parent's - // choice to delegate, not the quality of the children's research. - const { run } = await agent({ limits: { maxToolCallsPerTurn: 6 } }); - const conversation = await run( - slackMention( - "I need to pick a database for a new service. Research how PostgreSQL, MySQL, and SQLite each handle adding a NOT NULL column with a default to a very large table: locking, rewrite cost, and replication impact. Go to each project's own docs, then give me the tradeoffs.", - ), - { idleTimeoutMs: CHILD_RESEARCH_IDLE_MS }, - ); + test("when a request splits into independent research tracks, start a child agent for each track", async ({ + agent, + }) => { + // The parent decides to delegate in its first Turn. The research of the + // child agents and the parent's summary are not part of this case, and + // live they take more than 60 seconds. So the person says "stop" after + // the parent's first reply, and the stop ends the child agents too. + // A child agent waits at its first tool request until the stop is sent, + // so a replay and the recording run make the same requests. The wait has + // a bound, so a parent that calls a tool before it replies goes on. + // A small budget bounds the run when the parent does the research itself. + const { run } = await agent({ limits: { maxToolCallsPerTurn: 6 } }); + let spawnRequests = 0; + let stopSent!: () => void; + const stopSentOnce = new Promise((resolve) => { + stopSent = resolve; + }); + let stopping = false; + const conversation = await run( + slackMention( + "I need to pick a database for a new service. Research how PostgreSQL, MySQL, and SQLite each handle adding a NOT NULL column with a default to a very large table: locking, rewrite cost, and replication impact. Go to each project's own docs, then give me the tradeoffs.", + ), + { + onProgress: async (progress, { send }) => { + if (progress.type === "tool_request") { + if (progress.name === "spawnAgent") { + spawnRequests += 1; + } else if (spawnRequests > 0) { + await Promise.race([stopSentOnce, delay(15_000)]); + // The worker checks for a stop every 500 ms. + await delay(1_500); + } + } else if ( + progress.type === "reply" && + spawnRequests > 0 && + !stopping + ) { + stopping = true; + await send(slackThreadMessage("stop")); + stopSent(); + } + }, + }, + ); - const spawned = completedToolCalls("spawnAgent", conversation); - expect(spawned.length).toBeGreaterThanOrEqual(2); - }, - CHILD_RESEARCH_TEST_TIMEOUT_MS, - ); + const spawned = completedToolCalls("spawnAgent", conversation); + expect(spawned.length).toBeGreaterThanOrEqual(2); + }); test("when a request needs only a few direct tool calls, do the work without a child agent", async ({ run, diff --git a/packages/junior-evals/src/fixture/agent.ts b/packages/junior-evals/src/fixture/agent.ts index fe2e3a6121..3f8a594522 100644 --- a/packages/junior-evals/src/fixture/agent.ts +++ b/packages/junior-evals/src/fixture/agent.ts @@ -108,8 +108,6 @@ export interface CallOptions { progress: TurnProgress, actions: { send(input: Input): Promise }, ) => void | Promise; - /** Idle budget in place of 60 seconds. See `policies/evals.md`. */ - idleTimeoutMs?: number; } /** @@ -268,13 +266,14 @@ export async function createFixtureAgent( }); const startedAtMs = Date.now(); - const waitForIdle = async (limitMs = IDLE_TIMEOUT_MS): Promise => { + const waitForIdle = async (): Promise => { const startedAtMs = Date.now(); for (;;) { // The product delays some deliveries. For example, the worker checks // a busy Conversation again after 15 seconds. The budget starts when // the last delivery is due to start. - const deadline = Math.max(startedAtMs, queue.latestStartAtMs()) + limitMs; + const deadline = + Math.max(startedAtMs, queue.latestStartAtMs()) + IDLE_TIMEOUT_MS; const pending = [...queue.pending(), ...background]; if (pending.length === 0) { // Let work that a finished delivery started reach the queue. @@ -287,7 +286,7 @@ export async function createFixtureAgent( const remainingMs = deadline - Date.now(); if (remainingMs <= 0) { throw new Error( - `The agent was not idle within ${limitMs / 1000} seconds`, + `The agent was not idle within ${IDLE_TIMEOUT_MS / 1000} seconds`, ); } let timer: ReturnType | undefined; @@ -599,7 +598,7 @@ export async function createFixtureAgent( } try { await send(); - await waitForIdle(options.idleTimeoutMs); + await waitForIdle(); } catch (error) { // Keep the partial transcript before cleanup removes the test identities. // A timeout must show the work that was still running, not an empty run. diff --git a/policies/evals.md b/policies/evals.md index f9fb00b24c..edb0470431 100644 --- a/policies/evals.md +++ b/policies/evals.md @@ -81,7 +81,3 @@ Suite policy: value of a Junior-owned module. - Global setup may give the product the Sandbox snapshot ids of an earlier run, so that a run does not build the snapshots again. -- A case that measures delegation to child agents may give its call a longer - idle budget with `idleTimeoutMs`. The call waits for the research of the - child agents, and one child Turn can take more than 60 seconds. Name the - reason beside the value. From 0ed43ae338f37fe9297a48d2dad9f55dbe7b9656 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 05:19:17 +0000 Subject: [PATCH 5/6] fix(evals): Give delegation children fixed docs pages The delegation case measures the parent's choice to start child agents. The children read live PostgreSQL, MySQL, and SQLite docs pages, which made the run slow and depended on live pages that the case does not check. A new mockWebPages() fixture helper answers webFetch reads of some origins with fixed pages. The case gives the children three short docs pages and search results for them. The database docs origins are no longer reached through Roach, so they leave ALLOWED_ORIGINS. This replaces the earlier stop after the parent's first reply. There is no idle budget option and no policy exception. Refs #2413 Co-Authored-By: David Cramer --- packages/junior-evals/README.md | 4 + .../evals/agent/delegation.eval.ts | 91 +++++++++++-------- packages/junior-evals/src/fixture/web.ts | 23 ++++- packages/junior-evals/src/recording-rules.ts | 5 - 4 files changed, 80 insertions(+), 43 deletions(-) diff --git a/packages/junior-evals/README.md b/packages/junior-evals/README.md index d893e4fcd4..5b89aba811 100644 --- a/packages/junior-evals/README.md +++ b/packages/junior-evals/README.md @@ -324,6 +324,10 @@ cache keeps the file for the later runs of a pull request. the mock on or off, and no search reaches the real provider. A search finds nothing by default. `mockWebSearchResults()` from `src/fixture/web.ts` sets the results for one test. +- `mockWebPages()` from the same file gives fixed pages to `webFetch` for + some origins in one test. Use it when a case does not check the content + of a page, so a live page does not make the run slow. Other pages go to + Roach. The `bash` tool in the Vercel Sandbox does not use these mocks. ## Recordings diff --git a/packages/junior-evals/evals/agent/delegation.eval.ts b/packages/junior-evals/evals/agent/delegation.eval.ts index 750bbd940c..74b1db700c 100644 --- a/packages/junior-evals/evals/agent/delegation.eval.ts +++ b/packages/junior-evals/evals/agent/delegation.eval.ts @@ -1,9 +1,57 @@ import { describe, expect } from "vitest"; -import { slackMention, slackThreadMessage } from "@junior-evals/fixture/inputs"; +import { slackMention } from "@junior-evals/fixture/inputs"; import { completedToolCalls } from "@junior-evals/fixture/results"; import { test } from "@junior-evals/fixture/test"; +import { mockWebPages, mockWebSearchResults } from "@junior-evals/fixture/web"; -const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); +// Short docs pages answer at once. The case measures the parent's choice to +// delegate, and the children's research is not checked, so live docs pages +// only make the run slow. +const POSTGRES_DOCS = { + title: "PostgreSQL: ALTER TABLE", + url: "https://www.postgresql.org/docs/current/sql-altertable.html", + excerpt: + "ADD COLUMN with a non-volatile DEFAULT changes only the catalog. The table is not rewritten.", +}; +const MYSQL_DOCS = { + title: "MySQL 8.4: Online DDL Operations", + url: "https://dev.mysql.com/doc/refman/8.4/en/innodb-online-ddl-operations.html", + excerpt: + "ALGORITHM=INSTANT is the default for ADD COLUMN. Only metadata changes.", +}; +const SQLITE_DOCS = { + title: "SQLite: ALTER TABLE", + url: "https://www.sqlite.org/lang_altertable.html", + excerpt: + "ADD COLUMN changes only the schema text. Its time does not depend on the amount of data.", +}; + +function docsPage(title: string, paragraphs: string[]): string { + return `${title}

${title}

${paragraphs + .map((text) => `

${text}

`) + .join("")}`; +} + +const SQLITE_PAGE = docsPage(SQLITE_DOCS.title, [ + "ALTER TABLE ADD COLUMN changes only the schema text in sqlite_schema. Its time does not depend on the amount of data in the table.", + "A NOT NULL column needs a default that is not NULL. The default cannot be CURRENT_TIME, CURRENT_DATE, CURRENT_TIMESTAMP, or an expression in parentheses.", + "The statement takes the database write lock for a short time. SQLite has no built-in replication. Tools that copy WAL pages, such as Litestream, copy the small schema change.", +]); + +const DATABASE_DOCS_PAGES = { + "https://www.postgresql.org": docsPage(POSTGRES_DOCS.title, [ + "Since PostgreSQL 11, ADD COLUMN with a non-volatile DEFAULT stores the default in the catalog and does not rewrite the table, so it is fast on a very large table. A volatile default, such as clock_timestamp(), rewrites the whole table.", + "ALTER TABLE takes an ACCESS EXCLUSIVE lock for the short catalog change. The lock waits behind long transactions, and queries queue behind it. Set lock_timeout to limit the wait.", + "Streaming replication sends the small catalog change in the WAL. Logical replication does not replicate DDL: add the column on each subscriber first.", + ]), + "https://dev.mysql.com": docsPage(MYSQL_DOCS.title, [ + "In MySQL 8.0.29 and later, ADD COLUMN uses ALGORITHM=INSTANT by default. It changes only metadata, so it is fast on a very large table, and concurrent DML continues.", + "INSTANT does not apply to ROW_FORMAT=COMPRESSED tables, tables with a FULLTEXT index, or temporary tables, and a table allows at most 64 instant row versions. Then MySQL rebuilds the table with ALGORITHM=INPLACE. The statement takes a short exclusive metadata lock at its start and end.", + "The ALTER TABLE statement goes to replicas in the binary log. An instant change is also fast on a replica. A rebuild runs again on each replica and blocks the replication applier, so replicas lag.", + ]), + "https://sqlite.org": SQLITE_PAGE, + "https://www.sqlite.org": SQLITE_PAGE, +}; // The user never names spawnAgent here. These cases measure whether Junior // decides on its own when delegation pays for itself. @@ -11,46 +59,15 @@ describe("Implicit Delegation", () => { test("when a request splits into independent research tracks, start a child agent for each track", async ({ agent, }) => { - // The parent decides to delegate in its first Turn. The research of the - // child agents and the parent's summary are not part of this case, and - // live they take more than 60 seconds. So the person says "stop" after - // the parent's first reply, and the stop ends the child agents too. - // A child agent waits at its first tool request until the stop is sent, - // so a replay and the recording run make the same requests. The wait has - // a bound, so a parent that calls a tool before it replies goes on. - // A small budget bounds the run when the parent does the research itself. + // A small budget keeps each child short. The case measures the parent's + // choice to delegate, not the quality of the children's research. const { run } = await agent({ limits: { maxToolCallsPerTurn: 6 } }); - let spawnRequests = 0; - let stopSent!: () => void; - const stopSentOnce = new Promise((resolve) => { - stopSent = resolve; - }); - let stopping = false; + mockWebSearchResults([POSTGRES_DOCS, MYSQL_DOCS, SQLITE_DOCS]); + mockWebPages(DATABASE_DOCS_PAGES); const conversation = await run( slackMention( "I need to pick a database for a new service. Research how PostgreSQL, MySQL, and SQLite each handle adding a NOT NULL column with a default to a very large table: locking, rewrite cost, and replication impact. Go to each project's own docs, then give me the tradeoffs.", ), - { - onProgress: async (progress, { send }) => { - if (progress.type === "tool_request") { - if (progress.name === "spawnAgent") { - spawnRequests += 1; - } else if (spawnRequests > 0) { - await Promise.race([stopSentOnce, delay(15_000)]); - // The worker checks for a stop every 500 ms. - await delay(1_500); - } - } else if ( - progress.type === "reply" && - spawnRequests > 0 && - !stopping - ) { - stopping = true; - await send(slackThreadMessage("stop")); - stopSent(); - } - }, - }, ); const spawned = completedToolCalls("spawnAgent", conversation); diff --git a/packages/junior-evals/src/fixture/web.ts b/packages/junior-evals/src/fixture/web.ts index 349628daaf..7d9f0848ef 100644 --- a/packages/junior-evals/src/fixture/web.ts +++ b/packages/junior-evals/src/fixture/web.ts @@ -7,7 +7,8 @@ * `mockWebSearchResults()`. * * The pages that the `webFetch` tool reads go to Roach, which - * replays them. See `src/recording-rules.ts`. + * replays them. See `src/recording-rules.ts`. A test can answer some origins + * itself with `mockWebPages()`. */ import { randomUUID } from "node:crypto"; import { bypass, http, HttpResponse } from "msw"; @@ -15,12 +16,32 @@ import { onTestFinished } from "vitest"; import { USER_AGENT } from "@/chat/tools/web/constants"; import { mswServer } from "@junior-tests/msw/server"; +let mockedPages: Record = {}; + +/** + * Answer `webFetch` reads of each origin, such as `https://www.sqlite.org`, + * with its HTML page in this test. Every path of the origin gets the page. + * Other origins still go to Roach. + */ +export function mockWebPages(pages: Record): void { + mockedPages = pages; + onTestFinished(() => { + mockedPages = {}; + }); +} + /** Send the pages that `webFetch` reads to Roach. */ export function installWebReplay(): void { mswServer.use( http.get("*", ({ request }) => { // Only `webFetch` sends this user agent. if (request.headers.get("user-agent") !== USER_AGENT) return undefined; + const page = mockedPages[new URL(request.url).origin]; + if (page !== undefined) { + return new Response(page, { + headers: { "content-type": "text/html; charset=utf-8" }, + }); + } // `webFetch` uses `node:https`, which does not use the proxy. `fetch` // uses the proxy dispatcher of the test. `webFetch` follows redirects // itself, so each redirect is its own request. diff --git a/packages/junior-evals/src/recording-rules.ts b/packages/junior-evals/src/recording-rules.ts index e1cc579418..bcc05342b9 100644 --- a/packages/junior-evals/src/recording-rules.ts +++ b/packages/junior-evals/src/recording-rules.ts @@ -77,11 +77,6 @@ const ALLOWED_ORIGINS = [ "https://oidc.vercel.com", // Pages that `webFetch` evals read. "https://docs.slack.dev", - // Database docs that the child agents of the delegation eval read. - "https://dev.mysql.com", - "https://sqlite.org", - "https://www.postgresql.org", - "https://www.sqlite.org", ]; /** From dda1fa0d813a62559835906c8f6fbd81a3a7cd9a Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 15:49:57 +0000 Subject: [PATCH 6/6] fix(evals): Stop delegation children before they call the model The delegation case checks only the parent's choice to start child agents. The call still waited for the children's live research and the parent's summary, which take 45-126 seconds live. Fixed docs pages did not make the model turns shorter. The fixture now reports `child_agent` progress before it runs the work of a child agent, and the work waits for the handler. The case holds each child until the person says "stop" after the parent's reply. The stop ends the child agents before their first model call, so the call ends after the parent's first Turn and the replay makes the same requests as the recording run. This removes mockWebPages() and restores the database docs origins, which only the parent reads now when it does the research itself. Refs #2413 Co-Authored-By: David Cramer --- packages/junior-evals/README.md | 14 +-- .../evals/agent/delegation.eval.ts | 88 +++++++------------ packages/junior-evals/src/fixture/agent.ts | 16 ++-- packages/junior-evals/src/fixture/web.ts | 23 +---- packages/junior-evals/src/recording-rules.ts | 5 ++ 5 files changed, 57 insertions(+), 89 deletions(-) diff --git a/packages/junior-evals/README.md b/packages/junior-evals/README.md index 5b89aba811..0848f033d3 100644 --- a/packages/junior-evals/README.md +++ b/packages/junior-evals/README.md @@ -155,13 +155,17 @@ describe("Thread Continuity", () => { also accepts a recorded conversation from `src/fixture/recordings/`. Export one with `exportRecordedConversation()`. - `onProgress` reacts to what the turn does: `model_request`, - `tool_request`, `reply`, or `paused`. Its `send(input)` posts an input while - the turn waits, so the product decides whether it steers, waits, or stops - the turn. + `tool_request`, `reply`, `paused`, or `child_agent`. Its `send(input)` posts + an input while the turn waits, so the product decides whether it steers, + waits, or stops the turn. `paused` means that a turn stopped before it finished, for example at its deadline, and Junior queued the rest of it. The fixture sees this at the queue, which it replaces. The rest of the turn waits until the handler finishes, so a sent input arrives before the turn continues. + `child_agent` means that Junior is about to run the work of a child agent. + The fixture also sees this at the queue. The work waits until the handler + finishes. A case that checks only the parent's choice to delegate can stop + the child agents before they call the model. `sendDuringFirstModelRequest(inputs)` in `src/fixture/progress.ts` sends inputs while the first model request waits. - Insert functions in `src/fixture/insert.ts` write setup data through the @@ -324,10 +328,6 @@ cache keeps the file for the later runs of a pull request. the mock on or off, and no search reaches the real provider. A search finds nothing by default. `mockWebSearchResults()` from `src/fixture/web.ts` sets the results for one test. -- `mockWebPages()` from the same file gives fixed pages to `webFetch` for - some origins in one test. Use it when a case does not check the content - of a page, so a live page does not make the run slow. Other pages go to - Roach. The `bash` tool in the Vercel Sandbox does not use these mocks. ## Recordings diff --git a/packages/junior-evals/evals/agent/delegation.eval.ts b/packages/junior-evals/evals/agent/delegation.eval.ts index 74b1db700c..fa21587d23 100644 --- a/packages/junior-evals/evals/agent/delegation.eval.ts +++ b/packages/junior-evals/evals/agent/delegation.eval.ts @@ -1,57 +1,7 @@ import { describe, expect } from "vitest"; -import { slackMention } from "@junior-evals/fixture/inputs"; +import { slackMention, slackThreadMessage } from "@junior-evals/fixture/inputs"; import { completedToolCalls } from "@junior-evals/fixture/results"; import { test } from "@junior-evals/fixture/test"; -import { mockWebPages, mockWebSearchResults } from "@junior-evals/fixture/web"; - -// Short docs pages answer at once. The case measures the parent's choice to -// delegate, and the children's research is not checked, so live docs pages -// only make the run slow. -const POSTGRES_DOCS = { - title: "PostgreSQL: ALTER TABLE", - url: "https://www.postgresql.org/docs/current/sql-altertable.html", - excerpt: - "ADD COLUMN with a non-volatile DEFAULT changes only the catalog. The table is not rewritten.", -}; -const MYSQL_DOCS = { - title: "MySQL 8.4: Online DDL Operations", - url: "https://dev.mysql.com/doc/refman/8.4/en/innodb-online-ddl-operations.html", - excerpt: - "ALGORITHM=INSTANT is the default for ADD COLUMN. Only metadata changes.", -}; -const SQLITE_DOCS = { - title: "SQLite: ALTER TABLE", - url: "https://www.sqlite.org/lang_altertable.html", - excerpt: - "ADD COLUMN changes only the schema text. Its time does not depend on the amount of data.", -}; - -function docsPage(title: string, paragraphs: string[]): string { - return `${title}

${title}

${paragraphs - .map((text) => `

${text}

`) - .join("")}`; -} - -const SQLITE_PAGE = docsPage(SQLITE_DOCS.title, [ - "ALTER TABLE ADD COLUMN changes only the schema text in sqlite_schema. Its time does not depend on the amount of data in the table.", - "A NOT NULL column needs a default that is not NULL. The default cannot be CURRENT_TIME, CURRENT_DATE, CURRENT_TIMESTAMP, or an expression in parentheses.", - "The statement takes the database write lock for a short time. SQLite has no built-in replication. Tools that copy WAL pages, such as Litestream, copy the small schema change.", -]); - -const DATABASE_DOCS_PAGES = { - "https://www.postgresql.org": docsPage(POSTGRES_DOCS.title, [ - "Since PostgreSQL 11, ADD COLUMN with a non-volatile DEFAULT stores the default in the catalog and does not rewrite the table, so it is fast on a very large table. A volatile default, such as clock_timestamp(), rewrites the whole table.", - "ALTER TABLE takes an ACCESS EXCLUSIVE lock for the short catalog change. The lock waits behind long transactions, and queries queue behind it. Set lock_timeout to limit the wait.", - "Streaming replication sends the small catalog change in the WAL. Logical replication does not replicate DDL: add the column on each subscriber first.", - ]), - "https://dev.mysql.com": docsPage(MYSQL_DOCS.title, [ - "In MySQL 8.0.29 and later, ADD COLUMN uses ALGORITHM=INSTANT by default. It changes only metadata, so it is fast on a very large table, and concurrent DML continues.", - "INSTANT does not apply to ROW_FORMAT=COMPRESSED tables, tables with a FULLTEXT index, or temporary tables, and a table allows at most 64 instant row versions. Then MySQL rebuilds the table with ALGORITHM=INPLACE. The statement takes a short exclusive metadata lock at its start and end.", - "The ALTER TABLE statement goes to replicas in the binary log. An instant change is also fast on a replica. A rebuild runs again on each replica and blocks the replication applier, so replicas lag.", - ]), - "https://sqlite.org": SQLITE_PAGE, - "https://www.sqlite.org": SQLITE_PAGE, -}; // The user never names spawnAgent here. These cases measure whether Junior // decides on its own when delegation pays for itself. @@ -59,16 +9,44 @@ describe("Implicit Delegation", () => { test("when a request splits into independent research tracks, start a child agent for each track", async ({ agent, }) => { - // A small budget keeps each child short. The case measures the parent's - // choice to delegate, not the quality of the children's research. + // The parent makes its choice in its first Turn. The research of the + // child agents is not part of this case. So each child waits until the + // person reads the parent's reply and says "stop", and the stop ends the + // child agents before they call the model. The wait has a bound, so the + // call does not hang when the parent never replies. + // A small budget bounds the run when the parent does the research itself. const { run } = await agent({ limits: { maxToolCallsPerTurn: 6 } }); - mockWebSearchResults([POSTGRES_DOCS, MYSQL_DOCS, SQLITE_DOCS]); - mockWebPages(DATABASE_DOCS_PAGES); + let spawnRequested = false; + let stopping: Promise | undefined; + let releaseChildren!: () => void; + const childrenReleased = new Promise((resolve) => { + releaseChildren = resolve; + }); + let bound: ReturnType | undefined; const conversation = await run( slackMention( "I need to pick a database for a new service. Research how PostgreSQL, MySQL, and SQLite each handle adding a NOT NULL column with a default to a very large table: locking, rewrite cost, and replication impact. Go to each project's own docs, then give me the tradeoffs.", ), + { + onProgress: async (progress, { send }) => { + if (progress.type === "tool_request") { + if (progress.name === "spawnAgent") spawnRequested = true; + } else if (progress.type === "child_agent") { + bound ??= setTimeout(releaseChildren, 30_000); + await childrenReleased; + } else if (progress.type === "reply" && spawnRequested && !stopping) { + // Slack posts the reply only after this handler returns, so do + // not wait for the stop here. The stop is recorded when `send` + // returns. + stopping = send(slackThreadMessage("stop")).finally( + releaseChildren, + ); + } + }, + }, ); + clearTimeout(bound); + await stopping; const spawned = completedToolCalls("spawnAgent", conversation); expect(spawned.length).toBeGreaterThanOrEqual(2); diff --git a/packages/junior-evals/src/fixture/agent.ts b/packages/junior-evals/src/fixture/agent.ts index 9b6e7dcf60..0d347d31ca 100644 --- a/packages/junior-evals/src/fixture/agent.ts +++ b/packages/junior-evals/src/fixture/agent.ts @@ -103,7 +103,12 @@ export type TurnProgress = * Junior queued the rest of it. The rest waits until the handler finishes, * so a sent input arrives before the turn continues. */ - | { type: "paused" }; + | { type: "paused" } + /** + * Junior is about to run the work of a child agent that the Conversation + * started. The work waits until the handler finishes. + */ + | { type: "child_agent" }; export interface CallOptions { /** Earlier turns as items, or a recorded conversation. */ @@ -581,10 +586,11 @@ export async function createFixtureAgent( }; gateway.setProgressHook(report); queue.setRequeueHook(async (conversationId) => { - if ( - typeof target === "function" || - conversationId !== target.conversationId - ) { + if (typeof target === "function") return; + if (conversationId !== target.conversationId) { + // During a call, the worker queues work for another Conversation + // only when it runs a child agent. + await report({ type: "child_agent" }); return; } // The worker also queues work for a message that waits for its own diff --git a/packages/junior-evals/src/fixture/web.ts b/packages/junior-evals/src/fixture/web.ts index 7d9f0848ef..349628daaf 100644 --- a/packages/junior-evals/src/fixture/web.ts +++ b/packages/junior-evals/src/fixture/web.ts @@ -7,8 +7,7 @@ * `mockWebSearchResults()`. * * The pages that the `webFetch` tool reads go to Roach, which - * replays them. See `src/recording-rules.ts`. A test can answer some origins - * itself with `mockWebPages()`. + * replays them. See `src/recording-rules.ts`. */ import { randomUUID } from "node:crypto"; import { bypass, http, HttpResponse } from "msw"; @@ -16,32 +15,12 @@ import { onTestFinished } from "vitest"; import { USER_AGENT } from "@/chat/tools/web/constants"; import { mswServer } from "@junior-tests/msw/server"; -let mockedPages: Record = {}; - -/** - * Answer `webFetch` reads of each origin, such as `https://www.sqlite.org`, - * with its HTML page in this test. Every path of the origin gets the page. - * Other origins still go to Roach. - */ -export function mockWebPages(pages: Record): void { - mockedPages = pages; - onTestFinished(() => { - mockedPages = {}; - }); -} - /** Send the pages that `webFetch` reads to Roach. */ export function installWebReplay(): void { mswServer.use( http.get("*", ({ request }) => { // Only `webFetch` sends this user agent. if (request.headers.get("user-agent") !== USER_AGENT) return undefined; - const page = mockedPages[new URL(request.url).origin]; - if (page !== undefined) { - return new Response(page, { - headers: { "content-type": "text/html; charset=utf-8" }, - }); - } // `webFetch` uses `node:https`, which does not use the proxy. `fetch` // uses the proxy dispatcher of the test. `webFetch` follows redirects // itself, so each redirect is its own request. diff --git a/packages/junior-evals/src/recording-rules.ts b/packages/junior-evals/src/recording-rules.ts index bcc05342b9..e1cc579418 100644 --- a/packages/junior-evals/src/recording-rules.ts +++ b/packages/junior-evals/src/recording-rules.ts @@ -77,6 +77,11 @@ const ALLOWED_ORIGINS = [ "https://oidc.vercel.com", // Pages that `webFetch` evals read. "https://docs.slack.dev", + // Database docs that the child agents of the delegation eval read. + "https://dev.mysql.com", + "https://sqlite.org", + "https://www.postgresql.org", + "https://www.sqlite.org", ]; /**