fix(evals): Give delegation children fixed docs pages and assert the deployment watch

packages/junior-evals/README.md

@@ -324,6 +324,10 @@ cache keeps the file for the later runs of a pull request.
   the mock on or off, and no search reaches the real provider. A search finds
   nothing by default. `mockWebSearchResults()` from `src/fixture/web.ts` sets
   the results for one test.
+- `mockWebPages()` from the same file gives fixed pages to `webFetch` for
+  some origins in one test. Use it when a case does not check the content
+  of a page, so a live page does not make the run slow. Other pages go to
+  Roach. The `bash` tool in the Vercel Sandbox does not use these mocks.
 
 ## Recordings
 

packages/junior-evals/evals/agent/delegation.eval.ts

@@ -2,6 +2,56 @@ import { describe, expect } from "vitest";
 import { slackMention } from "@junior-evals/fixture/inputs";
 import { completedToolCalls } from "@junior-evals/fixture/results";
 import { test } from "@junior-evals/fixture/test";
+import { mockWebPages, mockWebSearchResults } from "@junior-evals/fixture/web";
+
+// Short docs pages answer at once. The case measures the parent's choice to
+// delegate, and the children's research is not checked, so live docs pages
+// only make the run slow.
+const POSTGRES_DOCS = {
+  title: "PostgreSQL: ALTER TABLE",
+  url: "https://www.postgresql.org/docs/current/sql-altertable.html",
+  excerpt:
+    "ADD COLUMN with a non-volatile DEFAULT changes only the catalog. The table is not rewritten.",
+};
+const MYSQL_DOCS = {
+  title: "MySQL 8.4: Online DDL Operations",
+  url: "https://dev.mysql.com/doc/refman/8.4/en/innodb-online-ddl-operations.html",
+  excerpt:
+    "ALGORITHM=INSTANT is the default for ADD COLUMN. Only metadata changes.",
+};
+const SQLITE_DOCS = {
+  title: "SQLite: ALTER TABLE",
+  url: "https://www.sqlite.org/lang_altertable.html",
+  excerpt:
+    "ADD COLUMN changes only the schema text. Its time does not depend on the amount of data.",
+};
+
+function docsPage(title: string, paragraphs: string[]): string {
+  return `<html><head><title>${title}</title></head><body><h1>${title}</h1>${paragraphs
+    .map((text) => `<p>${text}</p>`)
+    .join("")}</body></html>`;
+}
+
+const SQLITE_PAGE = docsPage(SQLITE_DOCS.title, [
+  "ALTER TABLE ADD COLUMN changes only the schema text in sqlite_schema. Its time does not depend on the amount of data in the table.",
+  "A NOT NULL column needs a default that is not NULL. The default cannot be CURRENT_TIME, CURRENT_DATE, CURRENT_TIMESTAMP, or an expression in parentheses.",
+  "The statement takes the database write lock for a short time. SQLite has no built-in replication. Tools that copy WAL pages, such as Litestream, copy the small schema change.",
+]);
+
+const DATABASE_DOCS_PAGES = {
+  "https://www.postgresql.org": docsPage(POSTGRES_DOCS.title, [
+    "Since PostgreSQL 11, ADD COLUMN with a non-volatile DEFAULT stores the default in the catalog and does not rewrite the table, so it is fast on a very large table. A volatile default, such as clock_timestamp(), rewrites the whole table.",
+    "ALTER TABLE takes an ACCESS EXCLUSIVE lock for the short catalog change. The lock waits behind long transactions, and queries queue behind it. Set lock_timeout to limit the wait.",
+    "Streaming replication sends the small catalog change in the WAL. Logical replication does not replicate DDL: add the column on each subscriber first.",
+  ]),
+  "https://dev.mysql.com": docsPage(MYSQL_DOCS.title, [
+    "In MySQL 8.0.29 and later, ADD COLUMN uses ALGORITHM=INSTANT by default. It changes only metadata, so it is fast on a very large table, and concurrent DML continues.",
+    "INSTANT does not apply to ROW_FORMAT=COMPRESSED tables, tables with a FULLTEXT index, or temporary tables, and a table allows at most 64 instant row versions. Then MySQL rebuilds the table with ALGORITHM=INPLACE. The statement takes a short exclusive metadata lock at its start and end.",
+    "The ALTER TABLE statement goes to replicas in the binary log. An instant change is also fast on a replica. A rebuild runs again on each replica and blocks the replication applier, so replicas lag.",
+  ]),
+  "https://sqlite.org": SQLITE_PAGE,
+  "https://www.sqlite.org": SQLITE_PAGE,
+};
 
 // The user never names spawnAgent here. These cases measure whether Junior
 // decides on its own when delegation pays for itself.
@@ -12,6 +62,8 @@ describe("Implicit Delegation", () => {
     // A small budget keeps each child short. The case measures the parent's
     // choice to delegate, not the quality of the children's research.
     const { run } = await agent({ limits: { maxToolCallsPerTurn: 6 } });
+    mockWebSearchResults([POSTGRES_DOCS, MYSQL_DOCS, SQLITE_DOCS]);
+    mockWebPages(DATABASE_DOCS_PAGES);
     const conversation = await run(
       slackMention(
         "I need to pick a database for a new service. Research how PostgreSQL, MySQL, and SQLite each handle adding a NOT NULL column with a default to a very large table: locking, rewrite cost, and replication impact. Go to each project's own docs, then give me the tradeoffs.",

packages/junior-evals/evals/coding/subscriptions.eval.ts

@@ -70,7 +70,7 @@ function watchedEvents(conversation: { toolCalls: ToolCall[] }) {
 }
 
 describe("Watches", () => {
-  test("looks up and subscribes to an exact deployment before GitHub creates it", async ({
+  test("subscribes to an exact deployment before GitHub creates it", async ({
     run,
   }) => {
     const commitSha = "c610b5d6a88c9da5d65627a1cdb3829b05c14f75";
@@ -93,15 +93,6 @@ describe("Watches", () => {
       }),
     );
 
-    expect(
-      completedToolCalls("github_getDeployment", conversation).map(
-        (call) => call.input,
-      ),
-    ).toContainEqual({
-      commitSha,
-      environment: "Production",
-      repo: "getsentry/junior-prod",
-    });
     expect(
       completedToolCalls("watchEvents", conversation).map((call) => call.input),
     ).toContainEqual(

packages/junior-evals/src/fixture/web.ts

@@ -7,20 +7,41 @@
  * `mockWebSearchResults()`.
  *
  * The pages that the `webFetch` tool reads go to Roach, which
- * replays them. See `src/recording-rules.ts`.
+ * replays them. See `src/recording-rules.ts`. A test can answer some origins
+ * itself with `mockWebPages()`.
  */
 import { randomUUID } from "node:crypto";
 import { bypass, http, HttpResponse } from "msw";
 import { onTestFinished } from "vitest";
 import { USER_AGENT } from "@/chat/tools/web/constants";
 import { mswServer } from "@junior-tests/msw/server";
 
+let mockedPages: Record<string, string> = {};
+
+/**
+ * Answer `webFetch` reads of each origin, such as `https://www.sqlite.org`,
+ * with its HTML page in this test. Every path of the origin gets the page.
+ * Other origins still go to Roach.
+ */
+export function mockWebPages(pages: Record<string, string>): void {
+  mockedPages = pages;
+  onTestFinished(() => {
+    mockedPages = {};
+  });
+}
+
 /** Send the pages that `webFetch` reads to Roach. */
 export function installWebReplay(): void {
   mswServer.use(
     http.get("*", ({ request }) => {
       // Only `webFetch` sends this user agent.
       if (request.headers.get("user-agent") !== USER_AGENT) return undefined;
+      const page = mockedPages[new URL(request.url).origin];
+      if (page !== undefined) {
+        return new Response(page, {
+          headers: { "content-type": "text/html; charset=utf-8" },
+        });
+      }
       // `webFetch` uses `node:https`, which does not use the proxy. `fetch`
       // uses the proxy dispatcher of the test. `webFetch` follows redirects
       // itself, so each redirect is its own request.

packages/junior-evals/src/recording-rules.ts

@@ -77,11 +77,6 @@ const ALLOWED_ORIGINS = [
   "https://oidc.vercel.com",
   // Pages that `webFetch` evals read.
   "https://docs.slack.dev",
-  // Database docs that the child agents of the delegation eval read.
-  "https://dev.mysql.com",
-  "https://sqlite.org",
-  "https://www.postgresql.org",
-  "https://www.sqlite.org",
 ];
 
 /**