From 0d1c9c071291b439e98ba5cfb8edf0e1a8a034e8 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Mon, 5 Oct 2026 02:57:24 +0000 Subject: [PATCH 1/2] feat(ci): add recorded Solheim provider smoke test --- .github/workflows/code-qa.yml | 6 + .github/workflows/solheim-provider-smoke.yml | 60 ++++ package.json | 7 + packages/types/src/__tests__/events.test.ts | 234 ++++++++++++++ packages/types/src/__tests__/ipc.test.ts | 55 ++++ packages/types/src/events.ts | 57 ++++ packages/types/src/ipc.ts | 4 +- pnpm-lock.yaml | 9 + scripts/solheim-smoke-workflow.test.mjs | 139 +++++++++ scripts/solheim-smoke/README.md | 56 ++++ scripts/solheim-smoke/config.test.ts | 65 ++++ scripts/solheim-smoke/config.ts | 85 +++++ scripts/solheim-smoke/driver.mts | 181 +++++++++++ scripts/solheim-smoke/evidence.test.ts | 91 ++++++ scripts/solheim-smoke/evidence.ts | 62 ++++ scripts/solheim-smoke/package.json | 4 + scripts/solheim-smoke/record.sh | 44 +++ scripts/solheim-smoke/task-start.test.ts | 215 +++++++++++++ scripts/solheim-smoke/task-start.ts | 96 ++++++ scripts/solheim-smoke/tsconfig.json | 17 + .../__tests__/api-start-task-logging.spec.ts | 295 ++++++++++++++++++ src/extension/api.ts | 178 +++++++++-- 22 files changed, 1934 insertions(+), 26 deletions(-) create mode 100644 .github/workflows/solheim-provider-smoke.yml create mode 100644 packages/types/src/__tests__/events.test.ts create mode 100644 scripts/solheim-smoke-workflow.test.mjs create mode 100644 scripts/solheim-smoke/README.md create mode 100644 scripts/solheim-smoke/config.test.ts create mode 100644 scripts/solheim-smoke/config.ts create mode 100644 scripts/solheim-smoke/driver.mts create mode 100644 scripts/solheim-smoke/evidence.test.ts create mode 100644 scripts/solheim-smoke/evidence.ts create mode 100644 scripts/solheim-smoke/package.json create mode 100644 scripts/solheim-smoke/record.sh create mode 100644 scripts/solheim-smoke/task-start.test.ts create mode 100644 scripts/solheim-smoke/task-start.ts create mode 100644 scripts/solheim-smoke/tsconfig.json create mode 100644 src/extension/__tests__/api-start-task-logging.spec.ts diff --git a/.github/workflows/code-qa.yml b/.github/workflows/code-qa.yml index dcc9fb509d..041a285753 100644 --- a/.github/workflows/code-qa.yml +++ b/.github/workflows/code-qa.yml @@ -92,6 +92,12 @@ jobs: run: pnpm check-types - name: Validate Code QA workflow run: pnpm test:code-qa-ci + - name: Validate Solheim provider smoke workflow + run: pnpm test:solheim-smoke-ci + - name: Test Solheim provider smoke units + run: pnpm test:solheim-smoke:unit + - name: Check Solheim provider smoke types + run: pnpm solheim-smoke:check-types - name: Model-check task lifecycle protocols run: pnpm lifecycle:model-check - name: Validate MCP OAuth integration diff --git a/.github/workflows/solheim-provider-smoke.yml b/.github/workflows/solheim-provider-smoke.yml new file mode 100644 index 0000000000..7f61107186 --- /dev/null +++ b/.github/workflows/solheim-provider-smoke.yml @@ -0,0 +1,60 @@ +name: Solheim provider smoke + +# Project A: infrastructure validation only. No PR diff, review judgment, router, +# deterministic review rule, or GitHub posting. Only trusted main-branch code runs. +on: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: solheim-provider-smoke + cancel-in-progress: false + +jobs: + smoke: + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-24.04 + timeout-minutes: 25 + # Retain the existing protected environment and secret names for compatibility. + # Repository policy must restrict deployments to main and require approval. + environment: final-vscode-review-smoke + steps: + - name: Checkout exact trusted main revision + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - name: Setup Node.js and pnpm + uses: ./.github/actions/setup-node-pnpm + with: + install-args: "--frozen-lockfile" + + - name: Build trusted extension and webview + run: | + pnpm -w bundle + pnpm --filter @roo-code/vscode-webview build + + - name: Install display dependencies + run: sudo apt-get update && sudo apt-get install -y xvfb ffmpeg libasound2t64 + + - name: Run Solheim provider smoke + id: provider + timeout-minutes: 8 + env: + SOLHEIM_API_KEY: ${{ secrets.FINAL_SMOKE_OPENAI_API_KEY }} + SOLHEIM_SMOKE_OUT_DIR: ${{ runner.temp }}/solheim-provider-smoke + run: xvfb-run -a -s "-screen 0 1280x720x24 -nolisten tcp" bash scripts/solheim-smoke/record.sh + + - name: Upload provider smoke verdict and recording + if: ${{ !cancelled() && (steps.provider.outcome == 'success' || steps.provider.outcome == 'failure') }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: solheim-provider-smoke-${{ github.sha }} + path: | + ${{ runner.temp }}/solheim-provider-smoke/verdict.json + ${{ runner.temp }}/solheim-provider-smoke/smoke.mp4 + if-no-files-found: error + retention-days: 7 diff --git a/package.json b/package.json index 1bf0b4e8dd..16b8328865 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,10 @@ "check-types": "turbo check-types --log-order grouped --output-logs new-only", "test": "turbo test --log-order grouped --output-logs new-only", "test:code-qa-ci": "node --test scripts/code-qa-workflow.test.mjs", + "solheim:smoke": "node --import tsx scripts/solheim-smoke/driver.mts", + "test:solheim-smoke-ci": "node --test scripts/solheim-smoke-workflow.test.mjs", + "test:solheim-smoke:unit": "node --import tsx --test scripts/solheim-smoke/*.test.ts", + "solheim-smoke:check-types": "tsc -p scripts/solheim-smoke/tsconfig.json", "test:mutation-ci": "node --test scripts/stryker-diff.test.mjs", "lifecycle:model-check": "tsx scripts/check-task-lifecycle.ts && tsx scripts/check-task-store-concurrency.ts && tsx scripts/check-provider-handoff-scheduler.ts && pnpm cleanup-protocol:model-check && pnpm parser-scope:model-check && tsx scripts/check-completion-persistence.ts && tsx scripts/check-delegated-mode-readers.ts", "fanout-protocol:model-check": "tsx scripts/check-task-fanout-protocol.ts", @@ -34,6 +38,9 @@ "devDependencies": { "@changesets/cli": "2.31.0", "@roo-code/config-typescript": "workspace:^", + "@roo-code/ipc": "workspace:^", + "@roo-code/types": "workspace:^", + "@vscode/test-electron": "2.5.2", "@stryker-mutator/core": "10.0.0", "@stryker-mutator/vitest-runner": "10.0.0", "@types/node": "22.20.1", diff --git a/packages/types/src/__tests__/events.test.ts b/packages/types/src/__tests__/events.test.ts new file mode 100644 index 0000000000..3525e75eac --- /dev/null +++ b/packages/types/src/__tests__/events.test.ts @@ -0,0 +1,234 @@ +import { + RooCodeEventName, + TASK_START_FAILURE_ERROR_CODE, + TASK_START_FAILURE_ERROR_MESSAGE, + TASK_START_STAGES, + taskEventSchema, + taskStartResponseSchema, +} from "../events.js" + +const requestId = "final-smoke-0123abcdef456789" + +describe("taskStartResponseSchema", () => { + it("accepts a success response with the accepted task ID", () => { + const result = taskStartResponseSchema.safeParse({ requestId, success: true, taskId: "task-1" }) + expect(result.success).toBe(true) + + if (result.success) { + expect(result.data).toEqual({ requestId, success: true, taskId: "task-1" }) + } + }) + + it("accepts the fixed failure response", () => { + const result = taskStartResponseSchema.safeParse({ + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + }) + expect(result.success).toBe(true) + + if (result.success && !result.data.success) { + expect(result.data.success).toBe(false) + expect(result.data.stage).toBeUndefined() + } + }) + + it("accepts a failure response carrying each allowed stage", () => { + for (const stage of TASK_START_STAGES) { + const result = taskStartResponseSchema.safeParse({ + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + stage, + }) + expect(result.success).toBe(true) + + if (result.success && !result.data.success) { + expect(result.data.stage).toBe(stage) + } + } + }) + + it("rejects a failure response carrying an unknown stage", () => { + const result = taskStartResponseSchema.safeParse({ + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + stage: "secretStore", + }) + expect(result.success).toBe(false) + }) + + it("rejects a failure response carrying a non-string stage", () => { + const result = taskStartResponseSchema.safeParse({ + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + stage: { name: "taskCreation" }, + }) + expect(result.success).toBe(false) + }) + + it("rejects a success response carrying a stage", () => { + const result = taskStartResponseSchema.safeParse({ + requestId, + success: true, + taskId: "task-1", + stage: "taskCreation", + }) + expect(result.success).toBe(false) + }) + + it("rejects a success response without a taskId", () => { + const result = taskStartResponseSchema.safeParse({ requestId, success: true }) + expect(result.success).toBe(false) + }) + + it("rejects a failure response carrying an unfixed error code", () => { + const result = taskStartResponseSchema.safeParse({ + requestId, + success: false, + errorCode: "provider_auth_failed", + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + }) + expect(result.success).toBe(false) + }) + + it("rejects a failure response carrying a raw error message", () => { + const result = taskStartResponseSchema.safeParse({ + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: "openai rejected the key sk-secret-123 at https://api.example.com", + }) + expect(result.success).toBe(false) + }) + + it("rejects an empty requestId", () => { + const result = taskStartResponseSchema.safeParse({ requestId: "", success: true, taskId: "task-1" }) + expect(result.success).toBe(false) + }) + + it("rejects a success response with an extra top-level field", () => { + const result = taskStartResponseSchema.safeParse({ + requestId, + success: true, + taskId: "task-1", + providerError: "openai rejected the key sk-secret-123", + }) + expect(result.success).toBe(false) + }) + + it("rejects a failure response with an extra field on the error object", () => { + const result = taskStartResponseSchema.safeParse({ + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + detail: { apiKey: "sk-secret-123", url: "https://api.example.com" }, + }) + expect(result.success).toBe(false) + }) +}) + +describe("taskEventSchema TaskStartResponse", () => { + it("routes a success response through the task event envelope", () => { + const result = taskEventSchema.safeParse({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [{ requestId, success: true, taskId: "task-1" }], + }) + expect(result.success).toBe(true) + }) + + it("routes a failure response through the task event envelope", () => { + const result = taskEventSchema.safeParse({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [ + { + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + }, + ], + }) + expect(result.success).toBe(true) + }) + + it("routes a failure response with a stage through the task event envelope", () => { + const result = taskEventSchema.safeParse({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [ + { + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + stage: "taskCreation", + }, + ], + }) + expect(result.success).toBe(true) + }) + + it("rejects a failure payload with an unknown stage through the task event envelope", () => { + const result = taskEventSchema.safeParse({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [ + { + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + stage: "keyring", + }, + ], + }) + expect(result.success).toBe(false) + }) + + it("rejects a failure payload with a stage plus an extra field through the task event envelope", () => { + const result = taskEventSchema.safeParse({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [ + { + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + stage: "taskCreation", + rawError: "sk-secret-123", + }, + ], + }) + expect(result.success).toBe(false) + }) + + it("rejects a success payload with an extra field through the task event envelope", () => { + const result = taskEventSchema.safeParse({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [{ requestId, success: true, taskId: "task-1", stack: "at run (api.ts:1:1)" }], + }) + expect(result.success).toBe(false) + }) + + it("rejects a failure payload with an extra error field through the task event envelope", () => { + const result = taskEventSchema.safeParse({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [ + { + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + rawError: "sk-secret-123", + }, + ], + }) + expect(result.success).toBe(false) + }) +}) diff --git a/packages/types/src/__tests__/ipc.test.ts b/packages/types/src/__tests__/ipc.test.ts index a843354a55..daf9b3cb1d 100644 --- a/packages/types/src/__tests__/ipc.test.ts +++ b/packages/types/src/__tests__/ipc.test.ts @@ -117,4 +117,59 @@ describe("IPC Types", () => { expect(result.success).toBe(false) }) }) + + describe("StartNewTask requestId", () => { + const legacyCommand = { + commandName: TaskCommandName.StartNewTask, + data: { + configuration: {}, + text: "review the diff", + }, + } + + it("should accept a legacy StartNewTask without requestId", () => { + const result = taskCommandSchema.safeParse(legacyCommand) + expect(result.success).toBe(true) + + if (result.success && result.data.commandName === TaskCommandName.StartNewTask) { + expect(result.data.data.requestId).toBeUndefined() + } + }) + + it("should accept and preserve a valid requestId", () => { + const result = taskCommandSchema.safeParse({ + ...legacyCommand, + data: { ...legacyCommand.data, requestId: "final-smoke-0123abcdef456789" }, + }) + expect(result.success).toBe(true) + + if (result.success && result.data.commandName === TaskCommandName.StartNewTask) { + expect(result.data.data.requestId).toBe("final-smoke-0123abcdef456789") + } + }) + + it("should reject an empty requestId", () => { + const result = taskCommandSchema.safeParse({ + ...legacyCommand, + data: { ...legacyCommand.data, requestId: "" }, + }) + expect(result.success).toBe(false) + }) + + it("should reject a requestId longer than 128 characters", () => { + const result = taskCommandSchema.safeParse({ + ...legacyCommand, + data: { ...legacyCommand.data, requestId: "a".repeat(129) }, + }) + expect(result.success).toBe(false) + }) + + it("should reject a requestId with characters outside the allowed set", () => { + const result = taskCommandSchema.safeParse({ + ...legacyCommand, + data: { ...legacyCommand.data, requestId: "id with spaces" }, + }) + expect(result.success).toBe(false) + }) + }) }) diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index 20f7f7e71e..204256980d 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -51,8 +51,59 @@ export enum RooCodeEventName { CommandsResponse = "commandsResponse", ModesResponse = "modesResponse", ModelsResponse = "modelsResponse", + /** Correlated reply to a StartNewTask command that carried a requestId. */ + TaskStartResponse = "taskStartResponse", } +/** + * TaskStartResponse + */ + +// The requestId is generated by the command sender and copied into the reply, +// so its shape is bounded and restricted to characters that cannot smuggle +// log structure. +export const taskStartRequestIdSchema = z + .string() + .min(1) + .max(128) + .regex(/^[A-Za-z0-9._:-]+$/) + +// The failure values are fixed constants. The response never carries raw +// thrown messages, prompts, images, provider settings, secrets, URLs, or +// stack traces. Both variants reject unknown keys, so an extra or +// secret-bearing field fails validation instead of being stripped. +export const TASK_START_FAILURE_ERROR_CODE = "task_start_failed" +export const TASK_START_FAILURE_ERROR_MESSAGE = "Task start failed" + +// Fixed stage vocabulary for a failed correlated start. The stage names the +// startup phase that stopped responding; it carries no error content. +export const TASK_START_STAGES = ["eviction", "settings", "taskCreation", "unknown"] as const + +export const taskStartStageSchema = z.enum(TASK_START_STAGES) + +export type TaskStartStage = (typeof TASK_START_STAGES)[number] + +export const taskStartResponseSchema = z.discriminatedUnion("success", [ + z + .object({ + requestId: taskStartRequestIdSchema, + success: z.literal(true), + taskId: z.string().min(1), + }) + .strict(), + z + .object({ + requestId: taskStartRequestIdSchema, + success: z.literal(false), + errorCode: z.literal(TASK_START_FAILURE_ERROR_CODE), + errorMessage: z.literal(TASK_START_FAILURE_ERROR_MESSAGE), + stage: taskStartStageSchema.optional(), + }) + .strict(), +]) + +export type TaskStartResponse = z.infer + /** * RooCodeEvents */ @@ -125,6 +176,7 @@ export const rooCodeEventsSchema = z.object({ ]), [RooCodeEventName.ModesResponse]: z.tuple([z.array(z.object({ slug: z.string(), name: z.string() }))]), [RooCodeEventName.ModelsResponse]: z.tuple([z.record(z.string(), modelInfoSchema)]), + [RooCodeEventName.TaskStartResponse]: z.tuple([taskStartResponseSchema]), }) export type RooCodeEvents = z.infer @@ -270,6 +322,11 @@ export const taskEventSchema = z.discriminatedUnion("eventName", [ payload: rooCodeEventsSchema.shape[RooCodeEventName.ModelsResponse], taskId: z.number().optional(), }), + z.object({ + eventName: z.literal(RooCodeEventName.TaskStartResponse), + payload: rooCodeEventsSchema.shape[RooCodeEventName.TaskStartResponse], + taskId: z.number().optional(), + }), ]) export type TaskEvent = z.infer diff --git a/packages/types/src/ipc.ts b/packages/types/src/ipc.ts index fea040af0b..e7df762069 100644 --- a/packages/types/src/ipc.ts +++ b/packages/types/src/ipc.ts @@ -1,6 +1,6 @@ import { z } from "zod" -import { type TaskEvent, taskEventSchema } from "./events.js" +import { type TaskEvent, taskEventSchema, taskStartRequestIdSchema } from "./events.js" import { rooCodeSettingsSchema } from "./global-settings.js" /** @@ -64,6 +64,8 @@ export const taskCommandSchema = z.discriminatedUnion("commandName", [ text: z.string(), images: z.array(z.string()).optional(), newTab: z.boolean().optional(), + // Senders without a requestId keep the fire-and-forget contract. + requestId: taskStartRequestIdSchema.optional(), }), }), z.object({ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b12ecd1c77..deeb91fd61 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -30,6 +30,12 @@ importers: '@roo-code/config-typescript': specifier: workspace:^ version: link:packages/config-typescript + '@roo-code/ipc': + specifier: workspace:^ + version: link:packages/ipc + '@roo-code/types': + specifier: workspace:^ + version: link:packages/types '@stryker-mutator/core': specifier: 10.0.0 version: 10.0.0(@types/node@22.20.1) @@ -39,6 +45,9 @@ importers: '@types/node': specifier: 22.20.1 version: 22.20.1 + '@vscode/test-electron': + specifier: 2.5.2 + version: 2.5.2 '@vscode/vsce': specifier: 3.9.2 version: 3.9.2 diff --git a/scripts/solheim-smoke-workflow.test.mjs b/scripts/solheim-smoke-workflow.test.mjs new file mode 100644 index 0000000000..1239cfe5a5 --- /dev/null +++ b/scripts/solheim-smoke-workflow.test.mjs @@ -0,0 +1,139 @@ +import assert from "node:assert/strict" +import fs from "node:fs" +import { URL } from "node:url" +import { describe, it } from "node:test" +import { spawnSync } from "node:child_process" +import os from "node:os" +import path from "node:path" +const workflow = fs.readFileSync(new URL("../.github/workflows/solheim-provider-smoke.yml", import.meta.url), "utf8") +const codeQa = fs.readFileSync(new URL("../.github/workflows/code-qa.yml", import.meta.url), "utf8") +const pkg = JSON.parse(fs.readFileSync(new URL("../package.json", import.meta.url), "utf8")) +const driver = fs.readFileSync(new URL("./solheim-smoke/driver.mts", import.meta.url), "utf8") +const recorder = fs.readFileSync(new URL("./solheim-smoke/record.sh", import.meta.url), "utf8") +const smokePkg = JSON.parse(fs.readFileSync(new URL("./solheim-smoke/package.json", import.meta.url), "utf8")) + +describe("Project A provider smoke workflow", () => { + it("runs smoke TypeScript as ESM without requiring compiled workspace packages", () => { + assert.equal(smokePkg.type, "module") + assert.equal(smokePkg.private, true) + }) + it("runs only by manual dispatch against the exact trusted main revision", () => { + assert.match(workflow, /workflow_dispatch:/) + assert.match(workflow, /if: github.ref == 'refs\/heads\/main'/) + assert.match(workflow, /ref: \$\{\{ github.sha \}\}/) + assert.ok(!workflow.includes("pull_request_review:")) + assert.ok(!workflow.includes("pull_request_target:")) + assert.match(workflow, /persist-credentials: false/) + assert.match(workflow, /environment: final-vscode-review-smoke/) + }) + it("has one credential-bearing step, no GitHub write permissions, and no posting", () => { + assert.equal((workflow.match(/secrets\./g) ?? []).length, 1) + const before = workflow.slice(0, workflow.indexOf("- name: Run Solheim provider smoke")) + assert.ok(!before.includes("secrets.")) + assert.match(workflow, /permissions:\n\s+contents: read/) + assert.ok(!/\bwrite\b/.test(workflow)) + assert.ok(!workflow.includes("github-script")) + assert.ok(!workflow.includes("post-review")) + }) + it("has one smoke job and no review, routing, shadow, or deterministic lanes", () => { + const jobs = workflow.split("jobs:\n")[1] ?? "" + assert.deepEqual( + [...jobs.matchAll(/^ {4}([a-z-]+):$/gm)].map((match) => match[1]), + ["smoke"], + ) + for (const name of [ + "theme-review", + "probe-dispatch", + "rules-review", + "merge-report", + "bounded-code-review", + "model-qualification", + ]) + assert.ok(!workflow.includes(name)) + assert.ok(!driver.includes("model-qualification")) + assert.ok(!driver.includes("AcknowledgeCompletion")) + }) + it("bounds runtime and serializes the single provider instance", () => { + assert.match(workflow, /cancel-in-progress: false/) + assert.match(workflow, /timeout-minutes: 8/) + }) + it("uploads only the verdict and video, never host logs or storage", () => { + const paths = workflow.split("path: |\n")[1]?.split(" if-no-files-found:")[0] + assert.deepEqual( + paths + ?.trim() + .split("\n") + .map((line) => line.trim()), + [ + "${{ runner.temp }}/solheim-provider-smoke/verdict.json", + "${{ runner.temp }}/solheim-provider-smoke/smoke.mp4", + ], + ) + assert.match(workflow, /!cancelled\(\)/) + assert.match(workflow, /if-no-files-found: error/) + assert.ok(!driver.includes("console.error(")) + assert.ok(!driver.includes("stderrLog")) + }) + it("records a fresh display with credential-free ffmpeg and bounded teardown", () => { + assert.match(workflow, /install -y xvfb ffmpeg/) + assert.match(workflow, /xvfb-run -a -s "-screen 0 1280x720x24 -nolisten tcp"/) + assert.match(recorder, /env -i "PATH=\$PATH" "DISPLAY=\$DISPLAY"/) + assert.match(recorder, /-t 480 -fs 125829120/) + assert.match(recorder, /frag_keyframe\+empty_moov/) + assert.match(recorder, /trap finish EXIT/) + assert.match(recorder, /attempt < 50/) + assert.ok(!recorder.includes("SOLHEIM_API_KEY")) + assert.match(workflow, /retention-days: 7/) + }) + it("keeps Code QA and package commands focused on smoke contracts", () => { + for (const command of ["test:solheim-smoke:unit", "test:solheim-smoke-ci", "solheim-smoke:check-types"]) + assert.ok(codeQa.includes(command)) + assert.ok(!codeQa.includes("model-qualification")) + assert.ok(!Object.keys(pkg.scripts).some((name) => name.includes("model-qualification"))) + assert.ok(!codeQa.includes("final-smoke")) + assert.equal(pkg.scripts["solheim:smoke"], "node --import tsx scripts/solheim-smoke/driver.mts") + assert.ok(!fs.existsSync(new URL("../.github/workflows/final-vscode-review-smoke.yml", import.meta.url))) + }) +}) + +// Runs record.sh with stub ffmpeg and ffprobe, so no display or codec is needed. +function runRecorder(args, { ffmpegWrites = true, ffprobeOk = true } = {}) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "record-sh-")) + const bin = path.join(dir, "bin") + fs.mkdirSync(bin) + const stub = (name, body) => + fs.writeFileSync(path.join(bin, name), `#!/usr/bin/env bash\n${body}\n`, { mode: 0o755 }) + // The stub ffmpeg writes the last argument, stops on INT, and exits. + stub( + "ffmpeg", + `${ffmpegWrites ? 'for a; do :; done; echo data > "$a"' : ":"}\ntrap 'exit 0' INT\nwhile true; do sleep 0.05; done`, + ) + stub("ffprobe", ffprobeOk ? "exit 0" : "exit 1") + const result = spawnSync("bash", [new URL("./solheim-smoke/record.sh", import.meta.url).pathname, ...args], { + env: { PATH: `${bin}:${process.env.PATH}`, DISPLAY: ":99", SOLHEIM_SMOKE_OUT_DIR: path.join(dir, "out") }, + encoding: "utf8", + timeout: 20_000, + }) + fs.rmSync(dir, { recursive: true, force: true }) + return result +} + +describe("record.sh exit and validation behavior", () => { + it("passes when the command succeeds and the recording is valid", () => { + assert.equal(runRecorder(["true"]).status, 0) + }) + it("keeps the exit code of a failing command", () => { + assert.equal(runRecorder(["bash", "-c", "exit 7"]).status, 7) + }) + it("fails when the recording is missing", () => { + const result = runRecorder(["true"], { ffmpegWrites: false }) + assert.equal(result.status, 1) + assert.match(result.stdout, /recording failed/) + }) + it("fails when ffprobe rejects the recording", () => { + assert.equal(runRecorder(["true"], { ffprobeOk: false }).status, 1) + }) + it("keeps a failing command code when the recording is also invalid", () => { + assert.equal(runRecorder(["bash", "-c", "exit 7"], { ffprobeOk: false }).status, 7) + }) +}) diff --git a/scripts/solheim-smoke/README.md b/scripts/solheim-smoke/README.md new file mode 100644 index 0000000000..35e7189138 --- /dev/null +++ b/scripts/solheim-smoke/README.md @@ -0,0 +1,56 @@ +# Project A: Solheim provider smoke + +Infrastructure validation only: a real VS Code extension activates, accepts a correlated IPC task, +receives output from Solheim's OpenAI-compatible provider, and produces a completion result. +It does not read a PR, assess an answer's meaning, select checks, find defects, or post to GitHub. + +The active workflow is `.github/workflows/solheim-provider-smoke.yml`. It is manual-only, restricted to +`refs/heads/main`, and checks out the exact triggering SHA. It builds trusted extension code before the +single credential-bearing step. There is one job, no PR input or approval gate, no review/posting job, +and `contents: read` is the only GitHub permission. This deliberately validates the provider integration +on main, not arbitrary PR artifacts. It is not a required PR review check. + +The protected environment `final-vscode-review-smoke` and its secret `FINAL_SMOKE_OPENAI_API_KEY` +retain their existing names to avoid an unnecessary repository-settings migration. Restrict its deployment +policy to main and require approval. No environment or secret settings have been changed by this cleanup. + +The provider endpoint and model are fixed: `https://api.solheim.ai/v1`, `qwen3.8-27b`. The task has no tool +groups; delegation, follow-ups, writes, shell and MCP are disabled. The driver opens an empty temporary +workspace with isolated VS Code storage and an allowlisted host environment. The provider credential is +supplied over IPC, not inherited by the child or passed on its command line. Temporary extension storage +can contain the configured credential while the session runs; it is removed at teardown and never uploaded. +This is process/environment isolation, not a sandbox for malicious extension code—CI runs trusted main only. + +Activation and completion each have a 120-second limit; task-start acknowledgment has a 35-second limit, +with 30-second per-stage bounds in the extension. The live CI step has an eight-minute outer timeout. +Teardown escalates through close, terminate, and kill within independent five-second windows. A non-empty +completion and actual output-token usage must belong to the accepted task. Activation alone cannot pass. +The driver never approves dialogs or tool execution; it closes the task during teardown. + +The action uploads `verdict.json` (fixed failure code, boolean gates, duration and provider/model names) +and `smoke.mp4`, a 1280×720, 12 fps screen recording of the fresh CI virtual display. It records up to +eight minutes / approximately 120 MiB, with bounded shutdown and seven-day artifact retention. Video +contains the visible smoke UI, potentially including the fixed prompt and model answer; unlike the verdict, +it is not text-redacted. No settings screen is opened, and no host logs, storage, task configuration or +credential files are uploaded. Fragmented MP4 preserves completed fragments on abrupt interruption. +Ordinary success and failure runs upload both artifacts; cancellation may prevent finalization/upload. + +Checks, without a provider key or network calls: + +```sh +pnpm test:solheim-smoke:unit +pnpm test:solheim-smoke-ci +pnpm solheim-smoke:check-types +``` + +For an explicitly authorized local live run, build the extension/webview and supply `SOLHEIM_API_KEY` +through the environment, never an argument. Set `SOLHEIM_SMOKE_OUT_DIR` to a scratch directory and run +`pnpm solheim:smoke`. Linux needs Xvfb or an existing display. Do not run an untrusted extension with a key. +To record locally, install ffmpeg and run `xvfb-run -a -s "-screen 0 1280x720x24 -nolisten tcp" +bash scripts/solheim-smoke/record.sh` instead. Do not attach the recorder to your existing desktop. + +The new simplified driver/workflow has not yet had a real-provider CI run. Earlier question-lane runs +established the underlying integration, not this refactor's live status. Review qualification stopped +after the frozen Stage 1B pilot produced zero incremental verified defects. Its research harness and +receipts are archived outside this smoke-only change; the chronology, results, caveats and preservation +details are recorded in [tracking issue #1896](https://github.com/Zoo-Code-Org/Zoo-Code/issues/1896). diff --git a/scripts/solheim-smoke/config.test.ts b/scripts/solheim-smoke/config.test.ts new file mode 100644 index 0000000000..345062f0d3 --- /dev/null +++ b/scripts/solheim-smoke/config.test.ts @@ -0,0 +1,65 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { + buildChildEnvironment, + buildLaunchArgs, + buildSmokeConfiguration, + SOLHEIM_BASE_URL, + SOLHEIM_MODEL, + SMOKE_PROMPT, +} from "./config.ts" + +describe("Solheim provider smoke configuration", () => { + it("uses one fixed provider and a small output budget", () => { + const config = buildSmokeConfiguration("test-credential") + assert.equal(config.openAiBaseUrl, SOLHEIM_BASE_URL) + assert.equal(config.openAiModelId, SOLHEIM_MODEL) + assert.equal(config.openAiApiKey, "test-credential") + assert.equal(config.modelMaxTokens, 512) + assert.deepEqual(JSON.parse(config.openAiExtraBody ?? ""), { chat_template_kwargs: { enable_thinking: false } }) + }) + it("is a no-tools infrastructure task, with no delegation or reviewer instructions", () => { + const config = buildSmokeConfiguration("test-credential") + assert.deepEqual(config.customModes?.[0]?.groups, []) + assert.equal(config.mode, "provider-smoke") + for (const name of [ + "alwaysAllowWrite", + "alwaysAllowExecute", + "alwaysAllowMcp", + "alwaysAllowSubtasks", + "alwaysAllowReadOnly", + ] as const) + assert.equal(config[name], false) + assert.ok(config.disabledTools?.includes("new_task")) + assert.ok(config.disabledTools?.includes("ask_followup_question")) + assert.match(SMOKE_PROMPT, /not a code review/) + }) + it("passes only allowlisted host variables into the isolated extension host", () => { + const env = buildChildEnvironment( + { + PATH: "/bin", + DISPLAY: ":1", + SOLHEIM_API_KEY: "secret", + GITHUB_TOKEN: "token", + NODE_OPTIONS: "--require=bad.js", + XDG_CONFIG_HOME: "/real-home", + EXTRA_CREDENTIAL: "other", + }, + "/tmp/isolated", + "/tmp/ipc", + ) + assert.equal(env.PATH, "/bin") + assert.equal(env.DISPLAY, ":1") + assert.equal(env.HOME, "/tmp/isolated") + assert.equal(env.XDG_CONFIG_HOME, "/tmp/isolated/.config") + for (const name of ["SOLHEIM_API_KEY", "GITHUB_TOKEN", "NODE_OPTIONS", "EXTRA_CREDENTIAL"]) + assert.equal(env[name], undefined) + }) + it("uses isolated storage and a keyring-free launch", () => { + const args = buildLaunchArgs("/empty-workspace", "/data", "/extensions", "/extension") + assert.equal(args[0], "/empty-workspace") + assert.ok(args.includes("--password-store=basic")) + assert.ok(args.includes("--extensionDevelopmentPath=/extension")) + assert.ok(args.includes("--user-data-dir=/data")) + }) +}) diff --git a/scripts/solheim-smoke/config.ts b/scripts/solheim-smoke/config.ts new file mode 100644 index 0000000000..69c5d6aa71 --- /dev/null +++ b/scripts/solheim-smoke/config.ts @@ -0,0 +1,85 @@ +import path from "node:path" +import type { RooCodeSettings } from "@roo-code/types" + +export const SOLHEIM_BASE_URL = "https://api.solheim.ai/v1" +export const SOLHEIM_MODEL = "qwen3.8-27b" +export const SMOKE_TIMEOUT_MS = 120_000 +export const SMOKE_PROMPT = + "This is a provider connectivity smoke test, not a code review. Call attempt_completion once with the result: Provider smoke completed. Do not read files, use other tools, delegate, or judge code." + +export function buildSmokeConfiguration(apiKey: string): RooCodeSettings { + return { + apiProvider: "openai", + openAiBaseUrl: SOLHEIM_BASE_URL, + openAiApiKey: apiKey, + openAiModelId: SOLHEIM_MODEL, + openAiR1FormatEnabled: true, + openAiExtraBody: JSON.stringify({ chat_template_kwargs: { enable_thinking: false } }), + modelMaxTokens: 512, + autoCondenseContext: false, + mode: "provider-smoke", + customModes: [ + { + slug: "provider-smoke", + name: "Provider smoke", + roleDefinition: "Complete one connectivity smoke task. You are not a reviewer.", + groups: [], + }, + ], + disabledTools: [ + "ask_followup_question", + "new_task", + "switch_mode", + "update_todo_list", + "skill", + "run_slash_command", + ], + autoApprovalEnabled: true, + alwaysAllowReadOnly: false, + alwaysAllowReadOnlyOutsideWorkspace: false, + alwaysAllowWrite: false, + alwaysAllowWriteOutsideWorkspace: false, + alwaysAllowWriteProtected: false, + alwaysAllowExecute: false, + alwaysAllowMcp: false, + alwaysAllowModeSwitch: false, + alwaysAllowSubtasks: false, + alwaysAllowFollowupQuestions: false, + } +} + +// Inherit only host-launch necessities, never the credential-bearing parent environment. +export function buildChildEnvironment( + env: NodeJS.ProcessEnv, + isolatedHome: string, + socketPath: string, +): NodeJS.ProcessEnv { + const child: NodeJS.ProcessEnv = {} + for (const name of ["PATH", "DISPLAY", "XAUTHORITY", "LANG", "LC_ALL", "TMPDIR", "SYSTEMROOT", "WINDIR"]) + if (env[name]) child[name] = env[name] + return { + ...child, + HOME: isolatedHome, + USERPROFILE: isolatedHome, + XDG_CONFIG_HOME: path.join(isolatedHome, ".config"), + XDG_CACHE_HOME: path.join(isolatedHome, ".cache"), + ROO_CODE_IPC_SOCKET_PATH: socketPath, + ROO_CODE_TASK_START_STAGE_TIMEOUT_MS: "30000", + } +} + +export function buildLaunchArgs(workspace: string, userData: string, extensions: string, extension: string): string[] { + return [ + workspace, + `--user-data-dir=${userData}`, + `--extensions-dir=${extensions}`, + `--extensionDevelopmentPath=${extension}`, + "--no-sandbox", + "--disable-gpu-sandbox", + "--password-store=basic", + "--disable-updates", + "--skip-welcome", + "--skip-release-notes", + "--disable-workspace-trust", + ] +} diff --git a/scripts/solheim-smoke/driver.mts b/scripts/solheim-smoke/driver.mts new file mode 100644 index 0000000000..2a44a262c7 --- /dev/null +++ b/scripts/solheim-smoke/driver.mts @@ -0,0 +1,181 @@ +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { randomUUID } from "node:crypto" +import { fileURLToPath } from "node:url" +import { spawn, type ChildProcess } from "node:child_process" +import { downloadAndUnzipVSCode } from "@vscode/test-electron" +import { IpcClient } from "@roo-code/ipc" +import { IpcMessageType, TaskCommandName, type TaskEvent } from "@roo-code/types" +import { + buildChildEnvironment, + buildLaunchArgs, + buildSmokeConfiguration, + SMOKE_PROMPT, + SMOKE_TIMEOUT_MS, + SOLHEIM_BASE_URL, + SOLHEIM_MODEL, +} from "./config.ts" +import { collectEvidence, smokeCode, type SmokeCode, type SmokeEvidence } from "./evidence.ts" +import { createTaskStartWaiter } from "./task-start.ts" + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") +const out = process.env.SOLHEIM_SMOKE_OUT_DIR +if (!out) throw new Error("Set SOLHEIM_SMOKE_OUT_DIR to a scratch directory") +await fs.mkdir(out, { recursive: true }) +const started = Date.now() +const evidence: SmokeEvidence = { + providerResponseSeen: false, + completionSeen: false, + unexpectedTool: false, + aborted: false, +} +let code: SmokeCode = "SETUP_FAILED" +let activated = false +let accepted = false +let sessionLost = false +let child: ChildProcess | undefined +let client: IpcClient | undefined +let temp: string | undefined +let exited = false +let waiter: ReturnType | undefined +const pause = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) +async function until(check: () => boolean, timeoutMs: number, stopOnSessionLoss = true): Promise { + const deadline = Date.now() + timeoutMs + while (!check() && (!stopOnSessionLoss || !sessionLost) && Date.now() < deadline) await pause(100) + return check() +} +function signalChild(signal: NodeJS.Signals): void { + if (!child?.pid || exited) return + try { + process.kill(-child.pid, signal) + } catch { + child.kill(signal) + } +} +function interrupted(): void { + sessionLost = true + waiter?.onExit() + signalChild("SIGTERM") +} +process.once("SIGINT", interrupted) +process.once("SIGTERM", interrupted) + +try { + const apiKey = process.env.SOLHEIM_API_KEY + if (!apiKey) throw new Error("Provider credential missing") + const extension = path.join(root, "src") + await fs.access(path.join(extension, "package.json")) + temp = await fs.mkdtemp(path.join(os.tmpdir(), "solheim-smoke-")) + const workspace = path.join(temp, "workspace") + const home = path.join(temp, "home") + await Promise.all([workspace, home].map((folder) => fs.mkdir(folder))) + const socket = path.join(temp, "ipc.sock") + const pkg = JSON.parse(await fs.readFile(path.join(root, "apps/vscode-e2e/package.json"), "utf8")) as { + devDependencies: Record + } + const executable = await downloadAndUnzipVSCode({ + version: pkg.devDependencies["@types/vscode"] ?? "1.100.0", + cachePath: path.join(root, "apps/vscode-e2e/.vscode-test"), + }) + const args = buildLaunchArgs(workspace, path.join(temp, "userdata"), path.join(temp, "extensions"), extension) + child = process.env.DISPLAY + ? spawn(executable, args, { + detached: true, + env: buildChildEnvironment(process.env, home, socket), + stdio: "ignore", + }) + : spawn("xvfb-run", ["-a", executable, ...args], { + detached: true, + env: buildChildEnvironment(process.env, home, socket), + stdio: "ignore", + }) + child.once("error", () => { + sessionLost = true + exited = true + waiter?.onExit() + }) + child.once("close", () => { + sessionLost = true + exited = true + waiter?.onExit() + }) + client = new IpcClient(socket, () => {}) + client.on(IpcMessageType.Disconnect, () => { + sessionLost = true + waiter?.onDisconnect() + }) + let taskId: string | undefined + client.on(IpcMessageType.TaskEvent, (event: TaskEvent) => { + const id = waiter?.onTaskEvent(event) + if (id && taskId === undefined) taskId = id + collectEvidence(evidence, event, taskId) + }) + activated = await until(() => client?.isReady === true, SMOKE_TIMEOUT_MS) + if (!activated) code = "ACTIVATION_FAILED" + else { + const requestId = randomUUID() + waiter = createTaskStartWaiter({ requestId, timeoutMs: 35_000 }) + client.sendCommand({ + commandName: TaskCommandName.StartNewTask, + data: { requestId, text: SMOKE_PROMPT, newTab: false, configuration: buildSmokeConfiguration(apiKey) }, + }) + try { + taskId = await waiter.result + accepted = true + } catch { + code = "TASK_START_FAILED" + } + if (accepted) { + await until( + () => + evidence.unexpectedTool || + evidence.aborted || + (evidence.providerResponseSeen && evidence.completionSeen), + SMOKE_TIMEOUT_MS, + ) + code = smokeCode(evidence, sessionLost) + } + } +} catch { + /* Keep failures fixed and credential-free, never print thrown provider/host text. */ +} finally { + // Teardown is independently bounded, including spawn/activation failures. + if (client?.isReady) client.sendCommand({ commandName: TaskCommandName.CloseTask }) + if (child && !exited) { + await until(() => exited, 5_000, false) + if (!exited) { + signalChild("SIGTERM") + await until(() => exited, 5_000, false) + } + if (!exited) { + signalChild("SIGKILL") + await until(() => exited, 5_000, false) + } + if (!exited && code === "OK") code = "TEARDOWN_FAILED" + } + client?.disconnect() + if (temp && (!child || exited)) await fs.rm(temp, { recursive: true, force: true }) +} +// Whitelisted metadata only: model answers and host logs are never saved or uploaded. +const verdict = { + kind: "provider-smoke", + model: SOLHEIM_MODEL, + provider: SOLHEIM_BASE_URL, + passed: code === "OK", + code, + gates: { + extensionActivated: activated, + ipcConnected: activated, + taskAccepted: accepted, + providerResponseSeen: evidence.providerResponseSeen, + completionSeen: evidence.completionSeen, + }, + durationMs: Date.now() - started, +} +// Write then rename, so a reader sees the old or the new verdict and never a partial one. +const verdictTemp = path.join(out, `verdict.json.${process.pid}.tmp`) +await fs.writeFile(verdictTemp, JSON.stringify(verdict, null, 2)) +await fs.rename(verdictTemp, path.join(out, "verdict.json")) +console.log(`Solheim provider smoke: ${code}`) +process.exit(code === "OK" ? 0 : 1) diff --git a/scripts/solheim-smoke/evidence.test.ts b/scripts/solheim-smoke/evidence.test.ts new file mode 100644 index 0000000000..b3be3faf4f --- /dev/null +++ b/scripts/solheim-smoke/evidence.test.ts @@ -0,0 +1,91 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { RooCodeEventName, type TaskEvent } from "@roo-code/types" +import { collectEvidence, smokeCode, type SmokeEvidence } from "./evidence.ts" +const fresh = (): SmokeEvidence => ({ + providerResponseSeen: false, + completionSeen: false, + unexpectedTool: false, + aborted: false, +}) +const usage = ( + taskId: string, + totalTokensOut: number, +): Extract => ({ + eventName: RooCodeEventName.TaskTokenUsageUpdated, + payload: [taskId, { totalTokensIn: 1, totalTokensOut, totalCost: 0, contextTokens: 1 }, {}], +}) +const message = (taskId: string, value: Record) => ({ + eventName: RooCodeEventName.Message, + payload: [{ taskId, action: "created", message: { ts: 1, ...value } }], +}) + +describe("provider smoke evidence", () => { + it("rejects malformed and uncorrelated events, including activity before task acceptance", () => { + const state = fresh() + collectEvidence(state, usage("root", 5), undefined) + collectEvidence(state, usage("other", 5), "root") + collectEvidence( + state, + { eventName: RooCodeEventName.TaskTokenUsageUpdated, payload: ["root", { totalTokensOut: "5" }, {}] }, + "root", + ) + assert.deepEqual(state, fresh()) + }) + it("requires actual output usage; echoed task text is not provider evidence", () => { + const state = fresh() + collectEvidence(state, message("root", { type: "say", say: "text", text: "task prompt" }), "root") + collectEvidence(state, usage("root", 0), "root") + assert.equal(state.providerResponseSeen, false) + collectEvidence(state, usage("root", 5), "root") + assert.equal(state.providerResponseSeen, true) + }) + it("accepts any non-empty completed response, without reviewing its meaning", () => { + const state = fresh() + collectEvidence( + state, + message("root", { type: "say", say: "completion_result", text: "any completion", partial: true }), + "root", + ) + assert.equal(state.completionSeen, false) + collectEvidence( + state, + message("root", { type: "say", say: "completion_result", text: "any completion" }), + "root", + ) + assert.equal(state.completionSeen, true) + assert.ok(!JSON.stringify(state).includes("any completion")) + }) + it("allows the final completion dialog but rejects other tools", () => { + const state = fresh() + collectEvidence(state, message("root", { type: "ask", ask: "completion_result" }), "root") + assert.equal(state.completionSeen, false) + assert.equal(state.unexpectedTool, false) + collectEvidence(state, message("other", { type: "ask", ask: "command", text: "ignored" }), "root") + assert.equal(state.unexpectedTool, false) + collectEvidence(state, message("root", { type: "ask", ask: "command", text: "never stored" }), "root") + assert.equal(state.unexpectedTool, true) + const noAsk = fresh() + const attempted = usage("root", 5) + attempted.payload[2] = { execute_command: { attempts: 1, failures: 1 } } + collectEvidence(noAsk, attempted, "root") + assert.equal(noAsk.unexpectedTool, true) + }) + it("cannot pass on activation alone or ignore a lost session", () => { + assert.equal(smokeCode(fresh(), false), "PROVIDER_TIMEOUT") + assert.equal(smokeCode({ ...fresh(), providerResponseSeen: true }, false), "COMPLETION_MISSING") + assert.equal(smokeCode({ ...fresh(), providerResponseSeen: true, completionSeen: true }, false), "OK") + assert.equal(smokeCode({ ...fresh(), providerResponseSeen: true, completionSeen: true }, true), "SESSION_LOST") + assert.equal( + smokeCode({ ...fresh(), providerResponseSeen: true, completionSeen: true, unexpectedTool: true }, false), + "UNEXPECTED_TOOL", + ) + }) + it("rejects delegation and aborts on the accepted root", () => { + const state = fresh() + collectEvidence(state, { eventName: RooCodeEventName.TaskDelegated, payload: ["root", "child"] }, "root") + assert.equal(state.unexpectedTool, true) + collectEvidence(state, { eventName: RooCodeEventName.TaskAborted, payload: ["root"] }, "root") + assert.equal(smokeCode(state, false), "SESSION_LOST") + }) +}) diff --git a/scripts/solheim-smoke/evidence.ts b/scripts/solheim-smoke/evidence.ts new file mode 100644 index 0000000000..3e436ca8c4 --- /dev/null +++ b/scripts/solheim-smoke/evidence.ts @@ -0,0 +1,62 @@ +import { RooCodeEventName, taskEventSchema, type TaskEvent } from "@roo-code/types" + +export type SmokeEvidence = { + providerResponseSeen: boolean + completionSeen: boolean + unexpectedTool: boolean + aborted: boolean +} + +// Only schema-valid events for the accepted root task can satisfy smoke gates. +// No model text, credentials, tool arguments, or task IDs are persisted. +export function collectEvidence(state: SmokeEvidence, raw: unknown, rootTaskId: string | undefined): void { + if (!rootTaskId) return + const parsed = taskEventSchema.safeParse(raw) + if (!parsed.success) return + const event: TaskEvent = parsed.data + if (event.eventName === RooCodeEventName.Message) { + const entry = event.payload[0] + if (entry.taskId !== rootTaskId || entry.message.partial === true) return + const message = entry.message + if (message.type === "say" && message.say === "completion_result" && message.text?.trim()) + state.completionSeen = true + if (message.type === "ask" && message.ask !== "completion_result") state.unexpectedTool = true + } else if (event.payload[0] === rootTaskId) { + if ( + event.eventName === RooCodeEventName.TaskTokenUsageUpdated || + event.eventName === RooCodeEventName.TaskCompleted + ) { + if ( + Object.entries(event.payload[2]).some( + ([tool, usage]) => tool !== "attempt_completion" && usage.attempts > 0, + ) + ) + state.unexpectedTool = true + } + if (event.eventName === RooCodeEventName.TaskTokenUsageUpdated && event.payload[1].totalTokensOut > 0) + state.providerResponseSeen = true + if (event.eventName === RooCodeEventName.TaskCompleted && event.payload[1].totalTokensOut > 0) + state.providerResponseSeen = true + if (event.eventName === RooCodeEventName.TaskAborted) state.aborted = true + if (event.eventName === RooCodeEventName.TaskDelegated || event.eventName === RooCodeEventName.TaskSpawned) + state.unexpectedTool = true + } +} + +export type SmokeCode = + | "OK" + | "SETUP_FAILED" + | "ACTIVATION_FAILED" + | "TASK_START_FAILED" + | "SESSION_LOST" + | "UNEXPECTED_TOOL" + | "PROVIDER_TIMEOUT" + | "COMPLETION_MISSING" + | "TEARDOWN_FAILED" +export function smokeCode(state: SmokeEvidence, sessionLost: boolean): SmokeCode { + if (sessionLost || state.aborted) return "SESSION_LOST" + if (state.unexpectedTool) return "UNEXPECTED_TOOL" + if (!state.providerResponseSeen) return "PROVIDER_TIMEOUT" + if (!state.completionSeen) return "COMPLETION_MISSING" + return "OK" +} diff --git a/scripts/solheim-smoke/package.json b/scripts/solheim-smoke/package.json new file mode 100644 index 0000000000..c4fb2812b9 --- /dev/null +++ b/scripts/solheim-smoke/package.json @@ -0,0 +1,4 @@ +{ + "private": true, + "type": "module" +} diff --git a/scripts/solheim-smoke/record.sh b/scripts/solheim-smoke/record.sh new file mode 100644 index 0000000000..f69e17b17f --- /dev/null +++ b/scripts/solheim-smoke/record.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Run inside a fresh xvfb-run display, never record an existing desktop. +set -euo pipefail +: "${DISPLAY:?Run this wrapper under xvfb-run}" +: "${SOLHEIM_SMOKE_OUT_DIR:?Set a scratch output directory}" +mkdir -p "$SOLHEIM_SMOKE_OUT_DIR" +recording="$SOLHEIM_SMOKE_OUT_DIR/smoke.mp4" + +# The recorder needs display access, not provider/GitHub credentials. Fragmented +# MP4 keeps completed fragments readable if the action is forcibly interrupted. +env -i "PATH=$PATH" "DISPLAY=$DISPLAY" "XAUTHORITY=${XAUTHORITY:-}" \ + ffmpeg -nostdin -hide_banner -loglevel error -y \ + -f x11grab -video_size 1280x720 -framerate 12 -i "$DISPLAY" \ + -an -c:v libx264 -preset ultrafast -crf 28 -pix_fmt yuv420p -g 24 \ + -t 480 -fs 125829120 -movflags +frag_keyframe+empty_moov \ + "$recording" >/dev/null 2>&1 & +recorder_pid=$! + +finish() { + result=$? + trap - EXIT + kill -INT "$recorder_pid" 2>/dev/null || true + # A hung recorder must not hold the action open indefinitely. + for ((attempt = 0; attempt < 50; attempt++)); do + kill -0 "$recorder_pid" 2>/dev/null || break + sleep 0.1 + done + kill -KILL "$recorder_pid" 2>/dev/null || true + wait "$recorder_pid" 2>/dev/null || true + if [[ ! -s "$recording" ]] || ! env -i "PATH=$PATH" \ + ffprobe -v error "$recording" >/dev/null 2>&1; then + echo "Solheim smoke recording failed" + [[ "$result" -ne 0 ]] || result=1 + fi + exit "$result" +} +trap finish EXIT +trap 'exit 130' INT +trap 'exit 143' TERM +sleep 0.5 +kill -0 "$recorder_pid" 2>/dev/null || exit 1 + +# Arguments are used for credential-free recorder tests; CI supplies none. +if [[ "$#" -gt 0 ]]; then "$@"; else pnpm solheim:smoke; fi diff --git a/scripts/solheim-smoke/task-start.test.ts b/scripts/solheim-smoke/task-start.test.ts new file mode 100644 index 0000000000..58ebb91fac --- /dev/null +++ b/scripts/solheim-smoke/task-start.test.ts @@ -0,0 +1,215 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" + +import { RooCodeEventName, TASK_START_FAILURE_ERROR_CODE, TASK_START_FAILURE_ERROR_MESSAGE } from "@roo-code/types" + +import { createTaskStartWaiter } from "./task-start.ts" + +const REQUEST_ID = "final-smoke-0123abcdef456789" + +const startResponse = (payload: unknown): unknown => ({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [payload], +}) + +const successPayload = (requestId = REQUEST_ID): unknown => ({ + requestId, + success: true, + taskId: "accepted-task-1", +}) + +const failurePayload = (requestId = REQUEST_ID): Record => ({ + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, +}) + +const settledOutcome = async (promise: Promise): Promise<"resolved" | "rejected" | "pending"> => { + const probe = promise.then( + () => "resolved" as const, + () => "rejected" as const, + ) + return Promise.race([probe, new Promise<"pending">((resolve) => setTimeout(() => resolve("pending"), 10))]) +} + +describe("createTaskStartWaiter", () => { + it("resolves with the accepted task ID on a matching success response", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 1_000 }) + + waiter.onTaskEvent(startResponse(successPayload())) + + assert.equal(await waiter.result, "accepted-task-1") + }) + + it("rejects with the fixed sanitized failure values on a matching failure response", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 1_000 }) + + waiter.onTaskEvent(startResponse(failurePayload())) + + await assert.rejects(waiter.result, (error: Error) => { + assert.equal(error.message, `${TASK_START_FAILURE_ERROR_CODE}: ${TASK_START_FAILURE_ERROR_MESSAGE}`) + return true + }) + }) + + it("rejects with the stage in the sanitized error on a failure response carrying a stage", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 1_000 }) + + waiter.onTaskEvent( + startResponse({ + ...failurePayload(), + stage: "taskCreation", + }), + ) + + await assert.rejects(waiter.result, (error: Error) => { + assert.equal( + error.message, + `${TASK_START_FAILURE_ERROR_CODE}: ${TASK_START_FAILURE_ERROR_MESSAGE} (stage=taskCreation)`, + ) + return true + }) + }) + + it("rejects with the stage unknown in the sanitized error when the response names it", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 1_000 }) + + waiter.onTaskEvent( + startResponse({ + ...failurePayload(), + stage: "unknown", + }), + ) + + await assert.rejects(waiter.result, (error: Error) => { + assert.match(error.message, /\(stage=unknown\)$/) + return true + }) + }) + + it("ignores a failure response carrying an unknown stage value", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 1_000 }) + + waiter.onTaskEvent( + startResponse({ + ...failurePayload(), + stage: "secretStore", + }), + ) + assert.equal(await settledOutcome(waiter.result), "pending") + + waiter.onTaskEvent(startResponse(failurePayload())) + await assert.rejects(waiter.result, /task_start_failed/) + }) + + it("ignores responses for a different requestId", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 1_000 }) + + waiter.onTaskEvent(startResponse(successPayload("some-other-request"))) + assert.equal(await settledOutcome(waiter.result), "pending") + + waiter.onTaskEvent(startResponse(successPayload())) + assert.equal(await waiter.result, "accepted-task-1") + }) + + it("ignores malformed and non-start events", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 1_000 }) + + waiter.onTaskEvent({ eventName: RooCodeEventName.TaskCreated, payload: ["task-x"] }) + waiter.onTaskEvent(startResponse({ requestId: REQUEST_ID, success: true })) + waiter.onTaskEvent(startResponse("not-an-object")) + waiter.onTaskEvent({ eventName: RooCodeEventName.TaskStartResponse }) + waiter.onTaskEvent(undefined) + assert.equal(await settledOutcome(waiter.result), "pending") + + waiter.onTaskEvent(startResponse(successPayload())) + assert.equal(await waiter.result, "accepted-task-1") + }) + + it("ignores a success response carrying extra top-level fields", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 1_000 }) + + waiter.onTaskEvent( + startResponse({ + requestId: REQUEST_ID, + success: true, + taskId: "accepted-task-1", + providerError: "key sk-secret-123 rejected", + }), + ) + assert.equal(await settledOutcome(waiter.result), "pending") + + waiter.onTaskEvent(startResponse(successPayload())) + assert.equal(await waiter.result, "accepted-task-1") + }) + + it("cannot settle on a failure response carrying extra top-level fields", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 1_000 }) + + waiter.onTaskEvent( + startResponse({ + requestId: REQUEST_ID, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + detail: { apiKey: "sk-secret-123", url: "https://api.example.com" }, + }), + ) + assert.equal(await settledOutcome(waiter.result), "pending") + + waiter.onTaskEvent(startResponse(failurePayload())) + await assert.rejects(waiter.result, /task_start_failed/) + }) + + it("settles only once: later responses cannot change the outcome", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 1_000 }) + + waiter.onTaskEvent(startResponse(failurePayload())) + await assert.rejects(waiter.result, /task_start_failed/) + + assert.equal(waiter.onTaskEvent(startResponse(successPayload())), undefined) + waiter.onDisconnect() + waiter.onExit() + await assert.rejects(waiter.result, /task_start_failed/) + }) + + it("rejects on timeout", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 10 }) + + await assert.rejects(waiter.result, (error: Error) => { + assert.match(error.message, /task start response missing after 10ms/) + assert.match(error.message, new RegExp(REQUEST_ID)) + return true + }) + }) + + it("rejects on IPC disconnect", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 10_000 }) + + waiter.onDisconnect() + + await assert.rejects(waiter.result, /IPC disconnected before the task start response arrived/) + }) + + it("rejects on VS Code exit", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 10_000 }) + + waiter.onExit() + + await assert.rejects(waiter.result, /VS Code exited before the task start response arrived/) + }) + + it("stops the timeout once settled", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 10 }) + + waiter.onTaskEvent(startResponse(successPayload())) + await waiter.result + + // A pending timer must not reject after success. Wait past the + // original deadline; a leak would reject the already-settled promise + // handler chain here. + await new Promise((resolve) => setTimeout(resolve, 30)) + assert.equal(await waiter.result, "accepted-task-1") + }) +}) diff --git a/scripts/solheim-smoke/task-start.ts b/scripts/solheim-smoke/task-start.ts new file mode 100644 index 0000000000..2aafaaea57 --- /dev/null +++ b/scripts/solheim-smoke/task-start.ts @@ -0,0 +1,96 @@ +import { RooCodeEventName, taskStartResponseSchema } from "@roo-code/types" + +export type TaskStartWaiter = { + onTaskEvent: (event: unknown) => string | undefined + onDisconnect: () => void + onExit: () => void + result: Promise +} + +const isRecord = (value: unknown): value is Record => typeof value === "object" && value !== null + +// Correlates one StartNewTask request with its sanitized response, similar to +// JSON-RPC: the sender generates the requestId and the reply copies it back. +// The waiter settles exactly once. A matched success resolves with the +// accepted task ID. A matched failure, a timeout, an IPC disconnect, or a +// VS Code exit rejects. Unmatched or malformed responses never settle it. +export function createTaskStartWaiter({ + requestId, + timeoutMs, +}: { + requestId: string + timeoutMs: number +}): TaskStartWaiter { + let resolveResult!: (taskId: string) => void + let rejectResult!: (error: Error) => void + const result = new Promise((resolve, reject) => { + resolveResult = resolve + rejectResult = reject + }) + + let settled = false + let timer: NodeJS.Timeout | undefined + + const settle = (finish: () => void): void => { + if (settled) { + return + } + settled = true + if (timer !== undefined) { + clearTimeout(timer) + timer = undefined + } + finish() + } + + const rejectWith = (message: string): void => { + settle(() => rejectResult(new Error(message))) + } + + timer = setTimeout(() => { + rejectWith(`task start response missing after ${timeoutMs}ms (requestId=${requestId})`) + }, timeoutMs) + + return { + onTaskEvent(event: unknown): string | undefined { + if (settled) return undefined + if (!isRecord(event) || event["eventName"] !== RooCodeEventName.TaskStartResponse) { + return undefined + } + + const payload = event["payload"] + if (!Array.isArray(payload) || payload.length === 0) { + return undefined + } + + const parsed = taskStartResponseSchema.safeParse(payload[0]) + if (!parsed.success || parsed.data.requestId !== requestId) { + return undefined + } + + const response = parsed.data + if (response.success) { + const acceptedTaskId = response.taskId + settle(() => resolveResult(acceptedTaskId)) + return acceptedTaskId + } + + const { errorCode, errorMessage, stage } = response + // The stage is a fixed sanitized token from the response schema; it + // names the startup phase that stopped responding. + const stageDetail = stage === undefined ? "" : ` (stage=${stage})` + rejectWith(`${errorCode}: ${errorMessage}${stageDetail}`) + return undefined + }, + + onDisconnect(): void { + rejectWith("IPC disconnected before the task start response arrived") + }, + + onExit(): void { + rejectWith("VS Code exited before the task start response arrived") + }, + + result, + } +} diff --git a/scripts/solheim-smoke/tsconfig.json b/scripts/solheim-smoke/tsconfig.json new file mode 100644 index 0000000000..55bddebe10 --- /dev/null +++ b/scripts/solheim-smoke/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "allowImportingTsExtensions": true, + "esModuleInterop": true, + "isolatedModules": true, + "lib": ["ES2022"], + "module": "esnext", + "moduleResolution": "bundler", + "noEmit": true, + "noUncheckedIndexedAccess": true, + "skipLibCheck": true, + "strict": true, + "target": "ES2022", + "types": ["node"] + }, + "include": ["./*.ts", "./*.mts"] +} diff --git a/src/extension/__tests__/api-start-task-logging.spec.ts b/src/extension/__tests__/api-start-task-logging.spec.ts new file mode 100644 index 0000000000..56b331dd0e --- /dev/null +++ b/src/extension/__tests__/api-start-task-logging.spec.ts @@ -0,0 +1,295 @@ +import { afterEach, describe, expect, it, vi } from "vitest" +import * as vscode from "vscode" + +import { + TaskCommandName, + RooCodeEventName, + TASK_START_FAILURE_ERROR_CODE, + TASK_START_FAILURE_ERROR_MESSAGE, + taskStartResponseSchema, + providerIdentifiers, + type RooCodeSettings, +} from "@roo-code/types" + +import { API } from "../api" +import { Package } from "../../shared/package" +import type { ClineProvider } from "../../core/webview/ClineProvider" + +const { commandHandlers, sentMessages } = vi.hoisted(() => ({ + commandHandlers: [] as Array<(clientId: string, command: unknown) => Promise>, + sentMessages: [] as Array<{ clientId: string; message: unknown }>, +})) + +vi.mock("@roo-code/ipc", () => ({ + IpcServer: class { + listen(): void {} + on(_messageType: unknown, handler: (clientId: string, command: unknown) => Promise): void { + commandHandlers.push(handler) + } + send(clientId: string, message: unknown): void { + sentMessages.push({ clientId, message }) + } + }, +})) + +type SentTaskEvent = { data: { eventName: RooCodeEventName; payload: unknown[] } } + +const sentTaskEvents = (): SentTaskEvent[] => sentMessages.map(({ message }) => message as SentTaskEvent) + +const buildStartCommand = (overrides?: { requestId?: string; text?: string }) => ({ + commandName: TaskCommandName.StartNewTask, + data: { + configuration: { + apiProvider: providerIdentifiers.openai, + openAiApiKey: "opaque-solheim-secret-0123456789abcdef", + openAiModelId: "qwen3.8-27b", + openAiBaseUrl: "https://api.solheim.ai/v1", + autoApprovalEnabled: true, + } as RooCodeSettings, + text: overrides?.text ?? "provider connectivity smoke", + requestId: overrides?.requestId, + }, +}) + +function withStageTimeoutOverride(ms: string): () => void { + const previous = process.env.ROO_CODE_TASK_START_STAGE_TIMEOUT_MS + process.env.ROO_CODE_TASK_START_STAGE_TIMEOUT_MS = ms + return () => { + if (previous === undefined) { + delete process.env.ROO_CODE_TASK_START_STAGE_TIMEOUT_MS + } else { + process.env.ROO_CODE_TASK_START_STAGE_TIMEOUT_MS = previous + } + } +} + +function buildApi(createTaskImpl?: () => Promise<{ taskId: string }>, postStateToWebviewImpl?: () => Promise) { + const outputChannel = { appendLine: vi.fn() } as unknown as vscode.OutputChannel + const provider = { + context: {}, + on: vi.fn(), + contextProxy: { setValues: vi.fn().mockResolvedValue(undefined) }, + providerSettingsManager: { saveConfig: vi.fn().mockResolvedValue("default"), setModeConfig: vi.fn() }, + postStateToWebview: postStateToWebviewImpl + ? vi.fn().mockImplementation(postStateToWebviewImpl) + : vi.fn().mockResolvedValue(undefined), + evictCurrentTask: vi.fn().mockResolvedValue(undefined), + postMessageToWebview: vi.fn().mockResolvedValue(undefined), + removeClineFromStack: vi.fn().mockResolvedValue(undefined), + createTask: createTaskImpl + ? vi.fn().mockImplementation(createTaskImpl) + : vi.fn().mockResolvedValue({ taskId: "task-1" }), + } as unknown as ClineProvider + + const consoleSpy = vi.spyOn(console, "log").mockImplementation(() => {}) + const executeCommand = vi.spyOn(vscode.commands, "executeCommand").mockResolvedValue(undefined) + const api = new API(outputChannel, provider, "/tmp/final-smoke-test.sock", true) + + const collectLogs = () => + [ + ...(outputChannel.appendLine as ReturnType).mock.calls.flat().map(String), + ...consoleSpy.mock.calls.flat().map((arg) => JSON.stringify(arg)), + ].join("\n") + + return { api, provider, executeCommand, collectLogs } +} + +describe("API StartNewTask IPC", () => { + afterEach(() => { + commandHandlers.length = 0 + sentMessages.length = 0 + vi.restoreAllMocks() + }) + + it("never logs the API key, base URL, or prompt text from the StartNewTask command", async () => { + const { collectLogs } = buildApi() + + const handler = commandHandlers.at(-1) + expect(handler).toBeDefined() + + await handler!("client-1", buildStartCommand()) + + const logged = collectLogs() + + expect(logged).not.toContain("opaque-solheim-secret-0123456789abcdef") + expect(logged).not.toContain('"openAiApiKey":"') + // Configuration is omitted entirely, even nominally non-secret fields. + expect(logged).not.toContain("openAiModelId") + expect(logged).not.toContain("qwen3.8-27b") + expect(logged).not.toContain("openAiBaseUrl") + expect(logged).not.toContain("openAiApiKey") + // Prompt content is metadata only: length, never text. + expect(logged).not.toContain("provider connectivity smoke") + expect(logged).toContain("promptLength") + }) + + it("omits the sidebar focus for an IPC start, so an unresolved focus command cannot block it", async () => { + const { executeCommand } = buildApi() + + const handler = commandHandlers.at(-1) + await handler!("client-1", buildStartCommand({ requestId: "req-legacy" })) + + expect(executeCommand).not.toHaveBeenCalledWith(`${Package.name}.SidebarProvider.focus`) + }) + + it("answers a correlated start with exactly one sanitized success response", async () => { + buildApi() + + const handler = commandHandlers.at(-1) + await handler!("client-1", buildStartCommand({ requestId: "req-1" })) + + expect(sentMessages).toHaveLength(1) + const events = sentTaskEvents() + expect(events[0]?.data.eventName).toBe(RooCodeEventName.TaskStartResponse) + + const parsed = taskStartResponseSchema.safeParse(events[0]?.data.payload[0]) + expect(parsed.success).toBe(true) + if (parsed.success) { + expect(parsed.data).toEqual({ requestId: "req-1", success: true, taskId: "task-1" }) + } + }) + + it("answers a failed correlated start with exactly one fixed safe failure response", async () => { + const rawError = new Error( + "openai rejected key opaque-solheim-secret-0123456789abcdef at https://api.solheim.ai/v1 for prompt review the pull request diff", + ) + buildApi(() => Promise.reject(rawError)) + + const handler = commandHandlers.at(-1) + await handler!("client-1", buildStartCommand({ requestId: "req-2" })) + + expect(sentMessages).toHaveLength(1) + const events = sentTaskEvents() + expect(events[0]?.data.eventName).toBe(RooCodeEventName.TaskStartResponse) + + const parsed = taskStartResponseSchema.safeParse(events[0]?.data.payload[0]) + expect(parsed.success).toBe(true) + if (parsed.success && !parsed.data.success) { + expect(parsed.data.errorCode).toBe(TASK_START_FAILURE_ERROR_CODE) + expect(parsed.data.errorMessage).toBe(TASK_START_FAILURE_ERROR_MESSAGE) + expect(parsed.data.stage).toBe("unknown") + } + + const sent = JSON.stringify(sentMessages) + expect(sent).not.toContain("opaque-solheim-secret-0123456789abcdef") + expect(sent).not.toContain("api.solheim.ai") + expect(sent).not.toContain("review the pull request diff") + }) + + it("bounds a hung task creation stage and answers with exactly one sanitized stage failure", async () => { + const restore = withStageTimeoutOverride("100") + try { + const { collectLogs } = buildApi(() => new Promise(() => {})) + + const handler = commandHandlers.at(-1) + await handler!("client-1", buildStartCommand({ requestId: "req-timeout-task" })) + + expect(sentMessages).toHaveLength(1) + const events = sentTaskEvents() + expect(events[0]?.data.eventName).toBe(RooCodeEventName.TaskStartResponse) + + const parsed = taskStartResponseSchema.safeParse(events[0]?.data.payload[0]) + expect(parsed.success).toBe(true) + if (parsed.success && !parsed.data.success) { + expect(parsed.data).toEqual({ + requestId: "req-timeout-task", + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + stage: "taskCreation", + }) + } + + const logged = collectLogs() + expect(logged).toContain("taskCreation") + expect(logged).toContain("req-timeout-task") + expect(logged).not.toContain("opaque-solheim-secret-0123456789abcdef") + } finally { + restore() + } + }) + + it("removes a late task and refuses a retry until the timed-out creation settles", async () => { + const restore = withStageTimeoutOverride("50") + try { + let finishCreation: (task: { taskId: string }) => void = () => {} + const { provider } = buildApi( + () => + new Promise((resolve) => { + finishCreation = resolve + }), + ) + const handler = commandHandlers.at(-1)! + + await handler("client-1", buildStartCommand({ requestId: "req-first" })) + await handler("client-1", buildStartCommand({ requestId: "req-retry" })) + + expect(provider.createTask).toHaveBeenCalledTimes(1) + const retry = taskStartResponseSchema.parse(sentTaskEvents()[1]?.data.payload[0]) + expect(retry.success).toBe(false) + + finishCreation({ taskId: "late-task" }) + await vi.waitFor(() => expect(provider.removeClineFromStack).toHaveBeenCalledTimes(1)) + + await handler("client-1", buildStartCommand({ requestId: "req-after" })) + expect(provider.createTask).toHaveBeenCalledTimes(2) + } finally { + restore() + } + }) + + it("bounds a hung settings stage and names it in the failure response", async () => { + const restore = withStageTimeoutOverride("100") + try { + buildApi(undefined, () => new Promise(() => {})) + + const handler = commandHandlers.at(-1) + await handler!("client-1", buildStartCommand({ requestId: "req-timeout-settings" })) + + expect(sentMessages).toHaveLength(1) + const parsed = taskStartResponseSchema.safeParse(sentTaskEvents()[0]?.data.payload[0]) + expect(parsed.success).toBe(true) + if (parsed.success && !parsed.data.success) { + expect(parsed.data.stage).toBe("settings") + } + } finally { + restore() + } + }) + + it("still answers with one success response when every stage is fast under a small bound", async () => { + const restore = withStageTimeoutOverride("100") + try { + buildApi() + + const handler = commandHandlers.at(-1) + await handler!("client-1", buildStartCommand({ requestId: "req-fast" })) + + expect(sentMessages).toHaveLength(1) + const parsed = taskStartResponseSchema.safeParse(sentTaskEvents()[0]?.data.payload[0]) + expect(parsed.success).toBe(true) + if (parsed.success) { + expect(parsed.data).toEqual({ requestId: "req-fast", success: true, taskId: "task-1" }) + } + } finally { + restore() + } + }) + + it("sends no start response for legacy IPC starts without requestId", async () => { + buildApi() + + const handler = commandHandlers.at(-1) + await handler!("client-1", buildStartCommand()) + + expect(sentMessages).toHaveLength(0) + }) + + it("keeps the sidebar focus for direct API callers that do not opt out", async () => { + const { api, executeCommand } = buildApi() + + await api.startNewTask({ configuration: {} as RooCodeSettings, text: "direct caller" }) + + expect(executeCommand).toHaveBeenCalledWith(`${Package.name}.SidebarProvider.focus`) + }) +}) diff --git a/src/extension/api.ts b/src/extension/api.ts index 316e7a6c9d..711e03c5b2 100644 --- a/src/extension/api.ts +++ b/src/extension/api.ts @@ -13,6 +13,8 @@ import { type ProviderSettings, type ProviderSettingsEntry, type TaskEvent, + type TaskStartResponse, + type TaskStartStage, type CreateTaskOptions, type TaskApiConversationHistorySequence, type WebviewThemeFixture, @@ -21,7 +23,10 @@ import { isSecretStateKey, IpcOrigin, IpcMessageType, + TASK_START_FAILURE_ERROR_CODE, + TASK_START_FAILURE_ERROR_MESSAGE, } from "@roo-code/types" + import { IpcServer } from "@roo-code/ipc" import { Package } from "../shared/package" @@ -33,9 +38,51 @@ import { openClineInNewTab } from "../activate/registerCommands" import { getCommands } from "../services/command/commands" import { getModels } from "../api/providers/fetchers/modelCache" +// The default bound is per stage, not per start, so one slow stage cannot +// hide behind the budget of another. Secret storage writes through VS Code +// secret storage can hang indefinitely under Xvfb without a keyring service. +const DEFAULT_TASK_START_STAGE_TIMEOUT_MS = 60_000 + +function readTaskStartStageTimeoutMs(env: NodeJS.ProcessEnv): number { + const parsed = Number(env.ROO_CODE_TASK_START_STAGE_TIMEOUT_MS) + return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_TASK_START_STAGE_TIMEOUT_MS +} + +class TaskStartStageTimeoutError extends Error { + constructor(readonly stage: TaskStartStage) { + super(`task start stage timed out: ${stage}`) + } +} + +// Races the stage against its bound. Losing the race stops the wait only. +// The operation keeps running, so onAbandon receives it for later cleanup. +async function withStageTimeout( + stage: TaskStartStage, + timeoutMs: number, + operation: () => T | Thenable, + onAbandon?: (running: Promise) => void, +): Promise { + let timer: NodeJS.Timeout | undefined + const running = Promise.resolve(operation()) + try { + return await Promise.race([ + running, + new Promise((_, reject) => { + timer = setTimeout(() => { + onAbandon?.(running) + reject(new TaskStartStageTimeoutError(stage)) + }, timeoutMs) + }), + ]) + } finally { + clearTimeout(timer) + } +} + export class API extends EventEmitter implements RooCodeAPI { private readonly outputChannel: vscode.OutputChannel private readonly sidebarProvider: ClineProvider + private abandonedStarts = 0 private readonly context: vscode.ExtensionContext private readonly ipc?: IpcServer private readonly log: (...args: unknown[]) => void @@ -82,12 +129,49 @@ export class API extends EventEmitter implements RooCodeAPI { } switch (command.commandName) { - case TaskCommandName.StartNewTask: - this.log( - `[API] StartNewTask -> ${command.data.text}, ${JSON.stringify(command.data.configuration)}`, - ) - await this.startNewTask(command.data) + case TaskCommandName.StartNewTask: { + // Neither prompt nor settings values enter the log. + this.log("[API] StartNewTask", { + promptLength: command.data.text.length, + imageCount: command.data.images?.length ?? 0, + newTab: command.data.newTab === true, + requestId: command.data.requestId, + }) + + const { requestId } = command.data + + if (requestId === undefined) { + await this.startNewTask(command.data, { focusSidebar: false }) + break + } + + // Exactly one sanitized, client-scoped response per + // correlated start. The reply never echoes the failure. + // Each startup stage is bounded so a hung await (for + // example secret storage under a missing keyring) + // still produces the reply. + let response: TaskStartResponse + try { + const taskId = await this.startNewTask(command.data, { + focusSidebar: false, + stageTimeoutMs: readTaskStartStageTimeoutMs(process.env), + }) + response = { requestId, success: true, taskId } + } catch (error) { + const stage: TaskStartStage = + error instanceof TaskStartStageTimeoutError ? error.stage : "unknown" + this.log("[API] StartNewTask failed", { requestId, stage }) + response = { + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + stage, + } + } + sendResponse(RooCodeEventName.TaskStartResponse, [response]) break + } case TaskCommandName.CancelTask: this.log(`[API] CancelTask`) await this.cancelCurrentTask() @@ -170,41 +254,85 @@ export class API extends EventEmitter implements RooCodeAPI { return super.emit(eventName, ...args) } - public async startNewTask({ - configuration, - text, - images, - newTab, - }: { - configuration: RooCodeSettings - text?: string - images?: string[] - newTab?: boolean - }) { + public async startNewTask( + { + configuration, + text, + images, + newTab, + }: { + configuration: RooCodeSettings + text?: string + images?: string[] + newTab?: boolean + }, + startOptions?: { focusSidebar?: boolean; stageTimeoutMs?: number }, + ) { + // Stage names for a bounded correlated start. The settings persistence + // named in the startup evidence (createTask -> setValues -> + // storeSecret) runs inside the taskCreation stage at this layer, so a + // taskCreation timeout covers it. + // A timed-out stage keeps running. Count it, and reject a new bounded + // start until it settles. A late task from the taskCreation stage is + // removed, so it cannot run after the failure reply. + if (startOptions?.stageTimeoutMs !== undefined && this.abandonedStarts > 0) { + throw new Error("an earlier task start is still running") + } + const boundStage = ( + stage: TaskStartStage, + operation: () => T | Thenable, + onLate?: (late: Awaited) => unknown, + ): Promise => + startOptions?.stageTimeoutMs === undefined + ? Promise.resolve(operation()) + : withStageTimeout(stage, startOptions.stageTimeoutMs, operation, (running) => { + this.abandonedStarts++ + void running + .then((late) => onLate?.(late as Awaited)) + .catch(() => {}) + .finally(() => this.abandonedStarts--) + }) + let provider: ClineProvider if (newTab) { - await vscode.commands.executeCommand("workbench.action.files.revert") - await vscode.commands.executeCommand("workbench.action.closeAllEditors") + await boundStage("unknown", () => vscode.commands.executeCommand("workbench.action.files.revert")) + await boundStage("unknown", () => vscode.commands.executeCommand("workbench.action.closeAllEditors")) - provider = await openClineInNewTab({ context: this.context, outputChannel: this.outputChannel }) + provider = await boundStage("unknown", () => + openClineInNewTab({ context: this.context, outputChannel: this.outputChannel }), + ) this.registerListeners(provider) } else { - await vscode.commands.executeCommand(`${Package.name}.SidebarProvider.focus`) + // IPC callers omit focus: executeCommand cannot be cancelled and a + // stalled webview launch would block the whole start. + if (startOptions?.focusSidebar ?? true) { + await boundStage("unknown", () => + vscode.commands.executeCommand(`${Package.name}.SidebarProvider.focus`), + ) + } provider = this.sidebarProvider } - await provider.evictCurrentTask() - await provider.postStateToWebview() - await provider.postMessageToWebview({ type: "action", action: "chatButtonClicked" }) - await provider.postMessageToWebview({ type: "invoke", invoke: "newChat", text, images }) + await boundStage("eviction", () => provider.evictCurrentTask()) + await boundStage("settings", () => provider.postStateToWebview()) + await boundStage("settings", () => + provider.postMessageToWebview({ type: "action", action: "chatButtonClicked" }), + ) + await boundStage("settings", () => + provider.postMessageToWebview({ type: "invoke", invoke: "newChat", text, images }), + ) const options: CreateTaskOptions = { consecutiveMistakeLimit: Number.MAX_SAFE_INTEGER, } - const task = await provider.createTask(text, images, undefined, options, configuration) + const task = await boundStage( + "taskCreation", + () => provider.createTask(text, images, undefined, options, configuration), + (late) => late && provider.removeClineFromStack(), + ) if (!task) { throw new Error("Failed to create task due to policy restrictions") From 2db82ef27757ead8a2717866858dde935e55d353 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Tue, 6 Oct 2026 01:08:18 +0000 Subject: [PATCH 2/2] fix(smoke): keep startup timeouts in isolated controller --- packages/ipc/package.json | 1 + packages/ipc/src/__tests__/ipc-server.test.ts | 65 +++++ packages/ipc/src/ipc-server.ts | 9 +- scripts/solheim-smoke-workflow.test.mjs | 1 + scripts/solheim-smoke/README.md | 8 +- scripts/solheim-smoke/config.ts | 1 - scripts/solheim-smoke/driver.mts | 262 +++++++----------- scripts/solheim-smoke/run.test.ts | 245 ++++++++++++++++ scripts/solheim-smoke/run.ts | 167 +++++++++++ .../__tests__/api-start-task-logging.spec.ts | 180 ++++-------- src/extension/api.ts | 114 +------- 11 files changed, 664 insertions(+), 389 deletions(-) create mode 100644 packages/ipc/src/__tests__/ipc-server.test.ts create mode 100644 scripts/solheim-smoke/run.test.ts create mode 100644 scripts/solheim-smoke/run.ts diff --git a/packages/ipc/package.json b/packages/ipc/package.json index 374c4240b6..4dd505bb18 100644 --- a/packages/ipc/package.json +++ b/packages/ipc/package.json @@ -5,6 +5,7 @@ "type": "module", "exports": "./src/index.ts", "scripts": { + "test": "node --import tsx --test src/__tests__/*.test.ts", "lint": "eslint src --ext=ts --max-warnings=0", "check-types": "tsc --noEmit", "clean": "rimraf .turbo" diff --git a/packages/ipc/src/__tests__/ipc-server.test.ts b/packages/ipc/src/__tests__/ipc-server.test.ts new file mode 100644 index 0000000000..3419ebf982 --- /dev/null +++ b/packages/ipc/src/__tests__/ipc-server.test.ts @@ -0,0 +1,65 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { IpcMessageType, IpcOrigin, TaskCommandName, ipcMessageSchema } from "@roo-code/types" +import { IpcServer } from "../ipc-server.js" + +function fixture() { + const logs: unknown[][] = [] + const commands: unknown[] = [] + const server = new IpcServer("/unused-test.sock", (...args) => logs.push(args)) + server.on(IpcMessageType.TaskCommand, (clientId, command) => commands.push({ clientId, command })) + return { logs, commands, receive: (data: unknown) => server["onMessage"](data) } +} +const key = "dummy-api-key-not-for-public-logging" +const command = (requestId: string) => ({ + type: IpcMessageType.TaskCommand, + origin: IpcOrigin.Client, + clientId: "client-1", + data: { + commandName: TaskCommandName.StartNewTask, + data: { requestId, text: "private prompt", configuration: { apiProvider: "openai", openAiApiKey: key } }, + }, +}) + +describe("IPC rejection logging", () => { + it("rejects an invalid request ID without logging its key, prompt or schema issue values", () => { + const f = fixture() + f.receive(command(`invalid request containing ${key}`)) + assert.deepEqual(f.commands, []) + assert.equal(f.logs.length, 1) + assert.equal(f.logs[0]?.[0], "[server#onMessage] invalid payload") + assert.ok(!JSON.stringify(f.logs).includes(key)) + assert.ok(!JSON.stringify(f.logs).includes("private prompt")) + assert.ok(!JSON.stringify(f.logs).includes("invalid request containing")) + }) + it("does not log credential-shaped primitive input or dynamic issue paths", () => { + const f = fixture() + const valid = command("valid-id") + const malformedHeaders = { + ...valid, + data: { + ...valid.data, + data: { + ...valid.data.data, + configuration: { ...valid.data.data.configuration, openAiHeaders: { [key]: null } }, + }, + }, + } + const rejected = ipcMessageSchema.safeParse(malformedHeaders) + assert.ok(!rejected.success) + assert.ok(rejected.error.issues.some((issue) => issue.path.includes(key))) + for (const data of [key, null, { type: key }, malformedHeaders]) { + f.receive(data) + } + assert.equal(f.logs.length, 4) + assert.ok(!JSON.stringify(f.logs).includes(key)) + assert.deepEqual(f.commands, []) + }) + it("still dispatches a valid correlated command to its client without logging its settings", () => { + const f = fixture() + f.receive(command("valid-id")) + assert.equal(f.commands.length, 1) + assert.deepEqual(f.commands[0], { clientId: "client-1", command: command("valid-id").data }) + assert.deepEqual(f.logs, []) + }) +}) diff --git a/packages/ipc/src/ipc-server.ts b/packages/ipc/src/ipc-server.ts index 200e241014..26dd626a9e 100644 --- a/packages/ipc/src/ipc-server.ts +++ b/packages/ipc/src/ipc-server.ts @@ -76,16 +76,15 @@ export class IpcServer extends EventEmitter implements RooCodeI private onMessage(data: unknown) { if (typeof data !== "object") { - this.log(`[server#onMessage] invalid data -> ${JSON.stringify(data)}`) + this.log("[server#onMessage] invalid data") return } const result = ipcMessageSchema.safeParse(data) if (!result.success) { - this.log( - `[server#onMessage] invalid payload -> ${JSON.stringify(result.error.issues)} -> ${JSON.stringify(data)}`, - ) + // Both the input and Zod issue values/paths may contain credentials. + this.log("[server#onMessage] invalid payload", { issueCount: result.error.issues.length }) return } @@ -98,7 +97,7 @@ export class IpcServer extends EventEmitter implements RooCodeI this.emit(IpcMessageType.TaskCommand, payload.clientId, payload.data) break default: - this.log(`[server#onMessage] unhandled payload: ${JSON.stringify(payload)}`) + this.log("[server#onMessage] unhandled payload") break } } diff --git a/scripts/solheim-smoke-workflow.test.mjs b/scripts/solheim-smoke-workflow.test.mjs index 1239cfe5a5..0bab69c438 100644 --- a/scripts/solheim-smoke-workflow.test.mjs +++ b/scripts/solheim-smoke-workflow.test.mjs @@ -56,6 +56,7 @@ describe("Project A provider smoke workflow", () => { it("bounds runtime and serializes the single provider instance", () => { assert.match(workflow, /cancel-in-progress: false/) assert.match(workflow, /timeout-minutes: 8/) + assert.match(workflow, /^ {8}timeout-minutes: 25$/m) }) it("uploads only the verdict and video, never host logs or storage", () => { const paths = workflow.split("path: |\n")[1]?.split(" if-no-files-found:")[0] diff --git a/scripts/solheim-smoke/README.md b/scripts/solheim-smoke/README.md index 35e7189138..219ce3341e 100644 --- a/scripts/solheim-smoke/README.md +++ b/scripts/solheim-smoke/README.md @@ -18,11 +18,15 @@ The provider endpoint and model are fixed: `https://api.solheim.ai/v1`, `qwen3.8 groups; delegation, follow-ups, writes, shell and MCP are disabled. The driver opens an empty temporary workspace with isolated VS Code storage and an allowlisted host environment. The provider credential is supplied over IPC, not inherited by the child or passed on its command line. Temporary extension storage -can contain the configured credential while the session runs; it is removed at teardown and never uploaded. +can contain the configured credential while the session runs. It is removed when no child remains alive; +if termination fails, storage cleanup is skipped and `gates.childExited` reports that separately from the +original smoke failure. Storage is never uploaded. This is process/environment isolation, not a sandbox for malicious extension code—CI runs trusted main only. Activation and completion each have a 120-second limit; task-start acknowledgment has a 35-second limit, -with 30-second per-stage bounds in the extension. The live CI step has an eight-minute outer timeout. +in the controller. The extension awaits startup and configuration persistence without racing shared-profile +writes against a timer. If the controller times out, it tears down its isolated VS Code process rather than +trying to roll back shared settings or remove a possibly unrelated task. The live CI step has an eight-minute outer timeout. Teardown escalates through close, terminate, and kill within independent five-second windows. A non-empty completion and actual output-token usage must belong to the accepted task. Activation alone cannot pass. The driver never approves dialogs or tool execution; it closes the task during teardown. diff --git a/scripts/solheim-smoke/config.ts b/scripts/solheim-smoke/config.ts index 69c5d6aa71..7b19187cdc 100644 --- a/scripts/solheim-smoke/config.ts +++ b/scripts/solheim-smoke/config.ts @@ -64,7 +64,6 @@ export function buildChildEnvironment( XDG_CONFIG_HOME: path.join(isolatedHome, ".config"), XDG_CACHE_HOME: path.join(isolatedHome, ".cache"), ROO_CODE_IPC_SOCKET_PATH: socketPath, - ROO_CODE_TASK_START_STAGE_TIMEOUT_MS: "30000", } } diff --git a/scripts/solheim-smoke/driver.mts b/scripts/solheim-smoke/driver.mts index 2a44a262c7..383e2db206 100644 --- a/scripts/solheim-smoke/driver.mts +++ b/scripts/solheim-smoke/driver.mts @@ -1,181 +1,121 @@ import fs from "node:fs/promises" import os from "node:os" import path from "node:path" -import { randomUUID } from "node:crypto" import { fileURLToPath } from "node:url" -import { spawn, type ChildProcess } from "node:child_process" +import { spawn } from "node:child_process" import { downloadAndUnzipVSCode } from "@vscode/test-electron" import { IpcClient } from "@roo-code/ipc" -import { IpcMessageType, TaskCommandName, type TaskEvent } from "@roo-code/types" -import { - buildChildEnvironment, - buildLaunchArgs, - buildSmokeConfiguration, - SMOKE_PROMPT, - SMOKE_TIMEOUT_MS, - SOLHEIM_BASE_URL, - SOLHEIM_MODEL, -} from "./config.ts" -import { collectEvidence, smokeCode, type SmokeCode, type SmokeEvidence } from "./evidence.ts" -import { createTaskStartWaiter } from "./task-start.ts" +import { IpcMessageType } from "@roo-code/types" +import { buildChildEnvironment, buildLaunchArgs } from "./config.ts" +import { runSmoke, smokeExitCode, type SmokeRuntime } from "./run.ts" const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") const out = process.env.SOLHEIM_SMOKE_OUT_DIR if (!out) throw new Error("Set SOLHEIM_SMOKE_OUT_DIR to a scratch directory") await fs.mkdir(out, { recursive: true }) -const started = Date.now() -const evidence: SmokeEvidence = { - providerResponseSeen: false, - completionSeen: false, - unexpectedTool: false, - aborted: false, -} -let code: SmokeCode = "SETUP_FAILED" -let activated = false -let accepted = false -let sessionLost = false -let child: ChildProcess | undefined -let client: IpcClient | undefined let temp: string | undefined -let exited = false -let waiter: ReturnType | undefined -const pause = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) -async function until(check: () => boolean, timeoutMs: number, stopOnSessionLoss = true): Promise { - const deadline = Date.now() + timeoutMs - while (!check() && (!stopOnSessionLoss || !sessionLost) && Date.now() < deadline) await pause(100) - return check() -} -function signalChild(signal: NodeJS.Signals): void { - if (!child?.pid || exited) return - try { - process.kill(-child.pid, signal) - } catch { - child.kill(signal) - } -} -function interrupted(): void { - sessionLost = true - waiter?.onExit() - signalChild("SIGTERM") -} -process.once("SIGINT", interrupted) -process.once("SIGTERM", interrupted) - -try { - const apiKey = process.env.SOLHEIM_API_KEY - if (!apiKey) throw new Error("Provider credential missing") - const extension = path.join(root, "src") - await fs.access(path.join(extension, "package.json")) - temp = await fs.mkdtemp(path.join(os.tmpdir(), "solheim-smoke-")) - const workspace = path.join(temp, "workspace") - const home = path.join(temp, "home") - await Promise.all([workspace, home].map((folder) => fs.mkdir(folder))) - const socket = path.join(temp, "ipc.sock") - const pkg = JSON.parse(await fs.readFile(path.join(root, "apps/vscode-e2e/package.json"), "utf8")) as { - devDependencies: Record - } - const executable = await downloadAndUnzipVSCode({ - version: pkg.devDependencies["@types/vscode"] ?? "1.100.0", - cachePath: path.join(root, "apps/vscode-e2e/.vscode-test"), - }) - const args = buildLaunchArgs(workspace, path.join(temp, "userdata"), path.join(temp, "extensions"), extension) - child = process.env.DISPLAY - ? spawn(executable, args, { - detached: true, - env: buildChildEnvironment(process.env, home, socket), - stdio: "ignore", - }) - : spawn("xvfb-run", ["-a", executable, ...args], { +let socket = "" +const runtime: SmokeRuntime = { + async launch() { + const extension = path.join(root, "src") + await fs.access(path.join(extension, "package.json")) + temp = await fs.mkdtemp(path.join(os.tmpdir(), "solheim-smoke-")) + const workspace = path.join(temp, "workspace") + const isolatedHome = path.join(temp, "home") + await Promise.all([workspace, isolatedHome].map((folder) => fs.mkdir(folder))) + socket = path.join(temp, "ipc.sock") + const pkg = JSON.parse(await fs.readFile(path.join(root, "apps/vscode-e2e/package.json"), "utf8")) as { + devDependencies: Record + } + const executable = await downloadAndUnzipVSCode({ + version: pkg.devDependencies["@types/vscode"] ?? "1.100.0", + cachePath: path.join(root, "apps/vscode-e2e/.vscode-test"), + }) + const args = buildLaunchArgs(workspace, path.join(temp, "userdata"), path.join(temp, "extensions"), extension) + const child = spawn( + process.env.DISPLAY ? executable : "xvfb-run", + process.env.DISPLAY ? args : ["-a", executable, ...args], + { detached: true, - env: buildChildEnvironment(process.env, home, socket), + env: buildChildEnvironment(process.env, isolatedHome, socket), stdio: "ignore", - }) - child.once("error", () => { - sessionLost = true - exited = true - waiter?.onExit() - }) - child.once("close", () => { - sessionLost = true - exited = true - waiter?.onExit() - }) - client = new IpcClient(socket, () => {}) - client.on(IpcMessageType.Disconnect, () => { - sessionLost = true - waiter?.onDisconnect() - }) - let taskId: string | undefined - client.on(IpcMessageType.TaskEvent, (event: TaskEvent) => { - const id = waiter?.onTaskEvent(event) - if (id && taskId === undefined) taskId = id - collectEvidence(evidence, event, taskId) - }) - activated = await until(() => client?.isReady === true, SMOKE_TIMEOUT_MS) - if (!activated) code = "ACTIVATION_FAILED" - else { - const requestId = randomUUID() - waiter = createTaskStartWaiter({ requestId, timeoutMs: 35_000 }) - client.sendCommand({ - commandName: TaskCommandName.StartNewTask, - data: { requestId, text: SMOKE_PROMPT, newTab: false, configuration: buildSmokeConfiguration(apiKey) }, - }) - try { - taskId = await waiter.result - accepted = true - } catch { - code = "TASK_START_FAILED" + }, + ) + let exited = false + let onExit = () => {} + const closed = () => { + exited = true + onExit() } - if (accepted) { - await until( - () => - evidence.unexpectedTool || - evidence.aborted || - (evidence.providerResponseSeen && evidence.completionSeen), - SMOKE_TIMEOUT_MS, - ) - code = smokeCode(evidence, sessionLost) + child.once("error", () => { + // Spawn failure has no live PID; a failed kill must not claim exit. + if (!child.pid) closed() + else onExit() + }) + child.once("close", closed) + return { + get exited() { + return exited + }, + onExit(listener) { + onExit = listener + if (exited) listener() + }, + signal(signal) { + if (!child.pid || exited) return + try { + process.kill(-child.pid, signal) + } catch { + child.kill(signal) + } + }, } - } -} catch { - /* Keep failures fixed and credential-free, never print thrown provider/host text. */ -} finally { - // Teardown is independently bounded, including spawn/activation failures. - if (client?.isReady) client.sendCommand({ commandName: TaskCommandName.CloseTask }) - if (child && !exited) { - await until(() => exited, 5_000, false) - if (!exited) { - signalChild("SIGTERM") - await until(() => exited, 5_000, false) + }, + connect() { + const client = new IpcClient(socket, () => {}) + return { + get isReady() { + return client.isReady + }, + onTaskEvent(listener) { + client.on(IpcMessageType.TaskEvent, listener) + }, + onDisconnect(listener) { + client.on(IpcMessageType.Disconnect, listener) + }, + sendCommand(command) { + client.sendCommand(command) + }, + disconnect() { + client.disconnect() + }, } - if (!exited) { - signalChild("SIGKILL") - await until(() => exited, 5_000, false) + }, + async cleanup() { + if (temp) await fs.rm(temp, { recursive: true, force: true }) + }, + async writeVerdict(verdict) { + // Atomic, whitelisted metadata only; no provider text or host logs. + const pending = path.join(out, `verdict.json.${process.pid}.tmp`) + await fs.writeFile(pending, JSON.stringify(verdict, null, 2)) + await fs.rename(pending, path.join(out, "verdict.json")) + }, + onInterrupt(listener) { + process.once("SIGINT", listener) + process.once("SIGTERM", listener) + return () => { + process.removeListener("SIGINT", listener) + process.removeListener("SIGTERM", listener) } - if (!exited && code === "OK") code = "TEARDOWN_FAILED" - } - client?.disconnect() - if (temp && (!child || exited)) await fs.rm(temp, { recursive: true, force: true }) -} -// Whitelisted metadata only: model answers and host logs are never saved or uploaded. -const verdict = { - kind: "provider-smoke", - model: SOLHEIM_MODEL, - provider: SOLHEIM_BASE_URL, - passed: code === "OK", - code, - gates: { - extensionActivated: activated, - ipcConnected: activated, - taskAccepted: accepted, - providerResponseSeen: evidence.providerResponseSeen, - completionSeen: evidence.completionSeen, }, - durationMs: Date.now() - started, + now: Date.now, + pause: (ms) => new Promise((resolve) => setTimeout(resolve, ms)), +} +try { + const verdict = await runSmoke(runtime, process.env.SOLHEIM_API_KEY ?? "") + console.log(`Solheim provider smoke: ${verdict.code}`) + process.exit(smokeExitCode(verdict)) +} catch { + console.log("Solheim provider smoke: SETUP_FAILED") + process.exit(1) } -// Write then rename, so a reader sees the old or the new verdict and never a partial one. -const verdictTemp = path.join(out, `verdict.json.${process.pid}.tmp`) -await fs.writeFile(verdictTemp, JSON.stringify(verdict, null, 2)) -await fs.rename(verdictTemp, path.join(out, "verdict.json")) -console.log(`Solheim provider smoke: ${code}`) -process.exit(code === "OK" ? 0 : 1) diff --git a/scripts/solheim-smoke/run.test.ts b/scripts/solheim-smoke/run.test.ts new file mode 100644 index 0000000000..1c64ec53e4 --- /dev/null +++ b/scripts/solheim-smoke/run.test.ts @@ -0,0 +1,245 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { RooCodeEventName, TaskCommandName, type TaskCommand } from "@roo-code/types" +import { runSmoke, smokeExitCode, type SmokeRuntime, type SmokeVerdict } from "./run.ts" + +type Scenario = + | "success" + | "activation" + | "start-failure" + | "no-provider" + | "no-completion" + | "tool" + | "disconnect" + | "uncorrelated" + | "aborted" + | "setup" +function fixture( + scenario: Scenario = "success", + options: { stuck?: boolean; closeThrows?: boolean; timeout?: () => void } = {}, +) { + let now = 0 + let exited = false + let onExit = () => {} + let onDisconnect = () => {} + let onEvent: (event: unknown) => void = () => {} + let interrupt = () => {} + let cleaned = false + let disconnected = false + let unsubscribed = false + const commands: TaskCommand[] = [] + const signals: NodeJS.Signals[] = [] + const written: SmokeVerdict[] = [] + const close = () => { + exited = true + onExit() + } + const runtime: SmokeRuntime = { + async launch() { + if (scenario === "setup") throw new Error("private setup detail") + return { + get exited() { + return exited + }, + onExit(listener) { + onExit = listener + }, + signal(signal) { + signals.push(signal) + if (!options.stuck) close() + }, + } + }, + connect() { + return { + isReady: scenario !== "activation", + onTaskEvent(listener) { + onEvent = listener + }, + onDisconnect(listener) { + onDisconnect = listener + }, + disconnect() { + disconnected = true + }, + sendCommand(command) { + commands.push(command) + if (command.commandName === TaskCommandName.CloseTask) { + if (options.closeThrows) throw new Error("private close error") + if (!options.stuck) close() + return + } + if (command.commandName !== TaskCommandName.StartNewTask) return + if (options.timeout) { + options.timeout() + return + } + onEvent({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [ + scenario === "start-failure" + ? { + requestId: command.data.requestId, + success: false, + errorCode: "task_start_failed", + errorMessage: "Task start failed", + } + : { requestId: command.data.requestId, success: true, taskId: "root" }, + ], + }) + const taskId = scenario === "uncorrelated" ? "other" : "root" + if (scenario !== "no-provider") + onEvent({ + eventName: RooCodeEventName.TaskTokenUsageUpdated, + payload: [ + taskId, + { totalTokensIn: 1, totalTokensOut: 5, totalCost: 0, contextTokens: 1 }, + {}, + ], + }) + if (scenario !== "no-completion") + onEvent({ + eventName: RooCodeEventName.Message, + payload: [ + { + taskId, + action: "created", + message: { + ts: 1, + type: "say", + say: "completion_result", + text: "private synthetic answer", + }, + }, + ], + }) + if (scenario === "tool") + onEvent({ + eventName: RooCodeEventName.TaskSpawned, + payload: ["root", "child"], + }) + if (scenario === "disconnect") onDisconnect() + if (scenario === "aborted") onEvent({ eventName: RooCodeEventName.TaskAborted, payload: ["root"] }) + }, + } + }, + async cleanup() { + cleaned = true + }, + async writeVerdict(verdict) { + written.push(verdict) + }, + onInterrupt(listener) { + interrupt = listener + return () => { + unsubscribed = true + } + }, + now: () => now, + async pause(ms) { + now += ms + }, + } + return { + runtime, + commands, + signals, + written, + interrupt: () => interrupt(), + get cleaned() { + return cleaned + }, + get disconnected() { + return disconnected + }, + get unsubscribed() { + return unsubscribed + }, + } +} + +describe("provider smoke orchestration", () => { + it("correlates startup and evidence, writes a metadata-only verdict, and exits zero", async () => { + const f = fixture() + const verdict = await runSmoke(f.runtime, "private-test-key") + assert.equal(verdict.code, "OK") + assert.equal(smokeExitCode(verdict), 0) + assert.deepEqual(verdict.gates, { + extensionActivated: true, + ipcConnected: true, + taskAccepted: true, + providerResponseSeen: true, + completionSeen: true, + childExited: true, + }) + assert.deepEqual(f.written, [verdict]) + assert.ok(f.cleaned && f.disconnected && f.unsubscribed) + assert.equal(f.commands.at(-1)?.commandName, TaskCommandName.CloseTask) + assert.ok(!JSON.stringify(verdict).includes("private")) + }) + for (const [scenario, code] of [ + ["activation", "ACTIVATION_FAILED"], + ["start-failure", "TASK_START_FAILED"], + ["no-provider", "PROVIDER_TIMEOUT"], + ["no-completion", "COMPLETION_MISSING"], + ["tool", "UNEXPECTED_TOOL"], + ["disconnect", "SESSION_LOST"], + ["uncorrelated", "PROVIDER_TIMEOUT"], + ["aborted", "SESSION_LOST"], + ["setup", "SETUP_FAILED"], + ] as const) { + it(`fails closed for ${scenario} and still writes the verdict`, async () => { + const f = fixture(scenario) + const verdict = await runSmoke(f.runtime, "private-test-key") + assert.equal(verdict.code, code) + assert.equal(verdict.passed, false) + assert.equal(smokeExitCode(verdict), 1) + assert.deepEqual(f.written, [verdict]) + assert.ok(f.cleaned && f.unsubscribed) + }) + } + it("bounds startup at the controller and closes its host when no reply arrives", async (t) => { + t.mock.timers.enable({ apis: ["setTimeout"] }) + const f = fixture("success", { timeout: () => t.mock.timers.tick(35_000) }) + const verdict = await runSmoke(f.runtime, "private-test-key") + assert.equal(verdict.code, "TASK_START_FAILED") + assert.equal(verdict.gates.taskAccepted, false) + assert.ok(f.cleaned) + }) + it("bounds TERM/KILL teardown and keeps live-host storage out of cleanup", async () => { + const f = fixture("success", { stuck: true }) + const verdict = await runSmoke(f.runtime, "private-test-key") + assert.equal(verdict.code, "TEARDOWN_FAILED") + assert.equal(verdict.durationMs, 15_000) + assert.deepEqual(f.signals, ["SIGTERM", "SIGKILL"]) + assert.equal(verdict.gates.childExited, false) + assert.equal(f.cleaned, false) + assert.equal(f.disconnected, true) + }) + it("preserves the original failure but reports failed teardown separately", async () => { + const f = fixture("start-failure", { stuck: true }) + const verdict = await runSmoke(f.runtime, "private-test-key") + assert.equal(verdict.code, "TASK_START_FAILED") + assert.equal(verdict.gates.childExited, false) + assert.equal(f.cleaned, false) + }) + it("interrupts an evidence wait, terminates only its host, and fails closed", async () => { + const f = fixture("no-completion") + const pause = f.runtime.pause + f.runtime.pause = async (ms) => { + f.interrupt() + await pause(ms) + } + const verdict = await runSmoke(f.runtime, "private-test-key") + assert.equal(verdict.code, "SESSION_LOST") + assert.deepEqual(f.signals, ["SIGTERM"]) + assert.ok(f.cleaned && f.unsubscribed) + }) + it("continues host termination and writes a failing verdict after a CloseTask exception", async () => { + const f = fixture("success", { closeThrows: true }) + const verdict = await runSmoke(f.runtime, "private-test-key") + assert.equal(verdict.code, "TEARDOWN_FAILED") + assert.deepEqual(f.signals, ["SIGTERM"]) + assert.ok(f.cleaned && f.disconnected) + }) +}) diff --git a/scripts/solheim-smoke/run.ts b/scripts/solheim-smoke/run.ts new file mode 100644 index 0000000000..698216598b --- /dev/null +++ b/scripts/solheim-smoke/run.ts @@ -0,0 +1,167 @@ +import { randomUUID } from "node:crypto" +import { TaskCommandName, type TaskCommand } from "@roo-code/types" +import { buildSmokeConfiguration, SMOKE_PROMPT, SMOKE_TIMEOUT_MS, SOLHEIM_BASE_URL, SOLHEIM_MODEL } from "./config.ts" +import { collectEvidence, smokeCode, type SmokeCode, type SmokeEvidence } from "./evidence.ts" +import { createTaskStartWaiter } from "./task-start.ts" + +export interface SmokeHost { + readonly exited: boolean + onExit(listener: () => void): void + signal(signal: NodeJS.Signals): void +} +export interface SmokeClient { + readonly isReady: boolean + onTaskEvent(listener: (event: unknown) => void): void + onDisconnect(listener: () => void): void + sendCommand(command: TaskCommand): void + disconnect(): void +} +export type SmokeVerdict = { + kind: "provider-smoke" + model: string + provider: string + passed: boolean + code: SmokeCode + gates: { + extensionActivated: boolean + ipcConnected: boolean + taskAccepted: boolean + providerResponseSeen: boolean + completionSeen: boolean + childExited: boolean + } + durationMs: number +} +export interface SmokeRuntime { + launch(): Promise + connect(): SmokeClient + cleanup(): Promise + writeVerdict(verdict: SmokeVerdict): Promise + onInterrupt(listener: () => void): () => void + now(): number + pause(ms: number): Promise +} + +// Only the adapter can spawn VS Code, read credentials or write files. Tests +// exercise the same controller with synthetic IPC and a deterministic clock. +export async function runSmoke(runtime: SmokeRuntime, apiKey: string): Promise { + const started = runtime.now() + const evidence: SmokeEvidence = { + providerResponseSeen: false, + completionSeen: false, + unexpectedTool: false, + aborted: false, + } + let code: SmokeCode = "SETUP_FAILED" + let activated = false + let accepted = false + let sessionLost = false + let host: SmokeHost | undefined + let client: SmokeClient | undefined + let waiter: ReturnType | undefined + async function until(check: () => boolean, timeoutMs: number, stopOnLoss = true): Promise { + const deadline = runtime.now() + timeoutMs + while (!check() && (!stopOnLoss || !sessionLost) && runtime.now() < deadline) await runtime.pause(100) + } + const markLost = () => { + sessionLost = true + waiter?.onExit() + } + const unsubscribe = runtime.onInterrupt(() => { + markLost() + try { + host?.signal("SIGTERM") + } catch { + /* Teardown retries independently. */ + } + }) + try { + if (!apiKey) throw new Error("Provider credential missing") + host = await runtime.launch() + host.onExit(markLost) + client = runtime.connect() + client.onDisconnect(() => { + sessionLost = true + waiter?.onDisconnect() + }) + let taskId: string | undefined + client.onTaskEvent((event) => { + const id = waiter?.onTaskEvent(event) + if (id && taskId === undefined) taskId = id + collectEvidence(evidence, event, taskId) + }) + await until(() => client?.isReady === true, SMOKE_TIMEOUT_MS) + activated = client.isReady + if (!activated) code = "ACTIVATION_FAILED" + else { + const requestId = randomUUID() + waiter = createTaskStartWaiter({ requestId, timeoutMs: 35_000 }) + // A synchronous send failure must not leave a rejected timer promise. + void waiter.result.catch(() => {}) + client.sendCommand({ + commandName: TaskCommandName.StartNewTask, + data: { requestId, text: SMOKE_PROMPT, newTab: false, configuration: buildSmokeConfiguration(apiKey) }, + }) + try { + taskId = await waiter.result + accepted = true + } catch { + code = "TASK_START_FAILED" + } + if (accepted) { + await until( + () => + evidence.unexpectedTool || + evidence.aborted || + (evidence.providerResponseSeen && evidence.completionSeen), + SMOKE_TIMEOUT_MS, + ) + code = smokeCode(evidence, sessionLost) + } + } + } catch { + // Never expose thrown provider, filesystem or host messages. + } finally { + waiter?.onExit() + const attempt = async (operation: () => unknown) => { + try { + await operation() + } catch { + if (code === "OK") code = "TEARDOWN_FAILED" + } + } + if (client?.isReady) await attempt(() => client?.sendCommand({ commandName: TaskCommandName.CloseTask })) + if (host && !host.exited) { + await attempt(() => until(() => host?.exited === true, 5_000, false)) + for (const signal of ["SIGTERM", "SIGKILL"] as const) { + if (host.exited) break + await attempt(() => host?.signal(signal)) + await attempt(() => until(() => host?.exited === true, 5_000, false)) + } + if (!host.exited && code === "OK") code = "TEARDOWN_FAILED" + } + await attempt(() => client?.disconnect()) + if (!host || host.exited) await attempt(() => runtime.cleanup()) + await attempt(unsubscribe) + } + const verdict: SmokeVerdict = { + kind: "provider-smoke", + model: SOLHEIM_MODEL, + provider: SOLHEIM_BASE_URL, + passed: code === "OK", + code, + gates: { + extensionActivated: activated, + ipcConnected: activated, + taskAccepted: accepted, + providerResponseSeen: evidence.providerResponseSeen, + completionSeen: evidence.completionSeen, + childExited: !host || host.exited, + }, + durationMs: runtime.now() - started, + } + await runtime.writeVerdict(verdict) + return verdict +} + +export const smokeExitCode = (verdict: SmokeVerdict): number => (verdict.passed ? 0 : 1) diff --git a/src/extension/__tests__/api-start-task-logging.spec.ts b/src/extension/__tests__/api-start-task-logging.spec.ts index 56b331dd0e..cb7ee10389 100644 --- a/src/extension/__tests__/api-start-task-logging.spec.ts +++ b/src/extension/__tests__/api-start-task-logging.spec.ts @@ -51,18 +51,6 @@ const buildStartCommand = (overrides?: { requestId?: string; text?: string }) => }, }) -function withStageTimeoutOverride(ms: string): () => void { - const previous = process.env.ROO_CODE_TASK_START_STAGE_TIMEOUT_MS - process.env.ROO_CODE_TASK_START_STAGE_TIMEOUT_MS = ms - return () => { - if (previous === undefined) { - delete process.env.ROO_CODE_TASK_START_STAGE_TIMEOUT_MS - } else { - process.env.ROO_CODE_TASK_START_STAGE_TIMEOUT_MS = previous - } - } -} - function buildApi(createTaskImpl?: () => Promise<{ taskId: string }>, postStateToWebviewImpl?: () => Promise) { const outputChannel = { appendLine: vi.fn() } as unknown as vscode.OutputChannel const provider = { @@ -98,6 +86,7 @@ describe("API StartNewTask IPC", () => { afterEach(() => { commandHandlers.length = 0 sentMessages.length = 0 + vi.useRealTimers() vi.restoreAllMocks() }) @@ -139,14 +128,15 @@ describe("API StartNewTask IPC", () => { await handler!("client-1", buildStartCommand({ requestId: "req-1" })) expect(sentMessages).toHaveLength(1) + expect(sentMessages[0]?.clientId).toBe("client-1") const events = sentTaskEvents() expect(events[0]?.data.eventName).toBe(RooCodeEventName.TaskStartResponse) - const parsed = taskStartResponseSchema.safeParse(events[0]?.data.payload[0]) - expect(parsed.success).toBe(true) - if (parsed.success) { - expect(parsed.data).toEqual({ requestId: "req-1", success: true, taskId: "task-1" }) - } + expect(taskStartResponseSchema.parse(events[0]?.data.payload[0])).toEqual({ + requestId: "req-1", + success: true, + taskId: "task-1", + }) }) it("answers a failed correlated start with exactly one fixed safe failure response", async () => { @@ -159,16 +149,17 @@ describe("API StartNewTask IPC", () => { await handler!("client-1", buildStartCommand({ requestId: "req-2" })) expect(sentMessages).toHaveLength(1) + expect(sentMessages[0]?.clientId).toBe("client-1") const events = sentTaskEvents() expect(events[0]?.data.eventName).toBe(RooCodeEventName.TaskStartResponse) - const parsed = taskStartResponseSchema.safeParse(events[0]?.data.payload[0]) - expect(parsed.success).toBe(true) - if (parsed.success && !parsed.data.success) { - expect(parsed.data.errorCode).toBe(TASK_START_FAILURE_ERROR_CODE) - expect(parsed.data.errorMessage).toBe(TASK_START_FAILURE_ERROR_MESSAGE) - expect(parsed.data.stage).toBe("unknown") - } + const parsed = taskStartResponseSchema.parse(events[0]?.data.payload[0]) + expect(parsed).toEqual({ + requestId: "req-2", + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + }) const sent = JSON.stringify(sentMessages) expect(sent).not.toContain("opaque-solheim-secret-0123456789abcdef") @@ -176,113 +167,60 @@ describe("API StartNewTask IPC", () => { expect(sent).not.toContain("review the pull request diff") }) - it("bounds a hung task creation stage and answers with exactly one sanitized stage failure", async () => { - const restore = withStageTimeoutOverride("100") - try { - const { collectLogs } = buildApi(() => new Promise(() => {})) - - const handler = commandHandlers.at(-1) - await handler!("client-1", buildStartCommand({ requestId: "req-timeout-task" })) - - expect(sentMessages).toHaveLength(1) - const events = sentTaskEvents() - expect(events[0]?.data.eventName).toBe(RooCodeEventName.TaskStartResponse) - - const parsed = taskStartResponseSchema.safeParse(events[0]?.data.payload[0]) - expect(parsed.success).toBe(true) - if (parsed.success && !parsed.data.success) { - expect(parsed.data).toEqual({ - requestId: "req-timeout-task", - success: false, - errorCode: TASK_START_FAILURE_ERROR_CODE, - errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, - stage: "taskCreation", - }) - } - - const logged = collectLogs() - expect(logged).toContain("taskCreation") - expect(logged).toContain("req-timeout-task") - expect(logged).not.toContain("opaque-solheim-secret-0123456789abcdef") - } finally { - restore() - } - }) - - it("removes a late task and refuses a retry until the timed-out creation settles", async () => { - const restore = withStageTimeoutOverride("50") - try { - let finishCreation: (task: { taskId: string }) => void = () => {} - const { provider } = buildApi( - () => - new Promise((resolve) => { - finishCreation = resolve - }), - ) - const handler = commandHandlers.at(-1)! - - await handler("client-1", buildStartCommand({ requestId: "req-first" })) - await handler("client-1", buildStartCommand({ requestId: "req-retry" })) - - expect(provider.createTask).toHaveBeenCalledTimes(1) - const retry = taskStartResponseSchema.parse(sentTaskEvents()[1]?.data.payload[0]) - expect(retry.success).toBe(false) - - finishCreation({ taskId: "late-task" }) - await vi.waitFor(() => expect(provider.removeClineFromStack).toHaveBeenCalledTimes(1)) - - await handler("client-1", buildStartCommand({ requestId: "req-after" })) - expect(provider.createTask).toHaveBeenCalledTimes(2) - } finally { - restore() - } - }) - - it("bounds a hung settings stage and names it in the failure response", async () => { - const restore = withStageTimeoutOverride("100") - try { - buildApi(undefined, () => new Promise(() => {})) - - const handler = commandHandlers.at(-1) - await handler!("client-1", buildStartCommand({ requestId: "req-timeout-settings" })) - - expect(sentMessages).toHaveLength(1) - const parsed = taskStartResponseSchema.safeParse(sentTaskEvents()[0]?.data.payload[0]) - expect(parsed.success).toBe(true) - if (parsed.success && !parsed.data.success) { - expect(parsed.data.stage).toBe("settings") - } - } finally { - restore() - } + it("does not race pending configuration/startup against a timeout", async () => { + vi.useFakeTimers() + let finishCreation!: (task: { taskId: string }) => void + const { provider } = buildApi( + () => + new Promise((resolve) => { + finishCreation = resolve + }), + ) + const running = commandHandlers.at(-1)!("client-1", buildStartCommand({ requestId: "req-pending" })) + await vi.advanceTimersByTimeAsync(60_000) + expect(provider.createTask).toHaveBeenCalledTimes(1) + expect(sentMessages).toHaveLength(0) + expect(provider.removeClineFromStack).not.toHaveBeenCalled() + finishCreation({ taskId: "finished-task" }) + await running + expect(taskStartResponseSchema.parse(sentTaskEvents()[0]?.data.payload[0])).toEqual({ + requestId: "req-pending", + success: true, + taskId: "finished-task", + }) }) - it("still answers with one success response when every stage is fast under a small bound", async () => { - const restore = withStageTimeoutOverride("100") - try { - buildApi() - - const handler = commandHandlers.at(-1) - await handler!("client-1", buildStartCommand({ requestId: "req-fast" })) - - expect(sentMessages).toHaveLength(1) - const parsed = taskStartResponseSchema.safeParse(sentTaskEvents()[0]?.data.payload[0]) - expect(parsed.success).toBe(true) - if (parsed.success) { - expect(parsed.data).toEqual({ requestId: "req-fast", success: true, taskId: "task-1" }) - } - } finally { - restore() - } + it("reports a delayed startup rejection only after it settles, without late task removal", async () => { + vi.useFakeTimers() + let rejectCreation!: (error: Error) => void + const { provider } = buildApi( + () => + new Promise((_resolve, reject) => { + rejectCreation = reject + }), + ) + const running = commandHandlers.at(-1)!("client-1", buildStartCommand({ requestId: "req-reject" })) + await vi.advanceTimersByTimeAsync(60_000) + expect(sentMessages).toHaveLength(0) + rejectCreation(new Error("private startup error")) + await running + expect(taskStartResponseSchema.parse(sentTaskEvents()[0]?.data.payload[0])).toEqual({ + requestId: "req-reject", + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + }) + expect(provider.removeClineFromStack).not.toHaveBeenCalled() }) it("sends no start response for legacy IPC starts without requestId", async () => { - buildApi() + const { executeCommand } = buildApi() const handler = commandHandlers.at(-1) await handler!("client-1", buildStartCommand()) expect(sentMessages).toHaveLength(0) + expect(executeCommand).not.toHaveBeenCalledWith(`${Package.name}.SidebarProvider.focus`) }) it("keeps the sidebar focus for direct API callers that do not opt out", async () => { diff --git a/src/extension/api.ts b/src/extension/api.ts index 711e03c5b2..8f4d61e73d 100644 --- a/src/extension/api.ts +++ b/src/extension/api.ts @@ -14,7 +14,6 @@ import { type ProviderSettingsEntry, type TaskEvent, type TaskStartResponse, - type TaskStartStage, type CreateTaskOptions, type TaskApiConversationHistorySequence, type WebviewThemeFixture, @@ -38,51 +37,9 @@ import { openClineInNewTab } from "../activate/registerCommands" import { getCommands } from "../services/command/commands" import { getModels } from "../api/providers/fetchers/modelCache" -// The default bound is per stage, not per start, so one slow stage cannot -// hide behind the budget of another. Secret storage writes through VS Code -// secret storage can hang indefinitely under Xvfb without a keyring service. -const DEFAULT_TASK_START_STAGE_TIMEOUT_MS = 60_000 - -function readTaskStartStageTimeoutMs(env: NodeJS.ProcessEnv): number { - const parsed = Number(env.ROO_CODE_TASK_START_STAGE_TIMEOUT_MS) - return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_TASK_START_STAGE_TIMEOUT_MS -} - -class TaskStartStageTimeoutError extends Error { - constructor(readonly stage: TaskStartStage) { - super(`task start stage timed out: ${stage}`) - } -} - -// Races the stage against its bound. Losing the race stops the wait only. -// The operation keeps running, so onAbandon receives it for later cleanup. -async function withStageTimeout( - stage: TaskStartStage, - timeoutMs: number, - operation: () => T | Thenable, - onAbandon?: (running: Promise) => void, -): Promise { - let timer: NodeJS.Timeout | undefined - const running = Promise.resolve(operation()) - try { - return await Promise.race([ - running, - new Promise((_, reject) => { - timer = setTimeout(() => { - onAbandon?.(running) - reject(new TaskStartStageTimeoutError(stage)) - }, timeoutMs) - }), - ]) - } finally { - clearTimeout(timer) - } -} - export class API extends EventEmitter implements RooCodeAPI { private readonly outputChannel: vscode.OutputChannel private readonly sidebarProvider: ClineProvider - private abandonedStarts = 0 private readonly context: vscode.ExtensionContext private readonly ipc?: IpcServer private readonly log: (...args: unknown[]) => void @@ -147,26 +104,22 @@ export class API extends EventEmitter implements RooCodeAPI { // Exactly one sanitized, client-scoped response per // correlated start. The reply never echoes the failure. - // Each startup stage is bounded so a hung await (for - // example secret storage under a missing keyring) - // still produces the reply. + // Await actual startup, including configuration writes. + // The smoke controller owns its timeout and terminates + // its isolated host; shared-profile writes are not raced. let response: TaskStartResponse try { const taskId = await this.startNewTask(command.data, { focusSidebar: false, - stageTimeoutMs: readTaskStartStageTimeoutMs(process.env), }) response = { requestId, success: true, taskId } - } catch (error) { - const stage: TaskStartStage = - error instanceof TaskStartStageTimeoutError ? error.stage : "unknown" - this.log("[API] StartNewTask failed", { requestId, stage }) + } catch { + this.log("[API] StartNewTask failed", { requestId }) response = { requestId, success: false, errorCode: TASK_START_FAILURE_ERROR_CODE, errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, - stage, } } sendResponse(RooCodeEventName.TaskStartResponse, [response]) @@ -266,73 +219,36 @@ export class API extends EventEmitter implements RooCodeAPI { images?: string[] newTab?: boolean }, - startOptions?: { focusSidebar?: boolean; stageTimeoutMs?: number }, + startOptions?: { focusSidebar?: boolean }, ) { - // Stage names for a bounded correlated start. The settings persistence - // named in the startup evidence (createTask -> setValues -> - // storeSecret) runs inside the taskCreation stage at this layer, so a - // taskCreation timeout covers it. - // A timed-out stage keeps running. Count it, and reject a new bounded - // start until it settles. A late task from the taskCreation stage is - // removed, so it cannot run after the failure reply. - if (startOptions?.stageTimeoutMs !== undefined && this.abandonedStarts > 0) { - throw new Error("an earlier task start is still running") - } - const boundStage = ( - stage: TaskStartStage, - operation: () => T | Thenable, - onLate?: (late: Awaited) => unknown, - ): Promise => - startOptions?.stageTimeoutMs === undefined - ? Promise.resolve(operation()) - : withStageTimeout(stage, startOptions.stageTimeoutMs, operation, (running) => { - this.abandonedStarts++ - void running - .then((late) => onLate?.(late as Awaited)) - .catch(() => {}) - .finally(() => this.abandonedStarts--) - }) - let provider: ClineProvider if (newTab) { - await boundStage("unknown", () => vscode.commands.executeCommand("workbench.action.files.revert")) - await boundStage("unknown", () => vscode.commands.executeCommand("workbench.action.closeAllEditors")) + await vscode.commands.executeCommand("workbench.action.files.revert") + await vscode.commands.executeCommand("workbench.action.closeAllEditors") - provider = await boundStage("unknown", () => - openClineInNewTab({ context: this.context, outputChannel: this.outputChannel }), - ) + provider = await openClineInNewTab({ context: this.context, outputChannel: this.outputChannel }) this.registerListeners(provider) } else { // IPC callers omit focus: executeCommand cannot be cancelled and a // stalled webview launch would block the whole start. if (startOptions?.focusSidebar ?? true) { - await boundStage("unknown", () => - vscode.commands.executeCommand(`${Package.name}.SidebarProvider.focus`), - ) + await vscode.commands.executeCommand(`${Package.name}.SidebarProvider.focus`) } provider = this.sidebarProvider } - await boundStage("eviction", () => provider.evictCurrentTask()) - await boundStage("settings", () => provider.postStateToWebview()) - await boundStage("settings", () => - provider.postMessageToWebview({ type: "action", action: "chatButtonClicked" }), - ) - await boundStage("settings", () => - provider.postMessageToWebview({ type: "invoke", invoke: "newChat", text, images }), - ) + await provider.evictCurrentTask() + await provider.postStateToWebview() + await provider.postMessageToWebview({ type: "action", action: "chatButtonClicked" }) + await provider.postMessageToWebview({ type: "invoke", invoke: "newChat", text, images }) const options: CreateTaskOptions = { consecutiveMistakeLimit: Number.MAX_SAFE_INTEGER, } - const task = await boundStage( - "taskCreation", - () => provider.createTask(text, images, undefined, options, configuration), - (late) => late && provider.removeClineFromStack(), - ) + const task = await provider.createTask(text, images, undefined, options, configuration) if (!task) { throw new Error("Failed to create task due to policy restrictions")