diff --git a/.github/workflows/code-qa.yml b/.github/workflows/code-qa.yml index dcc9fb509d..041a285753 100644 --- a/.github/workflows/code-qa.yml +++ b/.github/workflows/code-qa.yml @@ -92,6 +92,12 @@ jobs: run: pnpm check-types - name: Validate Code QA workflow run: pnpm test:code-qa-ci + - name: Validate Solheim provider smoke workflow + run: pnpm test:solheim-smoke-ci + - name: Test Solheim provider smoke units + run: pnpm test:solheim-smoke:unit + - name: Check Solheim provider smoke types + run: pnpm solheim-smoke:check-types - name: Model-check task lifecycle protocols run: pnpm lifecycle:model-check - name: Validate MCP OAuth integration diff --git a/.github/workflows/solheim-provider-smoke.yml b/.github/workflows/solheim-provider-smoke.yml new file mode 100644 index 0000000000..7f61107186 --- /dev/null +++ b/.github/workflows/solheim-provider-smoke.yml @@ -0,0 +1,60 @@ +name: Solheim provider smoke + +# Project A: infrastructure validation only. No PR diff, review judgment, router, +# deterministic review rule, or GitHub posting. Only trusted main-branch code runs. +on: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: solheim-provider-smoke + cancel-in-progress: false + +jobs: + smoke: + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-24.04 + timeout-minutes: 25 + # Retain the existing protected environment and secret names for compatibility. + # Repository policy must restrict deployments to main and require approval. + environment: final-vscode-review-smoke + steps: + - name: Checkout exact trusted main revision + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - name: Setup Node.js and pnpm + uses: ./.github/actions/setup-node-pnpm + with: + install-args: "--frozen-lockfile" + + - name: Build trusted extension and webview + run: | + pnpm -w bundle + pnpm --filter @roo-code/vscode-webview build + + - name: Install display dependencies + run: sudo apt-get update && sudo apt-get install -y xvfb ffmpeg libasound2t64 + + - name: Run Solheim provider smoke + id: provider + timeout-minutes: 8 + env: + SOLHEIM_API_KEY: ${{ secrets.FINAL_SMOKE_OPENAI_API_KEY }} + SOLHEIM_SMOKE_OUT_DIR: ${{ runner.temp }}/solheim-provider-smoke + run: xvfb-run -a -s "-screen 0 1280x720x24 -nolisten tcp" bash scripts/solheim-smoke/record.sh + + - name: Upload provider smoke verdict and recording + if: ${{ !cancelled() && (steps.provider.outcome == 'success' || steps.provider.outcome == 'failure') }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: solheim-provider-smoke-${{ github.sha }} + path: | + ${{ runner.temp }}/solheim-provider-smoke/verdict.json + ${{ runner.temp }}/solheim-provider-smoke/smoke.mp4 + if-no-files-found: error + retention-days: 7 diff --git a/package.json b/package.json index 1bf0b4e8dd..16b8328865 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,10 @@ "check-types": "turbo check-types --log-order grouped --output-logs new-only", "test": "turbo test --log-order grouped --output-logs new-only", "test:code-qa-ci": "node --test scripts/code-qa-workflow.test.mjs", + "solheim:smoke": "node --import tsx scripts/solheim-smoke/driver.mts", + "test:solheim-smoke-ci": "node --test scripts/solheim-smoke-workflow.test.mjs", + "test:solheim-smoke:unit": "node --import tsx --test scripts/solheim-smoke/*.test.ts", + "solheim-smoke:check-types": "tsc -p scripts/solheim-smoke/tsconfig.json", "test:mutation-ci": "node --test scripts/stryker-diff.test.mjs", "lifecycle:model-check": "tsx scripts/check-task-lifecycle.ts && tsx scripts/check-task-store-concurrency.ts && tsx scripts/check-provider-handoff-scheduler.ts && pnpm cleanup-protocol:model-check && pnpm parser-scope:model-check && tsx scripts/check-completion-persistence.ts && tsx scripts/check-delegated-mode-readers.ts", "fanout-protocol:model-check": "tsx scripts/check-task-fanout-protocol.ts", @@ -34,6 +38,9 @@ "devDependencies": { "@changesets/cli": "2.31.0", "@roo-code/config-typescript": "workspace:^", + "@roo-code/ipc": "workspace:^", + "@roo-code/types": "workspace:^", + "@vscode/test-electron": "2.5.2", "@stryker-mutator/core": "10.0.0", "@stryker-mutator/vitest-runner": "10.0.0", "@types/node": "22.20.1", diff --git a/packages/ipc/package.json b/packages/ipc/package.json index 374c4240b6..4dd505bb18 100644 --- a/packages/ipc/package.json +++ b/packages/ipc/package.json @@ -5,6 +5,7 @@ "type": "module", "exports": "./src/index.ts", "scripts": { + "test": "node --import tsx --test src/__tests__/*.test.ts", "lint": "eslint src --ext=ts --max-warnings=0", "check-types": "tsc --noEmit", "clean": "rimraf .turbo" diff --git a/packages/ipc/src/__tests__/ipc-server.test.ts b/packages/ipc/src/__tests__/ipc-server.test.ts new file mode 100644 index 0000000000..3419ebf982 --- /dev/null +++ b/packages/ipc/src/__tests__/ipc-server.test.ts @@ -0,0 +1,65 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { IpcMessageType, IpcOrigin, TaskCommandName, ipcMessageSchema } from "@roo-code/types" +import { IpcServer } from "../ipc-server.js" + +function fixture() { + const logs: unknown[][] = [] + const commands: unknown[] = [] + const server = new IpcServer("/unused-test.sock", (...args) => logs.push(args)) + server.on(IpcMessageType.TaskCommand, (clientId, command) => commands.push({ clientId, command })) + return { logs, commands, receive: (data: unknown) => server["onMessage"](data) } +} +const key = "dummy-api-key-not-for-public-logging" +const command = (requestId: string) => ({ + type: IpcMessageType.TaskCommand, + origin: IpcOrigin.Client, + clientId: "client-1", + data: { + commandName: TaskCommandName.StartNewTask, + data: { requestId, text: "private prompt", configuration: { apiProvider: "openai", openAiApiKey: key } }, + }, +}) + +describe("IPC rejection logging", () => { + it("rejects an invalid request ID without logging its key, prompt or schema issue values", () => { + const f = fixture() + f.receive(command(`invalid request containing ${key}`)) + assert.deepEqual(f.commands, []) + assert.equal(f.logs.length, 1) + assert.equal(f.logs[0]?.[0], "[server#onMessage] invalid payload") + assert.ok(!JSON.stringify(f.logs).includes(key)) + assert.ok(!JSON.stringify(f.logs).includes("private prompt")) + assert.ok(!JSON.stringify(f.logs).includes("invalid request containing")) + }) + it("does not log credential-shaped primitive input or dynamic issue paths", () => { + const f = fixture() + const valid = command("valid-id") + const malformedHeaders = { + ...valid, + data: { + ...valid.data, + data: { + ...valid.data.data, + configuration: { ...valid.data.data.configuration, openAiHeaders: { [key]: null } }, + }, + }, + } + const rejected = ipcMessageSchema.safeParse(malformedHeaders) + assert.ok(!rejected.success) + assert.ok(rejected.error.issues.some((issue) => issue.path.includes(key))) + for (const data of [key, null, { type: key }, malformedHeaders]) { + f.receive(data) + } + assert.equal(f.logs.length, 4) + assert.ok(!JSON.stringify(f.logs).includes(key)) + assert.deepEqual(f.commands, []) + }) + it("still dispatches a valid correlated command to its client without logging its settings", () => { + const f = fixture() + f.receive(command("valid-id")) + assert.equal(f.commands.length, 1) + assert.deepEqual(f.commands[0], { clientId: "client-1", command: command("valid-id").data }) + assert.deepEqual(f.logs, []) + }) +}) diff --git a/packages/ipc/src/ipc-server.ts b/packages/ipc/src/ipc-server.ts index 200e241014..26dd626a9e 100644 --- a/packages/ipc/src/ipc-server.ts +++ b/packages/ipc/src/ipc-server.ts @@ -76,16 +76,15 @@ export class IpcServer extends EventEmitter implements RooCodeI private onMessage(data: unknown) { if (typeof data !== "object") { - this.log(`[server#onMessage] invalid data -> ${JSON.stringify(data)}`) + this.log("[server#onMessage] invalid data") return } const result = ipcMessageSchema.safeParse(data) if (!result.success) { - this.log( - `[server#onMessage] invalid payload -> ${JSON.stringify(result.error.issues)} -> ${JSON.stringify(data)}`, - ) + // Both the input and Zod issue values/paths may contain credentials. + this.log("[server#onMessage] invalid payload", { issueCount: result.error.issues.length }) return } @@ -98,7 +97,7 @@ export class IpcServer extends EventEmitter implements RooCodeI this.emit(IpcMessageType.TaskCommand, payload.clientId, payload.data) break default: - this.log(`[server#onMessage] unhandled payload: ${JSON.stringify(payload)}`) + this.log("[server#onMessage] unhandled payload") break } } diff --git a/packages/types/src/__tests__/events.test.ts b/packages/types/src/__tests__/events.test.ts new file mode 100644 index 0000000000..3525e75eac --- /dev/null +++ b/packages/types/src/__tests__/events.test.ts @@ -0,0 +1,234 @@ +import { + RooCodeEventName, + TASK_START_FAILURE_ERROR_CODE, + TASK_START_FAILURE_ERROR_MESSAGE, + TASK_START_STAGES, + taskEventSchema, + taskStartResponseSchema, +} from "../events.js" + +const requestId = "final-smoke-0123abcdef456789" + +describe("taskStartResponseSchema", () => { + it("accepts a success response with the accepted task ID", () => { + const result = taskStartResponseSchema.safeParse({ requestId, success: true, taskId: "task-1" }) + expect(result.success).toBe(true) + + if (result.success) { + expect(result.data).toEqual({ requestId, success: true, taskId: "task-1" }) + } + }) + + it("accepts the fixed failure response", () => { + const result = taskStartResponseSchema.safeParse({ + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + }) + expect(result.success).toBe(true) + + if (result.success && !result.data.success) { + expect(result.data.success).toBe(false) + expect(result.data.stage).toBeUndefined() + } + }) + + it("accepts a failure response carrying each allowed stage", () => { + for (const stage of TASK_START_STAGES) { + const result = taskStartResponseSchema.safeParse({ + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + stage, + }) + expect(result.success).toBe(true) + + if (result.success && !result.data.success) { + expect(result.data.stage).toBe(stage) + } + } + }) + + it("rejects a failure response carrying an unknown stage", () => { + const result = taskStartResponseSchema.safeParse({ + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + stage: "secretStore", + }) + expect(result.success).toBe(false) + }) + + it("rejects a failure response carrying a non-string stage", () => { + const result = taskStartResponseSchema.safeParse({ + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + stage: { name: "taskCreation" }, + }) + expect(result.success).toBe(false) + }) + + it("rejects a success response carrying a stage", () => { + const result = taskStartResponseSchema.safeParse({ + requestId, + success: true, + taskId: "task-1", + stage: "taskCreation", + }) + expect(result.success).toBe(false) + }) + + it("rejects a success response without a taskId", () => { + const result = taskStartResponseSchema.safeParse({ requestId, success: true }) + expect(result.success).toBe(false) + }) + + it("rejects a failure response carrying an unfixed error code", () => { + const result = taskStartResponseSchema.safeParse({ + requestId, + success: false, + errorCode: "provider_auth_failed", + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + }) + expect(result.success).toBe(false) + }) + + it("rejects a failure response carrying a raw error message", () => { + const result = taskStartResponseSchema.safeParse({ + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: "openai rejected the key sk-secret-123 at https://api.example.com", + }) + expect(result.success).toBe(false) + }) + + it("rejects an empty requestId", () => { + const result = taskStartResponseSchema.safeParse({ requestId: "", success: true, taskId: "task-1" }) + expect(result.success).toBe(false) + }) + + it("rejects a success response with an extra top-level field", () => { + const result = taskStartResponseSchema.safeParse({ + requestId, + success: true, + taskId: "task-1", + providerError: "openai rejected the key sk-secret-123", + }) + expect(result.success).toBe(false) + }) + + it("rejects a failure response with an extra field on the error object", () => { + const result = taskStartResponseSchema.safeParse({ + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + detail: { apiKey: "sk-secret-123", url: "https://api.example.com" }, + }) + expect(result.success).toBe(false) + }) +}) + +describe("taskEventSchema TaskStartResponse", () => { + it("routes a success response through the task event envelope", () => { + const result = taskEventSchema.safeParse({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [{ requestId, success: true, taskId: "task-1" }], + }) + expect(result.success).toBe(true) + }) + + it("routes a failure response through the task event envelope", () => { + const result = taskEventSchema.safeParse({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [ + { + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + }, + ], + }) + expect(result.success).toBe(true) + }) + + it("routes a failure response with a stage through the task event envelope", () => { + const result = taskEventSchema.safeParse({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [ + { + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + stage: "taskCreation", + }, + ], + }) + expect(result.success).toBe(true) + }) + + it("rejects a failure payload with an unknown stage through the task event envelope", () => { + const result = taskEventSchema.safeParse({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [ + { + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + stage: "keyring", + }, + ], + }) + expect(result.success).toBe(false) + }) + + it("rejects a failure payload with a stage plus an extra field through the task event envelope", () => { + const result = taskEventSchema.safeParse({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [ + { + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + stage: "taskCreation", + rawError: "sk-secret-123", + }, + ], + }) + expect(result.success).toBe(false) + }) + + it("rejects a success payload with an extra field through the task event envelope", () => { + const result = taskEventSchema.safeParse({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [{ requestId, success: true, taskId: "task-1", stack: "at run (api.ts:1:1)" }], + }) + expect(result.success).toBe(false) + }) + + it("rejects a failure payload with an extra error field through the task event envelope", () => { + const result = taskEventSchema.safeParse({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [ + { + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + rawError: "sk-secret-123", + }, + ], + }) + expect(result.success).toBe(false) + }) +}) diff --git a/packages/types/src/__tests__/ipc.test.ts b/packages/types/src/__tests__/ipc.test.ts index a843354a55..daf9b3cb1d 100644 --- a/packages/types/src/__tests__/ipc.test.ts +++ b/packages/types/src/__tests__/ipc.test.ts @@ -117,4 +117,59 @@ describe("IPC Types", () => { expect(result.success).toBe(false) }) }) + + describe("StartNewTask requestId", () => { + const legacyCommand = { + commandName: TaskCommandName.StartNewTask, + data: { + configuration: {}, + text: "review the diff", + }, + } + + it("should accept a legacy StartNewTask without requestId", () => { + const result = taskCommandSchema.safeParse(legacyCommand) + expect(result.success).toBe(true) + + if (result.success && result.data.commandName === TaskCommandName.StartNewTask) { + expect(result.data.data.requestId).toBeUndefined() + } + }) + + it("should accept and preserve a valid requestId", () => { + const result = taskCommandSchema.safeParse({ + ...legacyCommand, + data: { ...legacyCommand.data, requestId: "final-smoke-0123abcdef456789" }, + }) + expect(result.success).toBe(true) + + if (result.success && result.data.commandName === TaskCommandName.StartNewTask) { + expect(result.data.data.requestId).toBe("final-smoke-0123abcdef456789") + } + }) + + it("should reject an empty requestId", () => { + const result = taskCommandSchema.safeParse({ + ...legacyCommand, + data: { ...legacyCommand.data, requestId: "" }, + }) + expect(result.success).toBe(false) + }) + + it("should reject a requestId longer than 128 characters", () => { + const result = taskCommandSchema.safeParse({ + ...legacyCommand, + data: { ...legacyCommand.data, requestId: "a".repeat(129) }, + }) + expect(result.success).toBe(false) + }) + + it("should reject a requestId with characters outside the allowed set", () => { + const result = taskCommandSchema.safeParse({ + ...legacyCommand, + data: { ...legacyCommand.data, requestId: "id with spaces" }, + }) + expect(result.success).toBe(false) + }) + }) }) diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index 20f7f7e71e..204256980d 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -51,8 +51,59 @@ export enum RooCodeEventName { CommandsResponse = "commandsResponse", ModesResponse = "modesResponse", ModelsResponse = "modelsResponse", + /** Correlated reply to a StartNewTask command that carried a requestId. */ + TaskStartResponse = "taskStartResponse", } +/** + * TaskStartResponse + */ + +// The requestId is generated by the command sender and copied into the reply, +// so its shape is bounded and restricted to characters that cannot smuggle +// log structure. +export const taskStartRequestIdSchema = z + .string() + .min(1) + .max(128) + .regex(/^[A-Za-z0-9._:-]+$/) + +// The failure values are fixed constants. The response never carries raw +// thrown messages, prompts, images, provider settings, secrets, URLs, or +// stack traces. Both variants reject unknown keys, so an extra or +// secret-bearing field fails validation instead of being stripped. +export const TASK_START_FAILURE_ERROR_CODE = "task_start_failed" +export const TASK_START_FAILURE_ERROR_MESSAGE = "Task start failed" + +// Fixed stage vocabulary for a failed correlated start. The stage names the +// startup phase that stopped responding; it carries no error content. +export const TASK_START_STAGES = ["eviction", "settings", "taskCreation", "unknown"] as const + +export const taskStartStageSchema = z.enum(TASK_START_STAGES) + +export type TaskStartStage = (typeof TASK_START_STAGES)[number] + +export const taskStartResponseSchema = z.discriminatedUnion("success", [ + z + .object({ + requestId: taskStartRequestIdSchema, + success: z.literal(true), + taskId: z.string().min(1), + }) + .strict(), + z + .object({ + requestId: taskStartRequestIdSchema, + success: z.literal(false), + errorCode: z.literal(TASK_START_FAILURE_ERROR_CODE), + errorMessage: z.literal(TASK_START_FAILURE_ERROR_MESSAGE), + stage: taskStartStageSchema.optional(), + }) + .strict(), +]) + +export type TaskStartResponse = z.infer + /** * RooCodeEvents */ @@ -125,6 +176,7 @@ export const rooCodeEventsSchema = z.object({ ]), [RooCodeEventName.ModesResponse]: z.tuple([z.array(z.object({ slug: z.string(), name: z.string() }))]), [RooCodeEventName.ModelsResponse]: z.tuple([z.record(z.string(), modelInfoSchema)]), + [RooCodeEventName.TaskStartResponse]: z.tuple([taskStartResponseSchema]), }) export type RooCodeEvents = z.infer @@ -270,6 +322,11 @@ export const taskEventSchema = z.discriminatedUnion("eventName", [ payload: rooCodeEventsSchema.shape[RooCodeEventName.ModelsResponse], taskId: z.number().optional(), }), + z.object({ + eventName: z.literal(RooCodeEventName.TaskStartResponse), + payload: rooCodeEventsSchema.shape[RooCodeEventName.TaskStartResponse], + taskId: z.number().optional(), + }), ]) export type TaskEvent = z.infer diff --git a/packages/types/src/ipc.ts b/packages/types/src/ipc.ts index fea040af0b..e7df762069 100644 --- a/packages/types/src/ipc.ts +++ b/packages/types/src/ipc.ts @@ -1,6 +1,6 @@ import { z } from "zod" -import { type TaskEvent, taskEventSchema } from "./events.js" +import { type TaskEvent, taskEventSchema, taskStartRequestIdSchema } from "./events.js" import { rooCodeSettingsSchema } from "./global-settings.js" /** @@ -64,6 +64,8 @@ export const taskCommandSchema = z.discriminatedUnion("commandName", [ text: z.string(), images: z.array(z.string()).optional(), newTab: z.boolean().optional(), + // Senders without a requestId keep the fire-and-forget contract. + requestId: taskStartRequestIdSchema.optional(), }), }), z.object({ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b12ecd1c77..deeb91fd61 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -30,6 +30,12 @@ importers: '@roo-code/config-typescript': specifier: workspace:^ version: link:packages/config-typescript + '@roo-code/ipc': + specifier: workspace:^ + version: link:packages/ipc + '@roo-code/types': + specifier: workspace:^ + version: link:packages/types '@stryker-mutator/core': specifier: 10.0.0 version: 10.0.0(@types/node@22.20.1) @@ -39,6 +45,9 @@ importers: '@types/node': specifier: 22.20.1 version: 22.20.1 + '@vscode/test-electron': + specifier: 2.5.2 + version: 2.5.2 '@vscode/vsce': specifier: 3.9.2 version: 3.9.2 diff --git a/scripts/solheim-smoke-workflow.test.mjs b/scripts/solheim-smoke-workflow.test.mjs new file mode 100644 index 0000000000..0bab69c438 --- /dev/null +++ b/scripts/solheim-smoke-workflow.test.mjs @@ -0,0 +1,140 @@ +import assert from "node:assert/strict" +import fs from "node:fs" +import { URL } from "node:url" +import { describe, it } from "node:test" +import { spawnSync } from "node:child_process" +import os from "node:os" +import path from "node:path" +const workflow = fs.readFileSync(new URL("../.github/workflows/solheim-provider-smoke.yml", import.meta.url), "utf8") +const codeQa = fs.readFileSync(new URL("../.github/workflows/code-qa.yml", import.meta.url), "utf8") +const pkg = JSON.parse(fs.readFileSync(new URL("../package.json", import.meta.url), "utf8")) +const driver = fs.readFileSync(new URL("./solheim-smoke/driver.mts", import.meta.url), "utf8") +const recorder = fs.readFileSync(new URL("./solheim-smoke/record.sh", import.meta.url), "utf8") +const smokePkg = JSON.parse(fs.readFileSync(new URL("./solheim-smoke/package.json", import.meta.url), "utf8")) + +describe("Project A provider smoke workflow", () => { + it("runs smoke TypeScript as ESM without requiring compiled workspace packages", () => { + assert.equal(smokePkg.type, "module") + assert.equal(smokePkg.private, true) + }) + it("runs only by manual dispatch against the exact trusted main revision", () => { + assert.match(workflow, /workflow_dispatch:/) + assert.match(workflow, /if: github.ref == 'refs\/heads\/main'/) + assert.match(workflow, /ref: \$\{\{ github.sha \}\}/) + assert.ok(!workflow.includes("pull_request_review:")) + assert.ok(!workflow.includes("pull_request_target:")) + assert.match(workflow, /persist-credentials: false/) + assert.match(workflow, /environment: final-vscode-review-smoke/) + }) + it("has one credential-bearing step, no GitHub write permissions, and no posting", () => { + assert.equal((workflow.match(/secrets\./g) ?? []).length, 1) + const before = workflow.slice(0, workflow.indexOf("- name: Run Solheim provider smoke")) + assert.ok(!before.includes("secrets.")) + assert.match(workflow, /permissions:\n\s+contents: read/) + assert.ok(!/\bwrite\b/.test(workflow)) + assert.ok(!workflow.includes("github-script")) + assert.ok(!workflow.includes("post-review")) + }) + it("has one smoke job and no review, routing, shadow, or deterministic lanes", () => { + const jobs = workflow.split("jobs:\n")[1] ?? "" + assert.deepEqual( + [...jobs.matchAll(/^ {4}([a-z-]+):$/gm)].map((match) => match[1]), + ["smoke"], + ) + for (const name of [ + "theme-review", + "probe-dispatch", + "rules-review", + "merge-report", + "bounded-code-review", + "model-qualification", + ]) + assert.ok(!workflow.includes(name)) + assert.ok(!driver.includes("model-qualification")) + assert.ok(!driver.includes("AcknowledgeCompletion")) + }) + it("bounds runtime and serializes the single provider instance", () => { + assert.match(workflow, /cancel-in-progress: false/) + assert.match(workflow, /timeout-minutes: 8/) + assert.match(workflow, /^ {8}timeout-minutes: 25$/m) + }) + it("uploads only the verdict and video, never host logs or storage", () => { + const paths = workflow.split("path: |\n")[1]?.split(" if-no-files-found:")[0] + assert.deepEqual( + paths + ?.trim() + .split("\n") + .map((line) => line.trim()), + [ + "${{ runner.temp }}/solheim-provider-smoke/verdict.json", + "${{ runner.temp }}/solheim-provider-smoke/smoke.mp4", + ], + ) + assert.match(workflow, /!cancelled\(\)/) + assert.match(workflow, /if-no-files-found: error/) + assert.ok(!driver.includes("console.error(")) + assert.ok(!driver.includes("stderrLog")) + }) + it("records a fresh display with credential-free ffmpeg and bounded teardown", () => { + assert.match(workflow, /install -y xvfb ffmpeg/) + assert.match(workflow, /xvfb-run -a -s "-screen 0 1280x720x24 -nolisten tcp"/) + assert.match(recorder, /env -i "PATH=\$PATH" "DISPLAY=\$DISPLAY"/) + assert.match(recorder, /-t 480 -fs 125829120/) + assert.match(recorder, /frag_keyframe\+empty_moov/) + assert.match(recorder, /trap finish EXIT/) + assert.match(recorder, /attempt < 50/) + assert.ok(!recorder.includes("SOLHEIM_API_KEY")) + assert.match(workflow, /retention-days: 7/) + }) + it("keeps Code QA and package commands focused on smoke contracts", () => { + for (const command of ["test:solheim-smoke:unit", "test:solheim-smoke-ci", "solheim-smoke:check-types"]) + assert.ok(codeQa.includes(command)) + assert.ok(!codeQa.includes("model-qualification")) + assert.ok(!Object.keys(pkg.scripts).some((name) => name.includes("model-qualification"))) + assert.ok(!codeQa.includes("final-smoke")) + assert.equal(pkg.scripts["solheim:smoke"], "node --import tsx scripts/solheim-smoke/driver.mts") + assert.ok(!fs.existsSync(new URL("../.github/workflows/final-vscode-review-smoke.yml", import.meta.url))) + }) +}) + +// Runs record.sh with stub ffmpeg and ffprobe, so no display or codec is needed. +function runRecorder(args, { ffmpegWrites = true, ffprobeOk = true } = {}) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "record-sh-")) + const bin = path.join(dir, "bin") + fs.mkdirSync(bin) + const stub = (name, body) => + fs.writeFileSync(path.join(bin, name), `#!/usr/bin/env bash\n${body}\n`, { mode: 0o755 }) + // The stub ffmpeg writes the last argument, stops on INT, and exits. + stub( + "ffmpeg", + `${ffmpegWrites ? 'for a; do :; done; echo data > "$a"' : ":"}\ntrap 'exit 0' INT\nwhile true; do sleep 0.05; done`, + ) + stub("ffprobe", ffprobeOk ? "exit 0" : "exit 1") + const result = spawnSync("bash", [new URL("./solheim-smoke/record.sh", import.meta.url).pathname, ...args], { + env: { PATH: `${bin}:${process.env.PATH}`, DISPLAY: ":99", SOLHEIM_SMOKE_OUT_DIR: path.join(dir, "out") }, + encoding: "utf8", + timeout: 20_000, + }) + fs.rmSync(dir, { recursive: true, force: true }) + return result +} + +describe("record.sh exit and validation behavior", () => { + it("passes when the command succeeds and the recording is valid", () => { + assert.equal(runRecorder(["true"]).status, 0) + }) + it("keeps the exit code of a failing command", () => { + assert.equal(runRecorder(["bash", "-c", "exit 7"]).status, 7) + }) + it("fails when the recording is missing", () => { + const result = runRecorder(["true"], { ffmpegWrites: false }) + assert.equal(result.status, 1) + assert.match(result.stdout, /recording failed/) + }) + it("fails when ffprobe rejects the recording", () => { + assert.equal(runRecorder(["true"], { ffprobeOk: false }).status, 1) + }) + it("keeps a failing command code when the recording is also invalid", () => { + assert.equal(runRecorder(["bash", "-c", "exit 7"], { ffprobeOk: false }).status, 7) + }) +}) diff --git a/scripts/solheim-smoke/README.md b/scripts/solheim-smoke/README.md new file mode 100644 index 0000000000..219ce3341e --- /dev/null +++ b/scripts/solheim-smoke/README.md @@ -0,0 +1,60 @@ +# Project A: Solheim provider smoke + +Infrastructure validation only: a real VS Code extension activates, accepts a correlated IPC task, +receives output from Solheim's OpenAI-compatible provider, and produces a completion result. +It does not read a PR, assess an answer's meaning, select checks, find defects, or post to GitHub. + +The active workflow is `.github/workflows/solheim-provider-smoke.yml`. It is manual-only, restricted to +`refs/heads/main`, and checks out the exact triggering SHA. It builds trusted extension code before the +single credential-bearing step. There is one job, no PR input or approval gate, no review/posting job, +and `contents: read` is the only GitHub permission. This deliberately validates the provider integration +on main, not arbitrary PR artifacts. It is not a required PR review check. + +The protected environment `final-vscode-review-smoke` and its secret `FINAL_SMOKE_OPENAI_API_KEY` +retain their existing names to avoid an unnecessary repository-settings migration. Restrict its deployment +policy to main and require approval. No environment or secret settings have been changed by this cleanup. + +The provider endpoint and model are fixed: `https://api.solheim.ai/v1`, `qwen3.8-27b`. The task has no tool +groups; delegation, follow-ups, writes, shell and MCP are disabled. The driver opens an empty temporary +workspace with isolated VS Code storage and an allowlisted host environment. The provider credential is +supplied over IPC, not inherited by the child or passed on its command line. Temporary extension storage +can contain the configured credential while the session runs. It is removed when no child remains alive; +if termination fails, storage cleanup is skipped and `gates.childExited` reports that separately from the +original smoke failure. Storage is never uploaded. +This is process/environment isolation, not a sandbox for malicious extension code—CI runs trusted main only. + +Activation and completion each have a 120-second limit; task-start acknowledgment has a 35-second limit, +in the controller. The extension awaits startup and configuration persistence without racing shared-profile +writes against a timer. If the controller times out, it tears down its isolated VS Code process rather than +trying to roll back shared settings or remove a possibly unrelated task. The live CI step has an eight-minute outer timeout. +Teardown escalates through close, terminate, and kill within independent five-second windows. A non-empty +completion and actual output-token usage must belong to the accepted task. Activation alone cannot pass. +The driver never approves dialogs or tool execution; it closes the task during teardown. + +The action uploads `verdict.json` (fixed failure code, boolean gates, duration and provider/model names) +and `smoke.mp4`, a 1280×720, 12 fps screen recording of the fresh CI virtual display. It records up to +eight minutes / approximately 120 MiB, with bounded shutdown and seven-day artifact retention. Video +contains the visible smoke UI, potentially including the fixed prompt and model answer; unlike the verdict, +it is not text-redacted. No settings screen is opened, and no host logs, storage, task configuration or +credential files are uploaded. Fragmented MP4 preserves completed fragments on abrupt interruption. +Ordinary success and failure runs upload both artifacts; cancellation may prevent finalization/upload. + +Checks, without a provider key or network calls: + +```sh +pnpm test:solheim-smoke:unit +pnpm test:solheim-smoke-ci +pnpm solheim-smoke:check-types +``` + +For an explicitly authorized local live run, build the extension/webview and supply `SOLHEIM_API_KEY` +through the environment, never an argument. Set `SOLHEIM_SMOKE_OUT_DIR` to a scratch directory and run +`pnpm solheim:smoke`. Linux needs Xvfb or an existing display. Do not run an untrusted extension with a key. +To record locally, install ffmpeg and run `xvfb-run -a -s "-screen 0 1280x720x24 -nolisten tcp" +bash scripts/solheim-smoke/record.sh` instead. Do not attach the recorder to your existing desktop. + +The new simplified driver/workflow has not yet had a real-provider CI run. Earlier question-lane runs +established the underlying integration, not this refactor's live status. Review qualification stopped +after the frozen Stage 1B pilot produced zero incremental verified defects. Its research harness and +receipts are archived outside this smoke-only change; the chronology, results, caveats and preservation +details are recorded in [tracking issue #1896](https://github.com/Zoo-Code-Org/Zoo-Code/issues/1896). diff --git a/scripts/solheim-smoke/config.test.ts b/scripts/solheim-smoke/config.test.ts new file mode 100644 index 0000000000..345062f0d3 --- /dev/null +++ b/scripts/solheim-smoke/config.test.ts @@ -0,0 +1,65 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { + buildChildEnvironment, + buildLaunchArgs, + buildSmokeConfiguration, + SOLHEIM_BASE_URL, + SOLHEIM_MODEL, + SMOKE_PROMPT, +} from "./config.ts" + +describe("Solheim provider smoke configuration", () => { + it("uses one fixed provider and a small output budget", () => { + const config = buildSmokeConfiguration("test-credential") + assert.equal(config.openAiBaseUrl, SOLHEIM_BASE_URL) + assert.equal(config.openAiModelId, SOLHEIM_MODEL) + assert.equal(config.openAiApiKey, "test-credential") + assert.equal(config.modelMaxTokens, 512) + assert.deepEqual(JSON.parse(config.openAiExtraBody ?? ""), { chat_template_kwargs: { enable_thinking: false } }) + }) + it("is a no-tools infrastructure task, with no delegation or reviewer instructions", () => { + const config = buildSmokeConfiguration("test-credential") + assert.deepEqual(config.customModes?.[0]?.groups, []) + assert.equal(config.mode, "provider-smoke") + for (const name of [ + "alwaysAllowWrite", + "alwaysAllowExecute", + "alwaysAllowMcp", + "alwaysAllowSubtasks", + "alwaysAllowReadOnly", + ] as const) + assert.equal(config[name], false) + assert.ok(config.disabledTools?.includes("new_task")) + assert.ok(config.disabledTools?.includes("ask_followup_question")) + assert.match(SMOKE_PROMPT, /not a code review/) + }) + it("passes only allowlisted host variables into the isolated extension host", () => { + const env = buildChildEnvironment( + { + PATH: "/bin", + DISPLAY: ":1", + SOLHEIM_API_KEY: "secret", + GITHUB_TOKEN: "token", + NODE_OPTIONS: "--require=bad.js", + XDG_CONFIG_HOME: "/real-home", + EXTRA_CREDENTIAL: "other", + }, + "/tmp/isolated", + "/tmp/ipc", + ) + assert.equal(env.PATH, "/bin") + assert.equal(env.DISPLAY, ":1") + assert.equal(env.HOME, "/tmp/isolated") + assert.equal(env.XDG_CONFIG_HOME, "/tmp/isolated/.config") + for (const name of ["SOLHEIM_API_KEY", "GITHUB_TOKEN", "NODE_OPTIONS", "EXTRA_CREDENTIAL"]) + assert.equal(env[name], undefined) + }) + it("uses isolated storage and a keyring-free launch", () => { + const args = buildLaunchArgs("/empty-workspace", "/data", "/extensions", "/extension") + assert.equal(args[0], "/empty-workspace") + assert.ok(args.includes("--password-store=basic")) + assert.ok(args.includes("--extensionDevelopmentPath=/extension")) + assert.ok(args.includes("--user-data-dir=/data")) + }) +}) diff --git a/scripts/solheim-smoke/config.ts b/scripts/solheim-smoke/config.ts new file mode 100644 index 0000000000..7b19187cdc --- /dev/null +++ b/scripts/solheim-smoke/config.ts @@ -0,0 +1,84 @@ +import path from "node:path" +import type { RooCodeSettings } from "@roo-code/types" + +export const SOLHEIM_BASE_URL = "https://api.solheim.ai/v1" +export const SOLHEIM_MODEL = "qwen3.8-27b" +export const SMOKE_TIMEOUT_MS = 120_000 +export const SMOKE_PROMPT = + "This is a provider connectivity smoke test, not a code review. Call attempt_completion once with the result: Provider smoke completed. Do not read files, use other tools, delegate, or judge code." + +export function buildSmokeConfiguration(apiKey: string): RooCodeSettings { + return { + apiProvider: "openai", + openAiBaseUrl: SOLHEIM_BASE_URL, + openAiApiKey: apiKey, + openAiModelId: SOLHEIM_MODEL, + openAiR1FormatEnabled: true, + openAiExtraBody: JSON.stringify({ chat_template_kwargs: { enable_thinking: false } }), + modelMaxTokens: 512, + autoCondenseContext: false, + mode: "provider-smoke", + customModes: [ + { + slug: "provider-smoke", + name: "Provider smoke", + roleDefinition: "Complete one connectivity smoke task. You are not a reviewer.", + groups: [], + }, + ], + disabledTools: [ + "ask_followup_question", + "new_task", + "switch_mode", + "update_todo_list", + "skill", + "run_slash_command", + ], + autoApprovalEnabled: true, + alwaysAllowReadOnly: false, + alwaysAllowReadOnlyOutsideWorkspace: false, + alwaysAllowWrite: false, + alwaysAllowWriteOutsideWorkspace: false, + alwaysAllowWriteProtected: false, + alwaysAllowExecute: false, + alwaysAllowMcp: false, + alwaysAllowModeSwitch: false, + alwaysAllowSubtasks: false, + alwaysAllowFollowupQuestions: false, + } +} + +// Inherit only host-launch necessities, never the credential-bearing parent environment. +export function buildChildEnvironment( + env: NodeJS.ProcessEnv, + isolatedHome: string, + socketPath: string, +): NodeJS.ProcessEnv { + const child: NodeJS.ProcessEnv = {} + for (const name of ["PATH", "DISPLAY", "XAUTHORITY", "LANG", "LC_ALL", "TMPDIR", "SYSTEMROOT", "WINDIR"]) + if (env[name]) child[name] = env[name] + return { + ...child, + HOME: isolatedHome, + USERPROFILE: isolatedHome, + XDG_CONFIG_HOME: path.join(isolatedHome, ".config"), + XDG_CACHE_HOME: path.join(isolatedHome, ".cache"), + ROO_CODE_IPC_SOCKET_PATH: socketPath, + } +} + +export function buildLaunchArgs(workspace: string, userData: string, extensions: string, extension: string): string[] { + return [ + workspace, + `--user-data-dir=${userData}`, + `--extensions-dir=${extensions}`, + `--extensionDevelopmentPath=${extension}`, + "--no-sandbox", + "--disable-gpu-sandbox", + "--password-store=basic", + "--disable-updates", + "--skip-welcome", + "--skip-release-notes", + "--disable-workspace-trust", + ] +} diff --git a/scripts/solheim-smoke/driver.mts b/scripts/solheim-smoke/driver.mts new file mode 100644 index 0000000000..383e2db206 --- /dev/null +++ b/scripts/solheim-smoke/driver.mts @@ -0,0 +1,121 @@ +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { fileURLToPath } from "node:url" +import { spawn } from "node:child_process" +import { downloadAndUnzipVSCode } from "@vscode/test-electron" +import { IpcClient } from "@roo-code/ipc" +import { IpcMessageType } from "@roo-code/types" +import { buildChildEnvironment, buildLaunchArgs } from "./config.ts" +import { runSmoke, smokeExitCode, type SmokeRuntime } from "./run.ts" + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") +const out = process.env.SOLHEIM_SMOKE_OUT_DIR +if (!out) throw new Error("Set SOLHEIM_SMOKE_OUT_DIR to a scratch directory") +await fs.mkdir(out, { recursive: true }) +let temp: string | undefined +let socket = "" +const runtime: SmokeRuntime = { + async launch() { + const extension = path.join(root, "src") + await fs.access(path.join(extension, "package.json")) + temp = await fs.mkdtemp(path.join(os.tmpdir(), "solheim-smoke-")) + const workspace = path.join(temp, "workspace") + const isolatedHome = path.join(temp, "home") + await Promise.all([workspace, isolatedHome].map((folder) => fs.mkdir(folder))) + socket = path.join(temp, "ipc.sock") + const pkg = JSON.parse(await fs.readFile(path.join(root, "apps/vscode-e2e/package.json"), "utf8")) as { + devDependencies: Record + } + const executable = await downloadAndUnzipVSCode({ + version: pkg.devDependencies["@types/vscode"] ?? "1.100.0", + cachePath: path.join(root, "apps/vscode-e2e/.vscode-test"), + }) + const args = buildLaunchArgs(workspace, path.join(temp, "userdata"), path.join(temp, "extensions"), extension) + const child = spawn( + process.env.DISPLAY ? executable : "xvfb-run", + process.env.DISPLAY ? args : ["-a", executable, ...args], + { + detached: true, + env: buildChildEnvironment(process.env, isolatedHome, socket), + stdio: "ignore", + }, + ) + let exited = false + let onExit = () => {} + const closed = () => { + exited = true + onExit() + } + child.once("error", () => { + // Spawn failure has no live PID; a failed kill must not claim exit. + if (!child.pid) closed() + else onExit() + }) + child.once("close", closed) + return { + get exited() { + return exited + }, + onExit(listener) { + onExit = listener + if (exited) listener() + }, + signal(signal) { + if (!child.pid || exited) return + try { + process.kill(-child.pid, signal) + } catch { + child.kill(signal) + } + }, + } + }, + connect() { + const client = new IpcClient(socket, () => {}) + return { + get isReady() { + return client.isReady + }, + onTaskEvent(listener) { + client.on(IpcMessageType.TaskEvent, listener) + }, + onDisconnect(listener) { + client.on(IpcMessageType.Disconnect, listener) + }, + sendCommand(command) { + client.sendCommand(command) + }, + disconnect() { + client.disconnect() + }, + } + }, + async cleanup() { + if (temp) await fs.rm(temp, { recursive: true, force: true }) + }, + async writeVerdict(verdict) { + // Atomic, whitelisted metadata only; no provider text or host logs. + const pending = path.join(out, `verdict.json.${process.pid}.tmp`) + await fs.writeFile(pending, JSON.stringify(verdict, null, 2)) + await fs.rename(pending, path.join(out, "verdict.json")) + }, + onInterrupt(listener) { + process.once("SIGINT", listener) + process.once("SIGTERM", listener) + return () => { + process.removeListener("SIGINT", listener) + process.removeListener("SIGTERM", listener) + } + }, + now: Date.now, + pause: (ms) => new Promise((resolve) => setTimeout(resolve, ms)), +} +try { + const verdict = await runSmoke(runtime, process.env.SOLHEIM_API_KEY ?? "") + console.log(`Solheim provider smoke: ${verdict.code}`) + process.exit(smokeExitCode(verdict)) +} catch { + console.log("Solheim provider smoke: SETUP_FAILED") + process.exit(1) +} diff --git a/scripts/solheim-smoke/evidence.test.ts b/scripts/solheim-smoke/evidence.test.ts new file mode 100644 index 0000000000..b3be3faf4f --- /dev/null +++ b/scripts/solheim-smoke/evidence.test.ts @@ -0,0 +1,91 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { RooCodeEventName, type TaskEvent } from "@roo-code/types" +import { collectEvidence, smokeCode, type SmokeEvidence } from "./evidence.ts" +const fresh = (): SmokeEvidence => ({ + providerResponseSeen: false, + completionSeen: false, + unexpectedTool: false, + aborted: false, +}) +const usage = ( + taskId: string, + totalTokensOut: number, +): Extract => ({ + eventName: RooCodeEventName.TaskTokenUsageUpdated, + payload: [taskId, { totalTokensIn: 1, totalTokensOut, totalCost: 0, contextTokens: 1 }, {}], +}) +const message = (taskId: string, value: Record) => ({ + eventName: RooCodeEventName.Message, + payload: [{ taskId, action: "created", message: { ts: 1, ...value } }], +}) + +describe("provider smoke evidence", () => { + it("rejects malformed and uncorrelated events, including activity before task acceptance", () => { + const state = fresh() + collectEvidence(state, usage("root", 5), undefined) + collectEvidence(state, usage("other", 5), "root") + collectEvidence( + state, + { eventName: RooCodeEventName.TaskTokenUsageUpdated, payload: ["root", { totalTokensOut: "5" }, {}] }, + "root", + ) + assert.deepEqual(state, fresh()) + }) + it("requires actual output usage; echoed task text is not provider evidence", () => { + const state = fresh() + collectEvidence(state, message("root", { type: "say", say: "text", text: "task prompt" }), "root") + collectEvidence(state, usage("root", 0), "root") + assert.equal(state.providerResponseSeen, false) + collectEvidence(state, usage("root", 5), "root") + assert.equal(state.providerResponseSeen, true) + }) + it("accepts any non-empty completed response, without reviewing its meaning", () => { + const state = fresh() + collectEvidence( + state, + message("root", { type: "say", say: "completion_result", text: "any completion", partial: true }), + "root", + ) + assert.equal(state.completionSeen, false) + collectEvidence( + state, + message("root", { type: "say", say: "completion_result", text: "any completion" }), + "root", + ) + assert.equal(state.completionSeen, true) + assert.ok(!JSON.stringify(state).includes("any completion")) + }) + it("allows the final completion dialog but rejects other tools", () => { + const state = fresh() + collectEvidence(state, message("root", { type: "ask", ask: "completion_result" }), "root") + assert.equal(state.completionSeen, false) + assert.equal(state.unexpectedTool, false) + collectEvidence(state, message("other", { type: "ask", ask: "command", text: "ignored" }), "root") + assert.equal(state.unexpectedTool, false) + collectEvidence(state, message("root", { type: "ask", ask: "command", text: "never stored" }), "root") + assert.equal(state.unexpectedTool, true) + const noAsk = fresh() + const attempted = usage("root", 5) + attempted.payload[2] = { execute_command: { attempts: 1, failures: 1 } } + collectEvidence(noAsk, attempted, "root") + assert.equal(noAsk.unexpectedTool, true) + }) + it("cannot pass on activation alone or ignore a lost session", () => { + assert.equal(smokeCode(fresh(), false), "PROVIDER_TIMEOUT") + assert.equal(smokeCode({ ...fresh(), providerResponseSeen: true }, false), "COMPLETION_MISSING") + assert.equal(smokeCode({ ...fresh(), providerResponseSeen: true, completionSeen: true }, false), "OK") + assert.equal(smokeCode({ ...fresh(), providerResponseSeen: true, completionSeen: true }, true), "SESSION_LOST") + assert.equal( + smokeCode({ ...fresh(), providerResponseSeen: true, completionSeen: true, unexpectedTool: true }, false), + "UNEXPECTED_TOOL", + ) + }) + it("rejects delegation and aborts on the accepted root", () => { + const state = fresh() + collectEvidence(state, { eventName: RooCodeEventName.TaskDelegated, payload: ["root", "child"] }, "root") + assert.equal(state.unexpectedTool, true) + collectEvidence(state, { eventName: RooCodeEventName.TaskAborted, payload: ["root"] }, "root") + assert.equal(smokeCode(state, false), "SESSION_LOST") + }) +}) diff --git a/scripts/solheim-smoke/evidence.ts b/scripts/solheim-smoke/evidence.ts new file mode 100644 index 0000000000..3e436ca8c4 --- /dev/null +++ b/scripts/solheim-smoke/evidence.ts @@ -0,0 +1,62 @@ +import { RooCodeEventName, taskEventSchema, type TaskEvent } from "@roo-code/types" + +export type SmokeEvidence = { + providerResponseSeen: boolean + completionSeen: boolean + unexpectedTool: boolean + aborted: boolean +} + +// Only schema-valid events for the accepted root task can satisfy smoke gates. +// No model text, credentials, tool arguments, or task IDs are persisted. +export function collectEvidence(state: SmokeEvidence, raw: unknown, rootTaskId: string | undefined): void { + if (!rootTaskId) return + const parsed = taskEventSchema.safeParse(raw) + if (!parsed.success) return + const event: TaskEvent = parsed.data + if (event.eventName === RooCodeEventName.Message) { + const entry = event.payload[0] + if (entry.taskId !== rootTaskId || entry.message.partial === true) return + const message = entry.message + if (message.type === "say" && message.say === "completion_result" && message.text?.trim()) + state.completionSeen = true + if (message.type === "ask" && message.ask !== "completion_result") state.unexpectedTool = true + } else if (event.payload[0] === rootTaskId) { + if ( + event.eventName === RooCodeEventName.TaskTokenUsageUpdated || + event.eventName === RooCodeEventName.TaskCompleted + ) { + if ( + Object.entries(event.payload[2]).some( + ([tool, usage]) => tool !== "attempt_completion" && usage.attempts > 0, + ) + ) + state.unexpectedTool = true + } + if (event.eventName === RooCodeEventName.TaskTokenUsageUpdated && event.payload[1].totalTokensOut > 0) + state.providerResponseSeen = true + if (event.eventName === RooCodeEventName.TaskCompleted && event.payload[1].totalTokensOut > 0) + state.providerResponseSeen = true + if (event.eventName === RooCodeEventName.TaskAborted) state.aborted = true + if (event.eventName === RooCodeEventName.TaskDelegated || event.eventName === RooCodeEventName.TaskSpawned) + state.unexpectedTool = true + } +} + +export type SmokeCode = + | "OK" + | "SETUP_FAILED" + | "ACTIVATION_FAILED" + | "TASK_START_FAILED" + | "SESSION_LOST" + | "UNEXPECTED_TOOL" + | "PROVIDER_TIMEOUT" + | "COMPLETION_MISSING" + | "TEARDOWN_FAILED" +export function smokeCode(state: SmokeEvidence, sessionLost: boolean): SmokeCode { + if (sessionLost || state.aborted) return "SESSION_LOST" + if (state.unexpectedTool) return "UNEXPECTED_TOOL" + if (!state.providerResponseSeen) return "PROVIDER_TIMEOUT" + if (!state.completionSeen) return "COMPLETION_MISSING" + return "OK" +} diff --git a/scripts/solheim-smoke/package.json b/scripts/solheim-smoke/package.json new file mode 100644 index 0000000000..c4fb2812b9 --- /dev/null +++ b/scripts/solheim-smoke/package.json @@ -0,0 +1,4 @@ +{ + "private": true, + "type": "module" +} diff --git a/scripts/solheim-smoke/record.sh b/scripts/solheim-smoke/record.sh new file mode 100644 index 0000000000..f69e17b17f --- /dev/null +++ b/scripts/solheim-smoke/record.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Run inside a fresh xvfb-run display, never record an existing desktop. +set -euo pipefail +: "${DISPLAY:?Run this wrapper under xvfb-run}" +: "${SOLHEIM_SMOKE_OUT_DIR:?Set a scratch output directory}" +mkdir -p "$SOLHEIM_SMOKE_OUT_DIR" +recording="$SOLHEIM_SMOKE_OUT_DIR/smoke.mp4" + +# The recorder needs display access, not provider/GitHub credentials. Fragmented +# MP4 keeps completed fragments readable if the action is forcibly interrupted. +env -i "PATH=$PATH" "DISPLAY=$DISPLAY" "XAUTHORITY=${XAUTHORITY:-}" \ + ffmpeg -nostdin -hide_banner -loglevel error -y \ + -f x11grab -video_size 1280x720 -framerate 12 -i "$DISPLAY" \ + -an -c:v libx264 -preset ultrafast -crf 28 -pix_fmt yuv420p -g 24 \ + -t 480 -fs 125829120 -movflags +frag_keyframe+empty_moov \ + "$recording" >/dev/null 2>&1 & +recorder_pid=$! + +finish() { + result=$? + trap - EXIT + kill -INT "$recorder_pid" 2>/dev/null || true + # A hung recorder must not hold the action open indefinitely. + for ((attempt = 0; attempt < 50; attempt++)); do + kill -0 "$recorder_pid" 2>/dev/null || break + sleep 0.1 + done + kill -KILL "$recorder_pid" 2>/dev/null || true + wait "$recorder_pid" 2>/dev/null || true + if [[ ! -s "$recording" ]] || ! env -i "PATH=$PATH" \ + ffprobe -v error "$recording" >/dev/null 2>&1; then + echo "Solheim smoke recording failed" + [[ "$result" -ne 0 ]] || result=1 + fi + exit "$result" +} +trap finish EXIT +trap 'exit 130' INT +trap 'exit 143' TERM +sleep 0.5 +kill -0 "$recorder_pid" 2>/dev/null || exit 1 + +# Arguments are used for credential-free recorder tests; CI supplies none. +if [[ "$#" -gt 0 ]]; then "$@"; else pnpm solheim:smoke; fi diff --git a/scripts/solheim-smoke/run.test.ts b/scripts/solheim-smoke/run.test.ts new file mode 100644 index 0000000000..1c64ec53e4 --- /dev/null +++ b/scripts/solheim-smoke/run.test.ts @@ -0,0 +1,245 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { RooCodeEventName, TaskCommandName, type TaskCommand } from "@roo-code/types" +import { runSmoke, smokeExitCode, type SmokeRuntime, type SmokeVerdict } from "./run.ts" + +type Scenario = + | "success" + | "activation" + | "start-failure" + | "no-provider" + | "no-completion" + | "tool" + | "disconnect" + | "uncorrelated" + | "aborted" + | "setup" +function fixture( + scenario: Scenario = "success", + options: { stuck?: boolean; closeThrows?: boolean; timeout?: () => void } = {}, +) { + let now = 0 + let exited = false + let onExit = () => {} + let onDisconnect = () => {} + let onEvent: (event: unknown) => void = () => {} + let interrupt = () => {} + let cleaned = false + let disconnected = false + let unsubscribed = false + const commands: TaskCommand[] = [] + const signals: NodeJS.Signals[] = [] + const written: SmokeVerdict[] = [] + const close = () => { + exited = true + onExit() + } + const runtime: SmokeRuntime = { + async launch() { + if (scenario === "setup") throw new Error("private setup detail") + return { + get exited() { + return exited + }, + onExit(listener) { + onExit = listener + }, + signal(signal) { + signals.push(signal) + if (!options.stuck) close() + }, + } + }, + connect() { + return { + isReady: scenario !== "activation", + onTaskEvent(listener) { + onEvent = listener + }, + onDisconnect(listener) { + onDisconnect = listener + }, + disconnect() { + disconnected = true + }, + sendCommand(command) { + commands.push(command) + if (command.commandName === TaskCommandName.CloseTask) { + if (options.closeThrows) throw new Error("private close error") + if (!options.stuck) close() + return + } + if (command.commandName !== TaskCommandName.StartNewTask) return + if (options.timeout) { + options.timeout() + return + } + onEvent({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [ + scenario === "start-failure" + ? { + requestId: command.data.requestId, + success: false, + errorCode: "task_start_failed", + errorMessage: "Task start failed", + } + : { requestId: command.data.requestId, success: true, taskId: "root" }, + ], + }) + const taskId = scenario === "uncorrelated" ? "other" : "root" + if (scenario !== "no-provider") + onEvent({ + eventName: RooCodeEventName.TaskTokenUsageUpdated, + payload: [ + taskId, + { totalTokensIn: 1, totalTokensOut: 5, totalCost: 0, contextTokens: 1 }, + {}, + ], + }) + if (scenario !== "no-completion") + onEvent({ + eventName: RooCodeEventName.Message, + payload: [ + { + taskId, + action: "created", + message: { + ts: 1, + type: "say", + say: "completion_result", + text: "private synthetic answer", + }, + }, + ], + }) + if (scenario === "tool") + onEvent({ + eventName: RooCodeEventName.TaskSpawned, + payload: ["root", "child"], + }) + if (scenario === "disconnect") onDisconnect() + if (scenario === "aborted") onEvent({ eventName: RooCodeEventName.TaskAborted, payload: ["root"] }) + }, + } + }, + async cleanup() { + cleaned = true + }, + async writeVerdict(verdict) { + written.push(verdict) + }, + onInterrupt(listener) { + interrupt = listener + return () => { + unsubscribed = true + } + }, + now: () => now, + async pause(ms) { + now += ms + }, + } + return { + runtime, + commands, + signals, + written, + interrupt: () => interrupt(), + get cleaned() { + return cleaned + }, + get disconnected() { + return disconnected + }, + get unsubscribed() { + return unsubscribed + }, + } +} + +describe("provider smoke orchestration", () => { + it("correlates startup and evidence, writes a metadata-only verdict, and exits zero", async () => { + const f = fixture() + const verdict = await runSmoke(f.runtime, "private-test-key") + assert.equal(verdict.code, "OK") + assert.equal(smokeExitCode(verdict), 0) + assert.deepEqual(verdict.gates, { + extensionActivated: true, + ipcConnected: true, + taskAccepted: true, + providerResponseSeen: true, + completionSeen: true, + childExited: true, + }) + assert.deepEqual(f.written, [verdict]) + assert.ok(f.cleaned && f.disconnected && f.unsubscribed) + assert.equal(f.commands.at(-1)?.commandName, TaskCommandName.CloseTask) + assert.ok(!JSON.stringify(verdict).includes("private")) + }) + for (const [scenario, code] of [ + ["activation", "ACTIVATION_FAILED"], + ["start-failure", "TASK_START_FAILED"], + ["no-provider", "PROVIDER_TIMEOUT"], + ["no-completion", "COMPLETION_MISSING"], + ["tool", "UNEXPECTED_TOOL"], + ["disconnect", "SESSION_LOST"], + ["uncorrelated", "PROVIDER_TIMEOUT"], + ["aborted", "SESSION_LOST"], + ["setup", "SETUP_FAILED"], + ] as const) { + it(`fails closed for ${scenario} and still writes the verdict`, async () => { + const f = fixture(scenario) + const verdict = await runSmoke(f.runtime, "private-test-key") + assert.equal(verdict.code, code) + assert.equal(verdict.passed, false) + assert.equal(smokeExitCode(verdict), 1) + assert.deepEqual(f.written, [verdict]) + assert.ok(f.cleaned && f.unsubscribed) + }) + } + it("bounds startup at the controller and closes its host when no reply arrives", async (t) => { + t.mock.timers.enable({ apis: ["setTimeout"] }) + const f = fixture("success", { timeout: () => t.mock.timers.tick(35_000) }) + const verdict = await runSmoke(f.runtime, "private-test-key") + assert.equal(verdict.code, "TASK_START_FAILED") + assert.equal(verdict.gates.taskAccepted, false) + assert.ok(f.cleaned) + }) + it("bounds TERM/KILL teardown and keeps live-host storage out of cleanup", async () => { + const f = fixture("success", { stuck: true }) + const verdict = await runSmoke(f.runtime, "private-test-key") + assert.equal(verdict.code, "TEARDOWN_FAILED") + assert.equal(verdict.durationMs, 15_000) + assert.deepEqual(f.signals, ["SIGTERM", "SIGKILL"]) + assert.equal(verdict.gates.childExited, false) + assert.equal(f.cleaned, false) + assert.equal(f.disconnected, true) + }) + it("preserves the original failure but reports failed teardown separately", async () => { + const f = fixture("start-failure", { stuck: true }) + const verdict = await runSmoke(f.runtime, "private-test-key") + assert.equal(verdict.code, "TASK_START_FAILED") + assert.equal(verdict.gates.childExited, false) + assert.equal(f.cleaned, false) + }) + it("interrupts an evidence wait, terminates only its host, and fails closed", async () => { + const f = fixture("no-completion") + const pause = f.runtime.pause + f.runtime.pause = async (ms) => { + f.interrupt() + await pause(ms) + } + const verdict = await runSmoke(f.runtime, "private-test-key") + assert.equal(verdict.code, "SESSION_LOST") + assert.deepEqual(f.signals, ["SIGTERM"]) + assert.ok(f.cleaned && f.unsubscribed) + }) + it("continues host termination and writes a failing verdict after a CloseTask exception", async () => { + const f = fixture("success", { closeThrows: true }) + const verdict = await runSmoke(f.runtime, "private-test-key") + assert.equal(verdict.code, "TEARDOWN_FAILED") + assert.deepEqual(f.signals, ["SIGTERM"]) + assert.ok(f.cleaned && f.disconnected) + }) +}) diff --git a/scripts/solheim-smoke/run.ts b/scripts/solheim-smoke/run.ts new file mode 100644 index 0000000000..698216598b --- /dev/null +++ b/scripts/solheim-smoke/run.ts @@ -0,0 +1,167 @@ +import { randomUUID } from "node:crypto" +import { TaskCommandName, type TaskCommand } from "@roo-code/types" +import { buildSmokeConfiguration, SMOKE_PROMPT, SMOKE_TIMEOUT_MS, SOLHEIM_BASE_URL, SOLHEIM_MODEL } from "./config.ts" +import { collectEvidence, smokeCode, type SmokeCode, type SmokeEvidence } from "./evidence.ts" +import { createTaskStartWaiter } from "./task-start.ts" + +export interface SmokeHost { + readonly exited: boolean + onExit(listener: () => void): void + signal(signal: NodeJS.Signals): void +} +export interface SmokeClient { + readonly isReady: boolean + onTaskEvent(listener: (event: unknown) => void): void + onDisconnect(listener: () => void): void + sendCommand(command: TaskCommand): void + disconnect(): void +} +export type SmokeVerdict = { + kind: "provider-smoke" + model: string + provider: string + passed: boolean + code: SmokeCode + gates: { + extensionActivated: boolean + ipcConnected: boolean + taskAccepted: boolean + providerResponseSeen: boolean + completionSeen: boolean + childExited: boolean + } + durationMs: number +} +export interface SmokeRuntime { + launch(): Promise + connect(): SmokeClient + cleanup(): Promise + writeVerdict(verdict: SmokeVerdict): Promise + onInterrupt(listener: () => void): () => void + now(): number + pause(ms: number): Promise +} + +// Only the adapter can spawn VS Code, read credentials or write files. Tests +// exercise the same controller with synthetic IPC and a deterministic clock. +export async function runSmoke(runtime: SmokeRuntime, apiKey: string): Promise { + const started = runtime.now() + const evidence: SmokeEvidence = { + providerResponseSeen: false, + completionSeen: false, + unexpectedTool: false, + aborted: false, + } + let code: SmokeCode = "SETUP_FAILED" + let activated = false + let accepted = false + let sessionLost = false + let host: SmokeHost | undefined + let client: SmokeClient | undefined + let waiter: ReturnType | undefined + async function until(check: () => boolean, timeoutMs: number, stopOnLoss = true): Promise { + const deadline = runtime.now() + timeoutMs + while (!check() && (!stopOnLoss || !sessionLost) && runtime.now() < deadline) await runtime.pause(100) + } + const markLost = () => { + sessionLost = true + waiter?.onExit() + } + const unsubscribe = runtime.onInterrupt(() => { + markLost() + try { + host?.signal("SIGTERM") + } catch { + /* Teardown retries independently. */ + } + }) + try { + if (!apiKey) throw new Error("Provider credential missing") + host = await runtime.launch() + host.onExit(markLost) + client = runtime.connect() + client.onDisconnect(() => { + sessionLost = true + waiter?.onDisconnect() + }) + let taskId: string | undefined + client.onTaskEvent((event) => { + const id = waiter?.onTaskEvent(event) + if (id && taskId === undefined) taskId = id + collectEvidence(evidence, event, taskId) + }) + await until(() => client?.isReady === true, SMOKE_TIMEOUT_MS) + activated = client.isReady + if (!activated) code = "ACTIVATION_FAILED" + else { + const requestId = randomUUID() + waiter = createTaskStartWaiter({ requestId, timeoutMs: 35_000 }) + // A synchronous send failure must not leave a rejected timer promise. + void waiter.result.catch(() => {}) + client.sendCommand({ + commandName: TaskCommandName.StartNewTask, + data: { requestId, text: SMOKE_PROMPT, newTab: false, configuration: buildSmokeConfiguration(apiKey) }, + }) + try { + taskId = await waiter.result + accepted = true + } catch { + code = "TASK_START_FAILED" + } + if (accepted) { + await until( + () => + evidence.unexpectedTool || + evidence.aborted || + (evidence.providerResponseSeen && evidence.completionSeen), + SMOKE_TIMEOUT_MS, + ) + code = smokeCode(evidence, sessionLost) + } + } + } catch { + // Never expose thrown provider, filesystem or host messages. + } finally { + waiter?.onExit() + const attempt = async (operation: () => unknown) => { + try { + await operation() + } catch { + if (code === "OK") code = "TEARDOWN_FAILED" + } + } + if (client?.isReady) await attempt(() => client?.sendCommand({ commandName: TaskCommandName.CloseTask })) + if (host && !host.exited) { + await attempt(() => until(() => host?.exited === true, 5_000, false)) + for (const signal of ["SIGTERM", "SIGKILL"] as const) { + if (host.exited) break + await attempt(() => host?.signal(signal)) + await attempt(() => until(() => host?.exited === true, 5_000, false)) + } + if (!host.exited && code === "OK") code = "TEARDOWN_FAILED" + } + await attempt(() => client?.disconnect()) + if (!host || host.exited) await attempt(() => runtime.cleanup()) + await attempt(unsubscribe) + } + const verdict: SmokeVerdict = { + kind: "provider-smoke", + model: SOLHEIM_MODEL, + provider: SOLHEIM_BASE_URL, + passed: code === "OK", + code, + gates: { + extensionActivated: activated, + ipcConnected: activated, + taskAccepted: accepted, + providerResponseSeen: evidence.providerResponseSeen, + completionSeen: evidence.completionSeen, + childExited: !host || host.exited, + }, + durationMs: runtime.now() - started, + } + await runtime.writeVerdict(verdict) + return verdict +} + +export const smokeExitCode = (verdict: SmokeVerdict): number => (verdict.passed ? 0 : 1) diff --git a/scripts/solheim-smoke/task-start.test.ts b/scripts/solheim-smoke/task-start.test.ts new file mode 100644 index 0000000000..58ebb91fac --- /dev/null +++ b/scripts/solheim-smoke/task-start.test.ts @@ -0,0 +1,215 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" + +import { RooCodeEventName, TASK_START_FAILURE_ERROR_CODE, TASK_START_FAILURE_ERROR_MESSAGE } from "@roo-code/types" + +import { createTaskStartWaiter } from "./task-start.ts" + +const REQUEST_ID = "final-smoke-0123abcdef456789" + +const startResponse = (payload: unknown): unknown => ({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [payload], +}) + +const successPayload = (requestId = REQUEST_ID): unknown => ({ + requestId, + success: true, + taskId: "accepted-task-1", +}) + +const failurePayload = (requestId = REQUEST_ID): Record => ({ + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, +}) + +const settledOutcome = async (promise: Promise): Promise<"resolved" | "rejected" | "pending"> => { + const probe = promise.then( + () => "resolved" as const, + () => "rejected" as const, + ) + return Promise.race([probe, new Promise<"pending">((resolve) => setTimeout(() => resolve("pending"), 10))]) +} + +describe("createTaskStartWaiter", () => { + it("resolves with the accepted task ID on a matching success response", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 1_000 }) + + waiter.onTaskEvent(startResponse(successPayload())) + + assert.equal(await waiter.result, "accepted-task-1") + }) + + it("rejects with the fixed sanitized failure values on a matching failure response", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 1_000 }) + + waiter.onTaskEvent(startResponse(failurePayload())) + + await assert.rejects(waiter.result, (error: Error) => { + assert.equal(error.message, `${TASK_START_FAILURE_ERROR_CODE}: ${TASK_START_FAILURE_ERROR_MESSAGE}`) + return true + }) + }) + + it("rejects with the stage in the sanitized error on a failure response carrying a stage", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 1_000 }) + + waiter.onTaskEvent( + startResponse({ + ...failurePayload(), + stage: "taskCreation", + }), + ) + + await assert.rejects(waiter.result, (error: Error) => { + assert.equal( + error.message, + `${TASK_START_FAILURE_ERROR_CODE}: ${TASK_START_FAILURE_ERROR_MESSAGE} (stage=taskCreation)`, + ) + return true + }) + }) + + it("rejects with the stage unknown in the sanitized error when the response names it", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 1_000 }) + + waiter.onTaskEvent( + startResponse({ + ...failurePayload(), + stage: "unknown", + }), + ) + + await assert.rejects(waiter.result, (error: Error) => { + assert.match(error.message, /\(stage=unknown\)$/) + return true + }) + }) + + it("ignores a failure response carrying an unknown stage value", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 1_000 }) + + waiter.onTaskEvent( + startResponse({ + ...failurePayload(), + stage: "secretStore", + }), + ) + assert.equal(await settledOutcome(waiter.result), "pending") + + waiter.onTaskEvent(startResponse(failurePayload())) + await assert.rejects(waiter.result, /task_start_failed/) + }) + + it("ignores responses for a different requestId", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 1_000 }) + + waiter.onTaskEvent(startResponse(successPayload("some-other-request"))) + assert.equal(await settledOutcome(waiter.result), "pending") + + waiter.onTaskEvent(startResponse(successPayload())) + assert.equal(await waiter.result, "accepted-task-1") + }) + + it("ignores malformed and non-start events", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 1_000 }) + + waiter.onTaskEvent({ eventName: RooCodeEventName.TaskCreated, payload: ["task-x"] }) + waiter.onTaskEvent(startResponse({ requestId: REQUEST_ID, success: true })) + waiter.onTaskEvent(startResponse("not-an-object")) + waiter.onTaskEvent({ eventName: RooCodeEventName.TaskStartResponse }) + waiter.onTaskEvent(undefined) + assert.equal(await settledOutcome(waiter.result), "pending") + + waiter.onTaskEvent(startResponse(successPayload())) + assert.equal(await waiter.result, "accepted-task-1") + }) + + it("ignores a success response carrying extra top-level fields", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 1_000 }) + + waiter.onTaskEvent( + startResponse({ + requestId: REQUEST_ID, + success: true, + taskId: "accepted-task-1", + providerError: "key sk-secret-123 rejected", + }), + ) + assert.equal(await settledOutcome(waiter.result), "pending") + + waiter.onTaskEvent(startResponse(successPayload())) + assert.equal(await waiter.result, "accepted-task-1") + }) + + it("cannot settle on a failure response carrying extra top-level fields", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 1_000 }) + + waiter.onTaskEvent( + startResponse({ + requestId: REQUEST_ID, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + detail: { apiKey: "sk-secret-123", url: "https://api.example.com" }, + }), + ) + assert.equal(await settledOutcome(waiter.result), "pending") + + waiter.onTaskEvent(startResponse(failurePayload())) + await assert.rejects(waiter.result, /task_start_failed/) + }) + + it("settles only once: later responses cannot change the outcome", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 1_000 }) + + waiter.onTaskEvent(startResponse(failurePayload())) + await assert.rejects(waiter.result, /task_start_failed/) + + assert.equal(waiter.onTaskEvent(startResponse(successPayload())), undefined) + waiter.onDisconnect() + waiter.onExit() + await assert.rejects(waiter.result, /task_start_failed/) + }) + + it("rejects on timeout", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 10 }) + + await assert.rejects(waiter.result, (error: Error) => { + assert.match(error.message, /task start response missing after 10ms/) + assert.match(error.message, new RegExp(REQUEST_ID)) + return true + }) + }) + + it("rejects on IPC disconnect", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 10_000 }) + + waiter.onDisconnect() + + await assert.rejects(waiter.result, /IPC disconnected before the task start response arrived/) + }) + + it("rejects on VS Code exit", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 10_000 }) + + waiter.onExit() + + await assert.rejects(waiter.result, /VS Code exited before the task start response arrived/) + }) + + it("stops the timeout once settled", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 10 }) + + waiter.onTaskEvent(startResponse(successPayload())) + await waiter.result + + // A pending timer must not reject after success. Wait past the + // original deadline; a leak would reject the already-settled promise + // handler chain here. + await new Promise((resolve) => setTimeout(resolve, 30)) + assert.equal(await waiter.result, "accepted-task-1") + }) +}) diff --git a/scripts/solheim-smoke/task-start.ts b/scripts/solheim-smoke/task-start.ts new file mode 100644 index 0000000000..2aafaaea57 --- /dev/null +++ b/scripts/solheim-smoke/task-start.ts @@ -0,0 +1,96 @@ +import { RooCodeEventName, taskStartResponseSchema } from "@roo-code/types" + +export type TaskStartWaiter = { + onTaskEvent: (event: unknown) => string | undefined + onDisconnect: () => void + onExit: () => void + result: Promise +} + +const isRecord = (value: unknown): value is Record => typeof value === "object" && value !== null + +// Correlates one StartNewTask request with its sanitized response, similar to +// JSON-RPC: the sender generates the requestId and the reply copies it back. +// The waiter settles exactly once. A matched success resolves with the +// accepted task ID. A matched failure, a timeout, an IPC disconnect, or a +// VS Code exit rejects. Unmatched or malformed responses never settle it. +export function createTaskStartWaiter({ + requestId, + timeoutMs, +}: { + requestId: string + timeoutMs: number +}): TaskStartWaiter { + let resolveResult!: (taskId: string) => void + let rejectResult!: (error: Error) => void + const result = new Promise((resolve, reject) => { + resolveResult = resolve + rejectResult = reject + }) + + let settled = false + let timer: NodeJS.Timeout | undefined + + const settle = (finish: () => void): void => { + if (settled) { + return + } + settled = true + if (timer !== undefined) { + clearTimeout(timer) + timer = undefined + } + finish() + } + + const rejectWith = (message: string): void => { + settle(() => rejectResult(new Error(message))) + } + + timer = setTimeout(() => { + rejectWith(`task start response missing after ${timeoutMs}ms (requestId=${requestId})`) + }, timeoutMs) + + return { + onTaskEvent(event: unknown): string | undefined { + if (settled) return undefined + if (!isRecord(event) || event["eventName"] !== RooCodeEventName.TaskStartResponse) { + return undefined + } + + const payload = event["payload"] + if (!Array.isArray(payload) || payload.length === 0) { + return undefined + } + + const parsed = taskStartResponseSchema.safeParse(payload[0]) + if (!parsed.success || parsed.data.requestId !== requestId) { + return undefined + } + + const response = parsed.data + if (response.success) { + const acceptedTaskId = response.taskId + settle(() => resolveResult(acceptedTaskId)) + return acceptedTaskId + } + + const { errorCode, errorMessage, stage } = response + // The stage is a fixed sanitized token from the response schema; it + // names the startup phase that stopped responding. + const stageDetail = stage === undefined ? "" : ` (stage=${stage})` + rejectWith(`${errorCode}: ${errorMessage}${stageDetail}`) + return undefined + }, + + onDisconnect(): void { + rejectWith("IPC disconnected before the task start response arrived") + }, + + onExit(): void { + rejectWith("VS Code exited before the task start response arrived") + }, + + result, + } +} diff --git a/scripts/solheim-smoke/tsconfig.json b/scripts/solheim-smoke/tsconfig.json new file mode 100644 index 0000000000..55bddebe10 --- /dev/null +++ b/scripts/solheim-smoke/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "allowImportingTsExtensions": true, + "esModuleInterop": true, + "isolatedModules": true, + "lib": ["ES2022"], + "module": "esnext", + "moduleResolution": "bundler", + "noEmit": true, + "noUncheckedIndexedAccess": true, + "skipLibCheck": true, + "strict": true, + "target": "ES2022", + "types": ["node"] + }, + "include": ["./*.ts", "./*.mts"] +} diff --git a/src/extension/__tests__/api-start-task-logging.spec.ts b/src/extension/__tests__/api-start-task-logging.spec.ts new file mode 100644 index 0000000000..cb7ee10389 --- /dev/null +++ b/src/extension/__tests__/api-start-task-logging.spec.ts @@ -0,0 +1,233 @@ +import { afterEach, describe, expect, it, vi } from "vitest" +import * as vscode from "vscode" + +import { + TaskCommandName, + RooCodeEventName, + TASK_START_FAILURE_ERROR_CODE, + TASK_START_FAILURE_ERROR_MESSAGE, + taskStartResponseSchema, + providerIdentifiers, + type RooCodeSettings, +} from "@roo-code/types" + +import { API } from "../api" +import { Package } from "../../shared/package" +import type { ClineProvider } from "../../core/webview/ClineProvider" + +const { commandHandlers, sentMessages } = vi.hoisted(() => ({ + commandHandlers: [] as Array<(clientId: string, command: unknown) => Promise>, + sentMessages: [] as Array<{ clientId: string; message: unknown }>, +})) + +vi.mock("@roo-code/ipc", () => ({ + IpcServer: class { + listen(): void {} + on(_messageType: unknown, handler: (clientId: string, command: unknown) => Promise): void { + commandHandlers.push(handler) + } + send(clientId: string, message: unknown): void { + sentMessages.push({ clientId, message }) + } + }, +})) + +type SentTaskEvent = { data: { eventName: RooCodeEventName; payload: unknown[] } } + +const sentTaskEvents = (): SentTaskEvent[] => sentMessages.map(({ message }) => message as SentTaskEvent) + +const buildStartCommand = (overrides?: { requestId?: string; text?: string }) => ({ + commandName: TaskCommandName.StartNewTask, + data: { + configuration: { + apiProvider: providerIdentifiers.openai, + openAiApiKey: "opaque-solheim-secret-0123456789abcdef", + openAiModelId: "qwen3.8-27b", + openAiBaseUrl: "https://api.solheim.ai/v1", + autoApprovalEnabled: true, + } as RooCodeSettings, + text: overrides?.text ?? "provider connectivity smoke", + requestId: overrides?.requestId, + }, +}) + +function buildApi(createTaskImpl?: () => Promise<{ taskId: string }>, postStateToWebviewImpl?: () => Promise) { + const outputChannel = { appendLine: vi.fn() } as unknown as vscode.OutputChannel + const provider = { + context: {}, + on: vi.fn(), + contextProxy: { setValues: vi.fn().mockResolvedValue(undefined) }, + providerSettingsManager: { saveConfig: vi.fn().mockResolvedValue("default"), setModeConfig: vi.fn() }, + postStateToWebview: postStateToWebviewImpl + ? vi.fn().mockImplementation(postStateToWebviewImpl) + : vi.fn().mockResolvedValue(undefined), + evictCurrentTask: vi.fn().mockResolvedValue(undefined), + postMessageToWebview: vi.fn().mockResolvedValue(undefined), + removeClineFromStack: vi.fn().mockResolvedValue(undefined), + createTask: createTaskImpl + ? vi.fn().mockImplementation(createTaskImpl) + : vi.fn().mockResolvedValue({ taskId: "task-1" }), + } as unknown as ClineProvider + + const consoleSpy = vi.spyOn(console, "log").mockImplementation(() => {}) + const executeCommand = vi.spyOn(vscode.commands, "executeCommand").mockResolvedValue(undefined) + const api = new API(outputChannel, provider, "/tmp/final-smoke-test.sock", true) + + const collectLogs = () => + [ + ...(outputChannel.appendLine as ReturnType).mock.calls.flat().map(String), + ...consoleSpy.mock.calls.flat().map((arg) => JSON.stringify(arg)), + ].join("\n") + + return { api, provider, executeCommand, collectLogs } +} + +describe("API StartNewTask IPC", () => { + afterEach(() => { + commandHandlers.length = 0 + sentMessages.length = 0 + vi.useRealTimers() + vi.restoreAllMocks() + }) + + it("never logs the API key, base URL, or prompt text from the StartNewTask command", async () => { + const { collectLogs } = buildApi() + + const handler = commandHandlers.at(-1) + expect(handler).toBeDefined() + + await handler!("client-1", buildStartCommand()) + + const logged = collectLogs() + + expect(logged).not.toContain("opaque-solheim-secret-0123456789abcdef") + expect(logged).not.toContain('"openAiApiKey":"') + // Configuration is omitted entirely, even nominally non-secret fields. + expect(logged).not.toContain("openAiModelId") + expect(logged).not.toContain("qwen3.8-27b") + expect(logged).not.toContain("openAiBaseUrl") + expect(logged).not.toContain("openAiApiKey") + // Prompt content is metadata only: length, never text. + expect(logged).not.toContain("provider connectivity smoke") + expect(logged).toContain("promptLength") + }) + + it("omits the sidebar focus for an IPC start, so an unresolved focus command cannot block it", async () => { + const { executeCommand } = buildApi() + + const handler = commandHandlers.at(-1) + await handler!("client-1", buildStartCommand({ requestId: "req-legacy" })) + + expect(executeCommand).not.toHaveBeenCalledWith(`${Package.name}.SidebarProvider.focus`) + }) + + it("answers a correlated start with exactly one sanitized success response", async () => { + buildApi() + + const handler = commandHandlers.at(-1) + await handler!("client-1", buildStartCommand({ requestId: "req-1" })) + + expect(sentMessages).toHaveLength(1) + expect(sentMessages[0]?.clientId).toBe("client-1") + const events = sentTaskEvents() + expect(events[0]?.data.eventName).toBe(RooCodeEventName.TaskStartResponse) + + expect(taskStartResponseSchema.parse(events[0]?.data.payload[0])).toEqual({ + requestId: "req-1", + success: true, + taskId: "task-1", + }) + }) + + it("answers a failed correlated start with exactly one fixed safe failure response", async () => { + const rawError = new Error( + "openai rejected key opaque-solheim-secret-0123456789abcdef at https://api.solheim.ai/v1 for prompt review the pull request diff", + ) + buildApi(() => Promise.reject(rawError)) + + const handler = commandHandlers.at(-1) + await handler!("client-1", buildStartCommand({ requestId: "req-2" })) + + expect(sentMessages).toHaveLength(1) + expect(sentMessages[0]?.clientId).toBe("client-1") + const events = sentTaskEvents() + expect(events[0]?.data.eventName).toBe(RooCodeEventName.TaskStartResponse) + + const parsed = taskStartResponseSchema.parse(events[0]?.data.payload[0]) + expect(parsed).toEqual({ + requestId: "req-2", + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + }) + + const sent = JSON.stringify(sentMessages) + expect(sent).not.toContain("opaque-solheim-secret-0123456789abcdef") + expect(sent).not.toContain("api.solheim.ai") + expect(sent).not.toContain("review the pull request diff") + }) + + it("does not race pending configuration/startup against a timeout", async () => { + vi.useFakeTimers() + let finishCreation!: (task: { taskId: string }) => void + const { provider } = buildApi( + () => + new Promise((resolve) => { + finishCreation = resolve + }), + ) + const running = commandHandlers.at(-1)!("client-1", buildStartCommand({ requestId: "req-pending" })) + await vi.advanceTimersByTimeAsync(60_000) + expect(provider.createTask).toHaveBeenCalledTimes(1) + expect(sentMessages).toHaveLength(0) + expect(provider.removeClineFromStack).not.toHaveBeenCalled() + finishCreation({ taskId: "finished-task" }) + await running + expect(taskStartResponseSchema.parse(sentTaskEvents()[0]?.data.payload[0])).toEqual({ + requestId: "req-pending", + success: true, + taskId: "finished-task", + }) + }) + + it("reports a delayed startup rejection only after it settles, without late task removal", async () => { + vi.useFakeTimers() + let rejectCreation!: (error: Error) => void + const { provider } = buildApi( + () => + new Promise((_resolve, reject) => { + rejectCreation = reject + }), + ) + const running = commandHandlers.at(-1)!("client-1", buildStartCommand({ requestId: "req-reject" })) + await vi.advanceTimersByTimeAsync(60_000) + expect(sentMessages).toHaveLength(0) + rejectCreation(new Error("private startup error")) + await running + expect(taskStartResponseSchema.parse(sentTaskEvents()[0]?.data.payload[0])).toEqual({ + requestId: "req-reject", + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + }) + expect(provider.removeClineFromStack).not.toHaveBeenCalled() + }) + + it("sends no start response for legacy IPC starts without requestId", async () => { + const { executeCommand } = buildApi() + + const handler = commandHandlers.at(-1) + await handler!("client-1", buildStartCommand()) + + expect(sentMessages).toHaveLength(0) + expect(executeCommand).not.toHaveBeenCalledWith(`${Package.name}.SidebarProvider.focus`) + }) + + it("keeps the sidebar focus for direct API callers that do not opt out", async () => { + const { api, executeCommand } = buildApi() + + await api.startNewTask({ configuration: {} as RooCodeSettings, text: "direct caller" }) + + expect(executeCommand).toHaveBeenCalledWith(`${Package.name}.SidebarProvider.focus`) + }) +}) diff --git a/src/extension/api.ts b/src/extension/api.ts index 316e7a6c9d..8f4d61e73d 100644 --- a/src/extension/api.ts +++ b/src/extension/api.ts @@ -13,6 +13,7 @@ import { type ProviderSettings, type ProviderSettingsEntry, type TaskEvent, + type TaskStartResponse, type CreateTaskOptions, type TaskApiConversationHistorySequence, type WebviewThemeFixture, @@ -21,7 +22,10 @@ import { isSecretStateKey, IpcOrigin, IpcMessageType, + TASK_START_FAILURE_ERROR_CODE, + TASK_START_FAILURE_ERROR_MESSAGE, } from "@roo-code/types" + import { IpcServer } from "@roo-code/ipc" import { Package } from "../shared/package" @@ -82,12 +86,45 @@ export class API extends EventEmitter implements RooCodeAPI { } switch (command.commandName) { - case TaskCommandName.StartNewTask: - this.log( - `[API] StartNewTask -> ${command.data.text}, ${JSON.stringify(command.data.configuration)}`, - ) - await this.startNewTask(command.data) + case TaskCommandName.StartNewTask: { + // Neither prompt nor settings values enter the log. + this.log("[API] StartNewTask", { + promptLength: command.data.text.length, + imageCount: command.data.images?.length ?? 0, + newTab: command.data.newTab === true, + requestId: command.data.requestId, + }) + + const { requestId } = command.data + + if (requestId === undefined) { + await this.startNewTask(command.data, { focusSidebar: false }) + break + } + + // Exactly one sanitized, client-scoped response per + // correlated start. The reply never echoes the failure. + // Await actual startup, including configuration writes. + // The smoke controller owns its timeout and terminates + // its isolated host; shared-profile writes are not raced. + let response: TaskStartResponse + try { + const taskId = await this.startNewTask(command.data, { + focusSidebar: false, + }) + response = { requestId, success: true, taskId } + } catch { + this.log("[API] StartNewTask failed", { requestId }) + response = { + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + } + } + sendResponse(RooCodeEventName.TaskStartResponse, [response]) break + } case TaskCommandName.CancelTask: this.log(`[API] CancelTask`) await this.cancelCurrentTask() @@ -170,17 +207,20 @@ export class API extends EventEmitter implements RooCodeAPI { return super.emit(eventName, ...args) } - public async startNewTask({ - configuration, - text, - images, - newTab, - }: { - configuration: RooCodeSettings - text?: string - images?: string[] - newTab?: boolean - }) { + public async startNewTask( + { + configuration, + text, + images, + newTab, + }: { + configuration: RooCodeSettings + text?: string + images?: string[] + newTab?: boolean + }, + startOptions?: { focusSidebar?: boolean }, + ) { let provider: ClineProvider if (newTab) { @@ -190,7 +230,11 @@ export class API extends EventEmitter implements RooCodeAPI { provider = await openClineInNewTab({ context: this.context, outputChannel: this.outputChannel }) this.registerListeners(provider) } else { - await vscode.commands.executeCommand(`${Package.name}.SidebarProvider.focus`) + // IPC callers omit focus: executeCommand cannot be cancelled and a + // stalled webview launch would block the whole start. + if (startOptions?.focusSidebar ?? true) { + await vscode.commands.executeCommand(`${Package.name}.SidebarProvider.focus`) + } provider = this.sidebarProvider }