diff --git a/.github/workflows/node-ci.yml b/.github/workflows/node-ci.yml index bcf114640a..8806fde724 100644 --- a/.github/workflows/node-ci.yml +++ b/.github/workflows/node-ci.yml @@ -345,13 +345,14 @@ jobs: matrix: os: [ubuntu-latest, macos-latest, windows-latest] steps: - - name: Checkout plugin source without the SDK + - name: Checkout plugin and shared SDK source uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false sparse-checkout: | plugins/codex-security .github/actions/setup-tools + sdk/typescript - *set-up-mcp-tools - name: Install plugin dependencies run: pnpm --dir plugins/codex-security/mcp-app install --frozen-lockfile diff --git a/plugins/codex-security/mcp-app/deep-scan-main.ts b/plugins/codex-security/mcp-app/deep-scan-main.ts index a8a36e3b10..04891099c0 100644 --- a/plugins/codex-security/mcp-app/deep-scan-main.ts +++ b/plugins/codex-security/mcp-app/deep-scan-main.ts @@ -9,11 +9,30 @@ import { startDeepScanEngine } from "./src/deep-scan/engine.js"; import { resolveDeepWorkerParentSandbox } from "./src/deep-scan/parent-sandbox.js"; import { DeepScanCoordinatorRegistry } from "./src/deep-scan/registry.js"; import { WorkbenchDeepScanStore } from "./src/deep-scan/store.js"; +import { + beginDeepScanWithCapturedSettings, + captureDeepScanExecutionSettings, +} from "./src/deep-scan/recovery-settings.js"; /** Private SDK entry point. The child process isolates each scan's environment. */ export async function runCliDeepScan(): Promise { - const runWorkbench = (args: string[], input?: string | Buffer) => - runWorkbenchCommand(args, input, { isolatedPython: true }); + const runWorkbench = ( + args: string[], + input?: string | Buffer, + selectFinalization?: boolean, + withExecutionSettings?: boolean, + signal?: AbortSignal, + releaseCoordinator?: boolean, + ) => + runWorkbenchCommand( + args, + input, + selectFinalization, + withExecutionSettings, + signal, + releaseCoordinator, + true, + ); const lines = createInterface({ input: process.stdin, crlfDelay: Infinity }); const registry = new DeepScanCoordinatorRegistry(); const controller = new AbortController(); @@ -49,13 +68,24 @@ export async function runCliDeepScan(): Promise { }); if (controller.signal.aborted) throw new Error("Deep Scan canceled before coordinator startup."); - const run = await store.begin({ - scanId: input.scanId, - threadId: input.threadId, - model: input.model, - reasoningEffort: input.reasoningEffort, - scanRoot: join(process.env.CODEX_SECURITY_STATE_DIR!, "scans"), - }); + const run = await beginDeepScanWithCapturedSettings( + () => + captureDeepScanExecutionSettings( + { model: input.model, reasoningEffort: input.reasoningEffort }, + parentSandbox, + process.env, + { threadId: input.threadId, startedAt: new Date().toISOString() }, + ), + (executionSettings) => + store.begin({ + executionSettings, + scanId: input.scanId, + threadId: input.threadId, + model: input.model, + reasoningEffort: input.reasoningEffort, + scanRoot: join(process.env.CODEX_SECURITY_STATE_DIR!, "scans"), + }), + ); if (controller.signal.aborted) throw new Error("Deep Scan canceled before coordinator startup."); let terminal = run; diff --git a/plugins/codex-security/mcp-app/helpers-main.ts b/plugins/codex-security/mcp-app/helpers-main.ts index ec7f324026..294aa35b32 100644 --- a/plugins/codex-security/mcp-app/helpers-main.ts +++ b/plugins/codex-security/mcp-app/helpers-main.ts @@ -19,7 +19,9 @@ import { bindRepoScopesCommand } from "./src/helpers/bind-repo-scopes"; import { escapeControls, stringifyJson } from "./src/helpers/json"; import { decodeUtf8 } from "./src/helpers/utf8"; -// Importing the bundled parser does not invoke its CLI adapter. +export { resumeSelectedDeepScan } from "./src/deep-scan/finalization.js"; + +// Importing the bundled helper from the SDK does not invoke its CLI adapter. const entryPath = import.meta.url.startsWith("file:") ? fileURLToPath(import.meta.url) : import.meta.url; diff --git a/plugins/codex-security/mcp-app/package.json b/plugins/codex-security/mcp-app/package.json index 342d580518..e03b5002f6 100644 --- a/plugins/codex-security/mcp-app/package.json +++ b/plugins/codex-security/mcp-app/package.json @@ -6,7 +6,7 @@ "scripts": { "build": "node scripts/generate-artifact-schemas.mjs && tsc --noEmit", "build:mcp": "node scripts/build_mcp_app.mjs --output .preview/mcp", - "test:mcp": "node --experimental-strip-types --test --test-concurrency=2 --test-reporter=./scripts/test_reporter.mts \"tests/test_*.ts\"", + "test:mcp": "node --experimental-strip-types --test --test-concurrency=2 --test-reporter=./scripts/test_reporter.mts \"tests/test_*.ts\" \"tests/test_*.mjs\"", "typecheck": "node scripts/generate-artifact-schemas.mjs --check && tsc --noEmit" }, "dependencies": { @@ -14,10 +14,12 @@ "@openai/codex-sdk": "0.163.0", "ajv": "8.20.0", "smol-toml": "1.9.1", + "yauzl": "3.4.0", "zod": "^4.6.7" }, "devDependencies": { "@types/node": "^26.6.5", + "@types/yauzl": "3.4.0", "esbuild": "^0.28.2", "prettier": "3.9.10", "typescript": "^7.0.2" diff --git a/plugins/codex-security/mcp-app/pnpm-lock.yaml b/plugins/codex-security/mcp-app/pnpm-lock.yaml index ade7b0fe1a..db882cc895 100644 --- a/plugins/codex-security/mcp-app/pnpm-lock.yaml +++ b/plugins/codex-security/mcp-app/pnpm-lock.yaml @@ -20,6 +20,9 @@ importers: smol-toml: specifier: 1.9.1 version: 1.9.1 + yauzl: + specifier: 3.4.0 + version: 3.4.0 zod: specifier: ^4.6.7 version: 4.6.7 @@ -27,6 +30,9 @@ importers: '@types/node': specifier: ^26.6.5 version: 26.6.5 + '@types/yauzl': + specifier: 3.4.0 + version: 3.4.0 esbuild: specifier: ^0.28.2 version: 0.28.2 @@ -259,6 +265,9 @@ packages: '@types/node@26.6.5': resolution: {integrity: sha512-uMzsv9stDdofIZBx4UGh4Z9ZgXnGC9seRDscilBcTHAeOFk67MZg/k9bWVfO+bk4qZmG+l52hRnLwUiH2LnJTg==} + '@types/yauzl@3.4.0': + resolution: {integrity: sha512-NRPn5w6h8dhcnmx3YIRQcqMywY/+nND/uOkJessedcrowO3C0AssHp3tMJpxKAwOhFOo0OV1y9VtsC5hbKKBAw==} + '@typescript/typescript-aix-ppc64@7.0.2': resolution: {integrity: sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ==} engines: {node: '>=16.20.0'} @@ -636,6 +645,9 @@ packages: path-to-regexp@8.4.2: resolution: {integrity: sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==} + pend@1.2.0: + resolution: {integrity: sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==} + pkce-challenge@5.0.1: resolution: {integrity: sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==} engines: {node: '>=16.20.0'} @@ -747,6 +759,10 @@ packages: wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} + yauzl@3.4.0: + resolution: {integrity: sha512-jIH9yLR9wqr0wOS0TpBvo/g/2UgZH5qePVbjgRliiF0BYvOZyaBknKsF+x9Iht0O6sqgnB93rCICdOZFecJuDw==} + engines: {node: '>=12'} + zod-to-json-schema@3.25.2: resolution: {integrity: sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==} peerDependencies: @@ -896,6 +912,10 @@ snapshots: dependencies: undici-types: 8.9.0 + '@types/yauzl@3.4.0': + dependencies: + '@types/node': 26.6.5 + '@typescript/typescript-aix-ppc64@7.0.2': optional: true @@ -1236,6 +1256,8 @@ snapshots: path-to-regexp@8.4.2: {} + pend@1.2.0: {} + pkce-challenge@5.0.1: {} prettier@3.9.10: {} @@ -1381,6 +1403,10 @@ snapshots: wrappy@1.0.2: {} + yauzl@3.4.0: + dependencies: + pend: 1.2.0 + zod-to-json-schema@3.25.2(zod@4.6.7): dependencies: zod: 4.6.7 diff --git a/plugins/codex-security/mcp-app/scripts/build_mcp_app.mjs b/plugins/codex-security/mcp-app/scripts/build_mcp_app.mjs index 9bb09a1eb2..5fcb5913ef 100644 --- a/plugins/codex-security/mcp-app/scripts/build_mcp_app.mjs +++ b/plugins/codex-security/mcp-app/scripts/build_mcp_app.mjs @@ -97,6 +97,7 @@ export async function buildMcpApp({ output, native = "universal" }) { loader: { ".md": "text" }, logLevel: "info", logOverride: { "empty-import-meta": "silent" }, + nodePaths: [join(root, "node_modules")], outfile: bundle, platform: "node", plugins: [ diff --git a/plugins/codex-security/mcp-app/server.ts b/plugins/codex-security/mcp-app/server.ts index 1a7bb1d3fc..d9e101d368 100644 --- a/plugins/codex-security/mcp-app/server.ts +++ b/plugins/codex-security/mcp-app/server.ts @@ -23,6 +23,10 @@ import { DeepScanCoordinatorRegistry, AsyncLock, } from "./src/deep-scan/registry.js"; +import { + beginDeepScanWithCapturedSettings, + captureDeepScanExecutionSettings, +} from "./src/deep-scan/recovery-settings.js"; import { startDeepScanEngine } from "./src/deep-scan/engine.js"; import type { DeepScanParentTurn } from "./src/deep-scan/permission-profile-preflight.js"; import { @@ -896,7 +900,7 @@ export function createCodexSecurityServer(): McpServer { { title: "Start or Join Codex Security Deep Scan", description: - "Run or rejoin independent Standard security scans and semantically merge their validated findings. Pass scanId and its handoffClaimToken to resume, or targetPath to start headlessly. The call blocks until the aggregate draft is ready, fails, or is canceled. On success, use the returned scanId and scanDir; manifestPath identifies the canonical parent scan-manifest.json. Call complete_codex_security_scan once.", + "Run or rejoin independent Standard security scans and semantically merge their validated findings. Pass scanId and its handoffClaimToken to resume, or targetPath to start headlessly. The call blocks until the aggregate draft is ready, fails, or is canceled. On success, use the returned scanId and scanDir; manifestPath identifies the canonical parent scan-manifest.json; native scans can call complete_codex_security_scan once. SDK-managed scans leave completion to the SDK after the scan turn ends.", inputSchema: startDeepScanSchema, annotations: writingAnnotations, _meta: modelActionMeta, @@ -946,24 +950,52 @@ export function createCodexSecurityServer(): McpServer { } const preparation = await deepScanStartLock .run(async () => { - const begun = await deepScanStore.begin({ - scanId, - targetPath, - scope: hasTarget ? (scope ?? ".") : undefined, - userContext: normalizedUserContext, - handoffClaimToken, - threadId, - ...modelSettings, - scanRoot: await scanRoot(), - }); + // A joining observer does not need a usable current home. A new run, + // however, must save its original settings before creation can commit. + const begun = await beginDeepScanWithCapturedSettings( + () => + captureDeepScanExecutionSettings( + modelSettings, + parentSandbox, + process.env, + { threadId, startedAt: new Date().toISOString() }, + ), + async (executionSettings) => + deepScanStore.begin({ + executionSettings, + scanId, + targetPath, + scope: hasTarget ? (scope ?? ".") : undefined, + userContext: normalizedUserContext, + handoffClaimToken, + threadId, + ...modelSettings, + scanRoot: await scanRoot(), + }), + ); if (handoffClaimToken) { authenticatedArtifactClaims.set(begun.scanId, { claimToken: handoffClaimToken, threadId, }); } - const immediate = deepScanTerminalResult(begun); - if (immediate) return { begun, immediate }; + // Recipes identify SDK executions, including runs predating usageOwner. + const sdkOwned = + begun.usageOwner?.dedicated ?? + (await runWorkbench(["get-scan", "--scan-id", begun.scanId])) + .recipe != null; + const immediate = deepScanTerminalResult(begun, sdkOwned); + const completingLocally = + begun.status === "succeeded" && + begun.finalizationInput && + deepScanCoordinators.get(begun.scanId); + const recoverableSelection = + begun.status === "succeeded" && + begun.finalizationInput && + !sdkOwned && + begun.ownerThreadId === threadId; + if (immediate && !completingLocally && !recoverableSelection) + return { begun, immediate, sdkOwned }; const started = await startDeepScanEngine({ run: begun, registry: deepScanCoordinators, @@ -977,8 +1009,44 @@ export function createCodexSecurityServer(): McpServer { log: logDeepScanEvent, handoffClaimToken, threadId, + onFinalized: async (run, signal) => { + // Active owners finish through the completion tool after this call; + // detached native scans still publish without another observer. + if (sdkOwned || !abortSignalFromExtra(extra)?.aborted) return; + try { + await runWorkbench( + [ + "complete-scan", + "--scan-id", + run.scanId, + "--thread-id", + threadId, + ...optionalArg("--claim-token", handoffClaimToken), + ], + undefined, + false, + false, + signal, + ); + } catch (error) { + if (!signal.aborted) { + await deepScanStore + .releaseCoordinator(run.scanId) + .catch((releaseError) => { + logDeepScanEvent({ + event: "coordinator_release_failed", + scanId: run.scanId, + reason: boundedErrorData(releaseError).message, + }); + }); + } + throw new Error(deepScanInvocationFailureMessage(error), { + cause: error, + }); + } + }, }); - return { begun, ...started }; + return { begun, sdkOwned, ...started }; }) .catch((error: unknown) => ({ invocationFailure: toolErrorResult( @@ -987,16 +1055,14 @@ export function createCodexSecurityServer(): McpServer { })); if ("invocationFailure" in preparation) return preparation.invocationFailure; + // SDK completion belongs to the host; native selected results use the existing lease. if (preparation.immediate) return preparation.immediate; const { begun, coordinator, joined } = preparation; if (joined) { - logDeepScanEvent({ - event: "coordinator_joined", - scanId: begun.scanId, - }); + logDeepScanEvent({ event: "coordinator_joined", scanId: begun.scanId }); } const terminal = await coordinator.wait(abortSignalFromExtra(extra)); - const result = deepScanTerminalResult(terminal); + const result = deepScanTerminalResult(terminal, preparation.sdkOwned); if (!result) { return toolErrorResult( deepScanInvocationFailureMessage( @@ -1452,7 +1518,7 @@ export function createCodexSecurityServer(): McpServer { { title: "Complete Codex Security Scan", description: - "Finalization only: validate and seal already-authored scan-manifest.json, findings.json, and coverage.json, generate report.md, index findings, and mark the scan complete. For an app-backed running scan, scan-manifest.json is an unsealed draft and must omit scan.sealedAt and scan.artifacts; this tool supplies the exact workbench timestamps, seal, artifact digests, and derived finding identities. Call only after those canonical files exist; this tool does not create missing artifacts or run skipped phases. If it fails, surface the exact error and stop the current response without retrying completion or returning a final, no-findings, structured, or benchmark response.", + "Finalization only: validate and seal already-authored scan-manifest.json, findings.json, and coverage.json, generate report.md, index findings, and mark the scan complete. For an app-backed running scan, scan-manifest.json is an unsealed draft and must omit scan.sealedAt and scan.artifacts; this tool supplies the exact workbench timestamps, seal, artifact digests, and derived finding identities. Call only after those canonical files exist; this tool does not create missing artifacts or run skipped phases. SDK-managed Deep scans leave sealing to the SDK after the scan turn ends. If it fails, surface the exact error and stop the current response without retrying completion or returning a final, no-findings, structured, or benchmark response.", inputSchema: completeScanSchema, annotations: writingAnnotations, _meta: modelActionMeta, @@ -1464,18 +1530,35 @@ export function createCodexSecurityServer(): McpServer { ? turnMetadata.turn_id : undefined; try { + const context = await runWorkbench(["get-scan", "--scan-id", scanId]); + const scan = isJsonObject(context.scan) ? context.scan : undefined; + const progress = isJsonObject(scan?.progress) + ? scan.progress + : undefined; + if ( + scan?.mode === "deep" && + context.recipe != null && + progress?.status === "running" + ) { + // A tool call is still inside the owner turn; its final usage has not arrived. + return scanActionResult( + redactHandoffClaimToken(context), + "The SDK completes this Deep Scan after the scan turn finishes. Continue any remaining scan work, then end the turn without calling completion again. The SDK will account for the turn, enforce its budget, and seal and publish the scan.", + ); + } + const result = await runWorkbench([ + "complete-scan", + "--scan-id", + scanId, + ...optionalArg("--claim-token", handoffClaimToken), + ...optionalArg("--thread-id", threadIdFromExtra(extra)), + ...optionalArg( + "--turn-id", + typeof turnId === "string" ? turnId : undefined, + ), + ]); return scanActionResult( - await runWorkbench([ - "complete-scan", - "--scan-id", - scanId, - ...optionalArg("--claim-token", handoffClaimToken), - ...optionalArg("--thread-id", threadIdFromExtra(extra)), - ...optionalArg( - "--turn-id", - typeof turnId === "string" ? turnId : undefined, - ), - ]), + result, "Validated and indexed the completed Codex Security scan.", ); } catch (error) { @@ -2069,10 +2152,10 @@ function boundedErrorData(error: unknown): { message: string; name: string } { }; } -function deepScanTerminalResult(run: DeepScanRunState) { +function deepScanTerminalResult(run: DeepScanRunState, sdkOwned = false) { if (run.status === "succeeded") { if (!run.manifestPath) return undefined; - const instructions = `Deep Scan discovery completed. Independent Standard scans have already performed validation and attack-path analysis and have been consolidated into the canonical scan-manifest.json, findings.json, and coverage.json under ${run.scanDir}. The returned manifestPath is the canonical scan-manifest.json, not a legacy discovery manifest. Any instructions requiring parent candidate listing, centralized validation, attack-path analysis, or another draft apply only to the old discovery-only workflow and must be skipped. The authoritative scan ID is ${run.scanId}. Immediately call complete_codex_security_scan once using that scan ID to seal and publish the scan. Return output only after completion succeeds and generated report.md exists. If completion fails, surface that exact error and return no final, no-findings, structured, or benchmark response.`; + const instructions = `Deep Scan discovery completed. Independent Standard scans have already performed validation and attack-path analysis and have been consolidated into the canonical scan-manifest.json, findings.json, and coverage.json under ${run.scanDir}. The returned manifestPath is the canonical scan-manifest.json, not a legacy discovery manifest. Any instructions requiring parent candidate listing, centralized validation, attack-path analysis, or another draft apply only to the old discovery-only workflow and must be skipped. The authoritative scan ID is ${run.scanId}. ${sdkOwned ? "Leave the canonical artifacts unchanged and end this scan turn without calling a completion tool. The SDK will account for the completed turn, enforce its budget, and seal and publish the scan." : "Immediately call complete_codex_security_scan once using that scan ID to seal and publish the scan. Return output only after completion succeeds and generated report.md exists. If completion fails, surface that exact error and return no final, no-findings, structured, or benchmark response."}`; return scanActionResult( { scanId: run.scanId, diff --git a/plugins/codex-security/mcp-app/src/accepted-discovery-normalization.ts b/plugins/codex-security/mcp-app/src/accepted-discovery-normalization.ts new file mode 100644 index 0000000000..2d736694de --- /dev/null +++ b/plugins/codex-security/mcp-app/src/accepted-discovery-normalization.ts @@ -0,0 +1,153 @@ +import { isRecord as isObject } from "./record.js"; + +type JsonObject = Record; + +/** Earlier discovery admission normalized a clone while retaining the raw file. */ +export function normalizeAcceptedDiscoveryFinding(finding: JsonObject): void { + const canonicalEvidence = Array.isArray(finding.codeEvidence) + ? finding.codeEvidence + : []; + const evidenceIds = new Set( + canonicalEvidence.flatMap((evidence) => { + if (!isObject(evidence)) return []; + return isNonEmptyString(evidence.id) ? [evidence.id] : []; + }), + ); + if (Array.isArray(finding.code_evidence)) { + const compatibleEvidence: JsonObject[] = []; + for (const evidence of finding.code_evidence) { + if (!isObject(evidence)) continue; + const { id, code } = evidence; + if ( + !isNonEmptyString(id) || + !isNonEmptyString(code) || + evidenceIds.has(id) + ) + continue; + evidenceIds.add(id); + compatibleEvidence.push(evidence); + } + finding.code_evidence = compatibleEvidence; + } else { + delete finding.code_evidence; + } + + for (const [sectionName, listFields] of [ + ["rootCause", ["evidenceRefs", "evidence_refs"]], + ["root_cause", ["evidenceRefs", "evidence_refs"]], + [ + "validation", + [ + "assertions", + "counterEvidence", + "evidence", + "evidenceRefs", + "evidence_refs", + "limitations", + ], + ], + [ + "attackPath", + [ + "assumptions", + "blindspots", + "controls", + "evidenceRefs", + "evidence_refs", + "limitations", + "preconditions", + "steps", + ], + ], + ] satisfies Array<[string, string[]]>) { + const section = finding[sectionName]; + if (isObject(section)) normalizeStringLists(section, listFields); + } + + if (isObject(finding.rootCause)) { + if (!isNonEmptyString(finding.rootCause.summary)) delete finding.rootCause; + else removeUnsupportedStrings(finding.rootCause, ["code", "language"]); + } + if (isObject(finding.root_cause)) { + removeUnsupportedStrings(finding.root_cause, [ + "summary", + "code", + "language", + ]); + } else if (!isNonEmptyString(finding.root_cause)) { + delete finding.root_cause; + } + if (isObject(finding.validation)) { + removeUnsupportedStrings(finding.validation, [ + "method", + "status", + "summary", + "disposition", + "result", + ]); + } + if (!isObject(finding.attackPath)) return; + const attackPath = finding.attackPath; + removeUnsupportedStrings(attackPath, ["summary"]); + for (const field of ["dataFlow", "data_flow", "dataflow", "reachability"]) { + const detail = attackPath[field]; + if (!isObject(detail)) { + if (!isNonEmptyString(detail)) delete attackPath[field]; + continue; + } + removeUnsupportedStrings(detail, [ + "summary", + "source", + "sink", + "outcome", + ...(field === "reachability" ? ["attacker", "entrypoint"] : []), + ]); + normalizeStringLists(detail, [ + "evidenceRefs", + "evidence_refs", + "transformations", + ...(field === "reachability" ? ["preconditions"] : []), + ]); + } + for (const field of ["impact", "likelihood"]) { + const detail = attackPath[field]; + if (isObject(detail)) { + removeUnsupportedStrings(detail, ["level", "rationale", "why"]); + } else if ( + detail !== undefined && + detail !== null && + !isNonEmptyString(detail) + ) { + delete attackPath[field]; + } + } + + function normalizeStringLists(section: JsonObject, fields: string[]): void { + for (const field of fields) { + const value = section[field]; + const normalized = isNonEmptyString(value) + ? [value] + : Array.isArray(value) + ? value.filter(isNonEmptyString) + : undefined; + if (normalized !== undefined && normalized.length > 0) + section[field] = normalized; + else delete section[field]; + } + for (const field of ["evidenceRefs", "evidence_refs"]) { + const refs = section[field]; + if (Array.isArray(refs)) + section[field] = refs.filter((ref) => evidenceIds.has(ref)); + } + } +} + +function removeUnsupportedStrings(section: JsonObject, fields: string[]): void { + for (const field of fields) { + if (!isNonEmptyString(section[field])) delete section[field]; + } +} + +function isNonEmptyString(value: unknown): value is string { + return typeof value === "string" && value.trim().length > 0; +} diff --git a/plugins/codex-security/mcp-app/src/artifact-context.ts b/plugins/codex-security/mcp-app/src/artifact-context.ts index 22b87eacee..38d5253d4d 100644 --- a/plugins/codex-security/mcp-app/src/artifact-context.ts +++ b/plugins/codex-security/mcp-app/src/artifact-context.ts @@ -17,6 +17,8 @@ export type RunArtifactWorkbench = ( export interface ScanArtifactContextOptions { requireRunning?: boolean; requireClaim?: boolean; + /** Selected publication only uses the saved target contract, without reading its checkout. */ + requireCurrentTarget?: boolean; handoffClaimToken?: string; pluginRoot?: string; pythonCommand?: string; @@ -84,10 +86,13 @@ export async function createScanArtifactContext( rawRoot, "Codex Security scan artifact root", ), - repoRoot: await canonicalDirectory( - rawRepoRoot, - "Codex Security scan target root", - ), + repoRoot: + options.requireCurrentTarget === false + ? absoluteDirectory(rawRepoRoot, "Codex Security scan target root") + : await canonicalDirectory( + rawRepoRoot, + "Codex Security scan target root", + ), layout: "scan", scanId, ...defined("scope", optionalString(scan.scope)), @@ -120,10 +125,7 @@ export async function canonicalDirectory( value: string, label: string, ): Promise { - if (!value || !isAbsolute(value)) { - throw new Error(label + " must be an absolute directory."); - } - const requested = resolve(value); + const requested = absoluteDirectory(value, label); if (!(await fs.lstat(requested).catch(() => undefined))?.isDirectory()) { throw new Error(label + " is not a safe regular directory."); } @@ -134,6 +136,12 @@ export async function canonicalDirectory( } } +function absoluteDirectory(value: string, label: string): string { + if (!value || !isAbsolute(value)) + throw new Error(label + " must be an absolute directory."); + return resolve(value); +} + function requireString(value: unknown, message: string): string { const result = optionalString(value); if (!result) throw new Error(message); diff --git a/plugins/codex-security/mcp-app/src/artifact-contracts.ts b/plugins/codex-security/mcp-app/src/artifact-contracts.ts index 5ef65a7e30..80236acd58 100644 --- a/plugins/codex-security/mcp-app/src/artifact-contracts.ts +++ b/plugins/codex-security/mcp-app/src/artifact-contracts.ts @@ -387,6 +387,74 @@ const draft_scanDraftInput = z.strictObject({ coverage: draft_coverage, }); +// Immutable Deep results can predate the current producer contract. Preserve +// their finding fields when reading already accepted worker history. +const acceptedTextOrSummary = z.union([ + nonEmptyText, + draft_textOrSummary.options[1].extend({ + evidence_refs: draft_textList.optional(), + }), +]); +const acceptedReachability = z.union([ + nonEmptyText, + draft_findingReachability.options[1].extend({ + evidence_refs: draft_textList.optional(), + }), +]); +export const acceptedDeepScanDraftInputSchema = draft_scanDraftInput.extend({ + findings: z.array( + draft_finding.extend({ + code_evidence: z + .array(z.looseObject({ id: nonEmptyText, code: nonEmptyText })) + .optional(), + rootCause: z + .union([ + nonEmptyText, + z.looseObject({ + summary: nonEmptyText, + code: nonEmptyText.optional(), + language: nonEmptyText.optional(), + evidenceRefs: draft_textList.optional(), + evidence_refs: draft_textList.optional(), + }), + ]) + .optional(), + root_cause: z + .union([ + nonEmptyText, + z.looseObject({ + summary: nonEmptyText.optional(), + code: nonEmptyText.optional(), + language: nonEmptyText.optional(), + evidenceRefs: draft_textList.optional(), + evidence_refs: draft_textList.optional(), + }), + ]) + .optional(), + validation: z + .union([ + draft_findingValidation.extend({ + evidence_refs: draft_textList.optional(), + }), + z.null(), + ]) + .optional(), + attackPath: z + .union([ + draft_findingAttackPath.extend({ + evidence_refs: draft_textList.optional(), + dataflow: acceptedTextOrSummary.optional(), + dataFlow: acceptedTextOrSummary.optional(), + data_flow: acceptedTextOrSummary.optional(), + reachability: acceptedReachability.optional(), + }), + z.null(), + ]) + .optional(), + }), + ), +}); + /** Native tool contracts; JSON resources are generated compatibility exports. */ export const artifactSchemaDocuments = { "definitions/artifact-common": { diff --git a/plugins/codex-security/mcp-app/src/artifact-deep-reducer.ts b/plugins/codex-security/mcp-app/src/artifact-deep-reducer.ts index 90515b4912..92ff0800ed 100644 --- a/plugins/codex-security/mcp-app/src/artifact-deep-reducer.ts +++ b/plugins/codex-security/mcp-app/src/artifact-deep-reducer.ts @@ -1,7 +1,7 @@ -import { join } from "node:path"; +import { dirname, join, relative, sep } from "node:path"; import type { ArtifactContext } from "./artifact-context.js"; import { - parsePersistedScanDraft, + parseAcceptedDiscoveryDraft, saveScanDraftCheckpoint, } from "./artifact-scan-draft.js"; import { artifactSchemaDocuments } from "./artifact-contracts.js"; @@ -13,7 +13,9 @@ import { writeJsonAtomic, } from "./deep-scan/artifacts.js"; import { + deepReductionForPersistence, parseDeepReduction, + projectDiscoveryCoverage, parseStoredScanDraft, reconcileDeepReduction, type DeepReductionInput, @@ -27,6 +29,22 @@ export { deepReductionInputSchema } from "./deep-scan/artifact-validation.js"; /** Read the findings and scan context assigned to this reducer. */ export async function getCodexSecurityDeepReducerInputs( context: ArtifactContext, +): Promise { + const inputs = await readDeepReductionSources(context); + const { sourceCoverage: _coverage, ...previous } = inputs.previous ?? {}; + return { + discoveries: inputs.discoveries.map(({ workerId, result }) => ({ + workerId, + result, + })), + previous: + inputs.previous === null ? null : (previous as DeepReductionInput), + }; +} + +/** Capture host coverage alongside the reducer's immutable finding inputs. */ +export async function readDeepReductionSources( + context: ArtifactContext, ): Promise { return withLogicalReducerErrors(context, async () => { const bound = bindDeepReducer(context); @@ -40,7 +58,7 @@ export async function getCodexSecurityDeepReducerInputs( await readJsonObject(worker.resultPath), "Accepted Standard worker " + worker.id, bound.scanId, - parsePersistedScanDraft, + parseAcceptedDiscoveryDraft, " has an invalid Standard scan result: ", " belongs to a different scan.", ); @@ -48,12 +66,29 @@ export async function getCodexSecurityDeepReducerInputs( throw new Error( "An assigned Standard worker wrote only a checkpoint, not a complete result.", ); - for (const [index, finding] of result.findings.entries()) { - const provenance = finding.provenance as Record; - provenance.sourceFindingIds = [`${worker.id}:${index}`]; - } - const { coverage: _coverage, ...reduction } = result; - return { workerId: worker.id, result: reduction }; + result.findings = result.findings.map((finding, index) => ({ + ...finding, + provenance: { + ...finding.provenance, + sourceFindingIds: [`${worker.id}:${index}`], + }, + })); + const { coverage, ...reduction } = result; + return { + workerId: worker.id, + ...(worker.attempt === undefined ? {} : { attempt: worker.attempt }), + coverage: projectDiscoveryCoverage( + coverage, + worker, + relative( + bound.artifacts.scanDir, + worker.artifactDir ?? dirname(worker.resultPath), + ) + .split(sep) + .join("/"), + ), + result: reduction, + }; }), ); const previous = await readPreviousReduction(bound); @@ -88,7 +123,7 @@ export async function recordCodexSecurityDeepReduction( throw new Error( "Deep reduction is only a checkpoint, not a complete result.", ); - const inputs = await getCodexSecurityDeepReducerInputs(context); + const inputs = await readDeepReductionSources(context); const expectedScanId = bound.scanId ?? inputs.previous?.scanId ?? @@ -104,8 +139,12 @@ export async function recordCodexSecurityDeepReduction( inputs.previous, ); - await saveScanDraftCheckpoint(context, reduction); - await writeJsonAtomic(bound.resultPath, reduction); + const persisted = deepReductionForPersistence( + reduction, + bound.state.persistSourceCoverage, + ); + await saveScanDraftCheckpoint(context, persisted); + await writeJsonAtomic(bound.resultPath, persisted); const documentWarning = await saveThreatModelDocument( context, reduction.threatModel, @@ -169,7 +208,7 @@ async function readPreviousReduction( await readJsonObject(previousReducerResultPath), "The previous accepted Deep reduction", bound.scanId, - parseDeepReduction, + (value) => parseDeepReduction(value, true), " has an invalid Standard scan result: ", " belongs to a different scan.", ); diff --git a/plugins/codex-security/mcp-app/src/artifact-io.ts b/plugins/codex-security/mcp-app/src/artifact-io.ts index a3649ca3c7..181c67807e 100644 --- a/plugins/codex-security/mcp-app/src/artifact-io.ts +++ b/plugins/codex-security/mcp-app/src/artifact-io.ts @@ -7,12 +7,16 @@ import { dirname, isAbsolute, join, sep } from "node:path"; export interface DeepReducerWorkerContext { id: string; resultPath: string; + /** Original output owner for relative evidence, including accepted checkpoints. */ + artifactDir?: string; + attempt?: number; } export interface DeepReducerContext { scanRoot: string; claimedWorkers: DeepReducerWorkerContext[]; previousReducerResultPath?: string; + persistSourceCoverage?: boolean; } /** diff --git a/plugins/codex-security/mcp-app/src/artifact-scan-draft.ts b/plugins/codex-security/mcp-app/src/artifact-scan-draft.ts index 79e25f089d..9a1e0128d2 100644 --- a/plugins/codex-security/mcp-app/src/artifact-scan-draft.ts +++ b/plugins/codex-security/mcp-app/src/artifact-scan-draft.ts @@ -19,9 +19,13 @@ import { readArtifactTextWithMetadata, replaceArtifactJson, } from "./artifact-io.js"; -import { artifactSchemaDocuments } from "./artifact-contracts.js"; +import { + acceptedDeepScanDraftInputSchema, + artifactSchemaDocuments, +} from "./artifact-contracts.js"; import { projectArtifactToolSchema } from "./artifact-schema-loader.js"; import { saveThreatModelDocument } from "./threat-model-document.js"; +import { normalizeAcceptedDiscoveryFinding } from "./accepted-discovery-normalization.js"; import { requireFiniteNumbers } from "./helpers/json.js"; const draftDefinitions = artifactSchemaDocuments["tools/scan-draft"].$defs; @@ -32,8 +36,20 @@ export const scanDraftInputSchema = projectArtifactToolSchema( const fileScanDraftInputSchema = scanDraftInputSchema.extend({ scanId: z.string().min(1), }); +const canonicalFileScanDraftInputSchema = fileScanDraftInputSchema.extend({ + coverage: draftDefinitions.coverage.extend({ + surfaces: z.array( + draftDefinitions.surface.extend({ id: z.string().min(1).optional() }), + ), + }), +}); +const acceptedCanonicalFileScanDraftInputSchema = + canonicalFileScanDraftInputSchema.extend({ + findings: acceptedDeepScanDraftInputSchema.shape.findings, + }); export const completedScanInputSchema = draftDefinitions.completedScanInput; -export type ScanDraftInput = z.infer; +// Host drafts include accepted historical findings as well as current inputs. +export type ScanDraftInput = z.infer; type DraftCoverage = ScanDraftInput["coverage"]; export type CompletedScanInput = z.infer; @@ -78,7 +94,9 @@ export async function recordCodexSecurityScanDraftViaWorkbench( signal?: AbortSignal, publication?: DeepScanPublication, ): Promise { - const parsed = parseScanDraft(input); + const parsed = publication + ? normalizeParsedScanDraft(parseAcceptedDeepScanDraft(input)) + : parseScanDraft(input); requireBoundScan(context, parsed, true); const finalDeepDraft = context.mode === "deep" && parsed.complete !== false; if (finalDeepDraft && resolvedDeferred(parsed.coverage).length > 0) { @@ -1521,10 +1539,12 @@ async function preserveDeepThreatModel( input: previous?.threatModel === undefined ? input - : scanDraftInputSchema.parse({ + : { ...input, - threatModel: previous.threatModel, - }), + ...scanDraftInputSchema.pick({ threatModel: true }).parse({ + threatModel: previous.threatModel, + }), + }, previousDigest: digest, }; } @@ -2034,12 +2054,15 @@ function safeRelativePath(value: string, context: string): string { } /** Admit file-authored drafts through the current contract after removing host fields. */ -export function parseCanonicalScanDraft(input: { - scanId?: string; - manifest: JsonObject; - findings: JsonObject; - coverage: JsonObject; -}): ScanDraftInput { +export function parseCanonicalScanDraft( + input: { + scanId?: string; + manifest: JsonObject; + findings: JsonObject; + coverage: JsonObject; + }, + acceptedFindings = false, +): ScanDraftInput { const scan = requireObject(input.manifest.scan, "scan draft manifest.scan"); for (const scanId of [ scan.id, @@ -2052,22 +2075,32 @@ export function parseCanonicalScanDraft(input: { ); } } - return parseCanonicalCheckpoint({ - scanId: input.scanId, - ...(scan.complete === undefined ? {} : { complete: scan.complete }), - ...(scan.scope === undefined ? {} : { scope: scan.scope }), - ...(scan.threatModel === undefined - ? {} - : { threatModel: scan.threatModel }), - findings: input.findings.findings, - coverage: input.coverage, - }); + return parseCanonicalCheckpoint( + { + scanId: input.scanId, + ...(scan.complete === undefined ? {} : { complete: scan.complete }), + ...(scan.scope === undefined ? {} : { scope: scan.scope }), + ...(scan.threatModel === undefined + ? {} + : { threatModel: scan.threatModel }), + findings: input.findings.findings, + coverage: input.coverage, + }, + acceptedFindings, + ); } function parseCanonicalCheckpoint( input: Record, + acceptedFindings = false, ): ScanDraftInput { - const parsed = parsePersistedCheckpoint(input); + const parsed = parsePersistedCheckpoint( + input, + acceptedFindings + ? acceptedCanonicalFileScanDraftInputSchema + : canonicalFileScanDraftInputSchema, + acceptedFindings, + ); for (const [index, finding] of parsed.findings.entries()) { for (const field of ["locations", "codeEvidence"] as const) { for (const [locationIndex, location] of ( @@ -2084,7 +2117,10 @@ function parseCanonicalCheckpoint( } export function parseScanDraft(input: unknown): ScanDraftInput { - const parsed = parsePersistedScanDraft(input); + return normalizeParsedScanDraft(parsePersistedScanDraft(input)); +} + +function normalizeParsedScanDraft(parsed: ScanDraftInput): ScanDraftInput { const deferredIds = new Set(); const ambiguousIds = ambiguousGenericDeferredIds([parsed]); for (const row of parsed.coverage.deferred) { @@ -2121,8 +2157,32 @@ export function parsePersistedScanDraft( return parsed; } +/** Replay host-accepted Deep findings while preserving their original details. */ +export function parseAcceptedDeepScanDraft( + input: unknown, + schema: z.ZodType = acceptedDeepScanDraftInputSchema, +): ScanDraftInput { + const parsed = schema.parse(input); + validateFindingSemantics(parsed.findings, true); + delete parsed.coverage.completeness; + return parsed; +} + +/** Replay the read contract used when the raw discovery bytes were accepted. */ +export function parseAcceptedDiscoveryDraft(input: unknown): ScanDraftInput { + const compatible = structuredClone(input); + if (isObject(compatible) && Array.isArray(compatible.findings)) { + for (const finding of compatible.findings) { + if (isObject(finding)) normalizeAcceptedDiscoveryFinding(finding); + } + } + return parseAcceptedDeepScanDraft(compatible); +} + function parsePersistedCheckpoint( input: Record, + schema: z.ZodType = fileScanDraftInputSchema, + acceptedFindings = false, ): ScanDraftInput { const compatible = structuredClone(input); if (isObject(compatible.scope)) { @@ -2151,7 +2211,9 @@ function parsePersistedCheckpoint( delete finding.fingerprints; } } - return parsePersistedScanDraft(compatible, fileScanDraftInputSchema); + return acceptedFindings + ? parseAcceptedDeepScanDraft(compatible, schema) + : parsePersistedScanDraft(compatible, schema); } function requireBoundScan( @@ -2519,7 +2581,10 @@ function inventoryStrategy( return "repository"; } -export function validateFindingSemantics(findings: JsonObject[]): void { +export function validateFindingSemantics( + findings: JsonObject[], + accepted = false, +): void { for (const [findingIndex, finding] of findings.entries()) { const severity = finding.severity as JsonObject; if ( @@ -2545,25 +2610,30 @@ export function validateFindingSemantics(findings: JsonObject[]): void { } const evidenceIds = new Set(); - for (const [evidenceIndex, evidence] of ( - (finding.codeEvidence as JsonObject[] | undefined) ?? [] - ).entries()) { - const id = evidence.id as string; - if (evidenceIds.has(id)) { - throw new Error( - `scan draft: findings[${findingIndex}].codeEvidence[${evidenceIndex}].id ` + - `duplicates ${id}.`, - ); - } - evidenceIds.add(id); - if ( - typeof evidence.endLine === "number" && - evidence.endLine < (evidence.startLine as number) - ) { - throw new Error( - `scan draft: findings[${findingIndex}].codeEvidence[${evidenceIndex}].endLine ` + - "must not precede startLine.", - ); + const catalogs = accepted + ? ["codeEvidence", "code_evidence"] + : ["codeEvidence"]; + for (const catalog of catalogs) { + for (const [evidenceIndex, evidence] of ( + (finding[catalog] as JsonObject[] | undefined) ?? [] + ).entries()) { + const id = evidence.id as string; + if (evidenceIds.has(id)) { + throw new Error( + `scan draft: findings[${findingIndex}].${catalog}[${evidenceIndex}].id ` + + `duplicates ${id}.`, + ); + } + evidenceIds.add(id); + if ( + typeof evidence.endLine === "number" && + evidence.endLine < (evidence.startLine as number) + ) { + throw new Error( + `scan draft: findings[${findingIndex}].${catalog}[${evidenceIndex}].endLine ` + + "must not precede startLine.", + ); + } } } @@ -2572,8 +2642,12 @@ export function validateFindingSemantics(findings: JsonObject[]): void { ["validation", finding.validation], ["attackPath", finding.attackPath], ]; + if (accepted) referencedSections.push(["root_cause", finding.root_cause]); if (isObject(finding.attackPath)) { - for (const sectionName of ["dataflow", "reachability"]) { + const names = accepted + ? ["dataflow", "dataFlow", "data_flow", "reachability"] + : ["dataflow", "reachability"]; + for (const sectionName of names) { referencedSections.push([ `attackPath.${sectionName}`, finding.attackPath[sectionName], @@ -2582,16 +2656,21 @@ export function validateFindingSemantics(findings: JsonObject[]): void { } for (const [sectionName, section] of referencedSections) { if (!isObject(section)) continue; - const references = section.evidenceRefs; - if (references === undefined) continue; - if ( - !Array.isArray(references) || - references.some((reference) => !evidenceIds.has(reference)) - ) { - throw new Error( - `scan draft: findings[${findingIndex}].${sectionName}.evidenceRefs ` + - "must refer to that finding's existing code-evidence IDs.", - ); + const fields = accepted + ? ["evidenceRefs", "evidence_refs"] + : ["evidenceRefs"]; + for (const field of fields) { + const references = section[field]; + if (references === undefined) continue; + if ( + !Array.isArray(references) || + references.some((reference) => !evidenceIds.has(reference)) + ) { + throw new Error( + `scan draft: findings[${findingIndex}].${sectionName}.${field} ` + + "must refer to that finding's existing code-evidence IDs.", + ); + } } } } diff --git a/plugins/codex-security/mcp-app/src/deep-scan/artifact-validation.ts b/plugins/codex-security/mcp-app/src/deep-scan/artifact-validation.ts index 7ad6f74409..cf927ff04f 100644 --- a/plugins/codex-security/mcp-app/src/deep-scan/artifact-validation.ts +++ b/plugins/codex-security/mcp-app/src/deep-scan/artifact-validation.ts @@ -1,3 +1,5 @@ +import { z } from "zod"; +import { join, posix } from "node:path"; import { isDeepStrictEqual } from "node:util"; import { dirname, relative, sep } from "node:path"; import type { JsonObject } from "../types.js"; @@ -11,6 +13,8 @@ import { artifactSchemaDocuments } from "../artifact-contracts.js"; import { projectArtifactToolSchema } from "../artifact-schema-loader.js"; import { requireFiniteNumbers } from "../helpers/json.js"; import { + parseAcceptedDeepScanDraft, + parseAcceptedDiscoveryDraft, parsePersistedScanDraft, preserveFindingDetails, saveScanDraftCheckpoint, @@ -25,13 +29,21 @@ import { } from "./artifacts.js"; import type { DeepScanArtifacts } from "./artifacts.js"; -export type DeepReductionInput = Omit; +export type DeepReductionInput = Omit & { + /** Host projection of accepted source coverage; never supplied by the reducer. */ + sourceCoverage?: ScanDraftInput["coverage"]; +}; export const deepReductionInputSchema = projectArtifactToolSchema( artifactSchemaDocuments["tools/deep-reducer"].$defs.reductionInput, ); export interface DeepReductionSources { - discoveries: { workerId: string; result: DeepReductionInput }[]; + discoveries: { + workerId: string; + attempt?: number; + coverage?: ScanDraftInput["coverage"]; + result: DeepReductionInput; + }[]; previous: DeepReductionInput | null; } @@ -40,10 +52,48 @@ export interface ReducerArtifactValidation { result: DeepReductionInput; } +export function deepReductionToScanDraft( + result: DeepReductionInput, +): ScanDraftInput { + const { sourceCoverage, ...draft } = structuredClone(result); + return { ...draft, coverage: sourceCoverage ?? unknownSourceCoverage() }; +} + +/** Older workflow readers reject the host field; retain their persisted shape. */ +export function deepReductionForPersistence( + result: DeepReductionInput, + persistSourceCoverage = false, +): DeepReductionInput { + if (persistSourceCoverage) return result; + const { sourceCoverage: _coverage, ...legacy } = result; + return legacy; +} + +function workerCoverageIds( + rows: readonly JsonObject[], + workerId: string, +): string[] { + const usedIds = new Set( + rows.flatMap((row) => (typeof row.id === "string" ? [row.id] : [])), + ); + const emittedIds = new Set(); + return rows.map((row, index) => { + const baseId = typeof row.id === "string" ? row.id : String(index); + let id = baseId; + if (typeof row.id !== "string" || emittedIds.has(id)) { + for (let suffix = 2; usedIds.has(id); suffix++) + id = `${baseId}-${suffix}`; + } + usedIds.add(id); + emittedIds.add(id); + return `${workerId}/${id}`; + }); +} + /** Reduction combines findings; it does not complete the source workers' review. */ export function aggregateDiscoveryCoverage( artifacts: DeepScanArtifacts, - discoveries: Pick[], + discoveries: Pick[], ): ScanDraftInput["coverage"] { const coverage: ScanDraftInput["coverage"] = { surfaces: [], @@ -52,7 +102,7 @@ export function aggregateDiscoveryCoverage( }; for (const worker of discoveries) { const source = structuredClone(worker.result.coverage); - const prefix = relative(artifacts.scanDir, dirname(worker.resultPath)) + const prefix = relative(artifacts.scanDir, worker.artifactDir) .split(sep) .join("/"); const scopedId = (id: string) => `${worker.id}/${id}`; @@ -62,24 +112,12 @@ export function aggregateDiscoveryCoverage( "deferred", ] as const) { const rows = source[field] as JsonObject[]; - const usedIds = new Set( - rows.flatMap((row) => (typeof row.id === "string" ? [row.id] : [])), - ); - const emittedIds = new Set(); + const ids = workerCoverageIds(rows, worker.id); for (const [index, row] of rows.entries()) { const retained: JsonObject = { ...row, workerId: worker.id }; // Independent reviews reuse local IDs. Preserve their links without // allowing one worker's disposition to stand in for another's work. - const baseId = typeof row.id === "string" ? row.id : String(index); - let id = baseId; - if (typeof row.id !== "string" || emittedIds.has(id)) { - for (let suffix = 2; usedIds.has(id); suffix++) { - id = `${baseId}-${suffix}`; - } - } - usedIds.add(id); - emittedIds.add(id); - retained.id = scopedId(id); + retained.id = ids[index]; if (field === "deferred" && Array.isArray(row.surfaceIds)) { retained.surfaceIds = (row.surfaceIds as string[]).map(scopedId); } @@ -106,9 +144,32 @@ export function aggregateDiscoveryCoverage( return coverage; } -export function parseDeepReduction(input: unknown): DeepReductionInput { - const parsed = deepReductionInputSchema.parse(input); - validateFindingSemantics(parsed.findings); +export function parseDeepReduction( + input: unknown, + persisted = false, +): DeepReductionInput { + const { sourceCoverage, ...submitted } = z + .record(z.string(), z.unknown()) + .parse(input); + const standard = { + ...submitted, + coverage: { surfaces: [], explicitExclusions: [], deferred: [] }, + }; + const parsed = persisted + ? (({ coverage: _coverage, ...draft }) => draft)( + parseAcceptedDeepScanDraft(standard), + ) + : deepReductionInputSchema.parse(submitted); + validateFindingSemantics(parsed.findings, persisted); + if (persisted && sourceCoverage !== undefined) { + return { + ...parsed, + sourceCoverage: parseAcceptedDeepScanDraft({ + ...standard, + coverage: sourceCoverage, + }).coverage, + }; + } return parsed; } @@ -117,20 +178,45 @@ export async function validateDiscoveryArtifacts( artifacts: DeepScanArtifacts, resultPath: string, expectedScanId: string, + validateWriteups = true, + accepted = false, + artifactDir = dirname(resultPath), ): Promise { - await requireRegularFile(resultPath, artifacts.workersRoot); - const result = parseStoredScanDraft( - await readJsonObject(resultPath), - "Standard scan worker", + const result = await readDiscoveryAuditDraft( + artifacts, + resultPath, expectedScanId, - parsePersistedScanDraft, + validateWriteups, + accepted, + artifactDir, ); if (result.complete === false) throw new Error( "Standard scan worker wrote only a checkpoint; its audit is not complete.", ); + return result; +} + +export async function readDiscoveryAuditDraft( + artifacts: DeepScanArtifacts, + resultPath: string, + expectedScanId: string, + validateWriteups = true, + accepted = false, + artifactDir = dirname(resultPath), +): Promise { + await requireRegularFile(resultPath, artifacts.workersRoot); + const result = parseStoredScanDraft( + await readJsonObject(resultPath), + "Standard scan worker", + expectedScanId, + accepted ? parseAcceptedDiscoveryDraft : parsePersistedScanDraft, + ); + // Unfinished drafts remain resumable; complete drafts must retain their evidence. + if (result.complete === false) return result; + if (validateWriteups) await validateWriteupFiles(artifacts, result); validateRetainedCoverage(result.coverage); - const root = dirname(resultPath); + const root = artifactDir; const receiptPaths = new Map(); for (const row of result.coverage.surfaces as JsonObject[]) { for (const ref of (row.receiptRefs as string[] | undefined) ?? []) { @@ -153,7 +239,29 @@ export async function validateDiscoveryArtifacts( return result; } -/** Validate the complete aggregate and derive convergence from retained source ancestry. */ +async function validateWriteupFiles( + artifacts: DeepScanArtifacts, + result: Pick, +): Promise { + for (const [index, finding] of result.findings.entries()) { + const reportPath = (finding.writeup as { reportPath: string } | undefined) + ?.reportPath; + if (!reportPath) continue; + try { + await requireRegularFile( + join(artifacts.scanDir, reportPath), + artifacts.scanDir, + ); + } catch (cause) { + throw new Error( + `findings[${index}].writeup.reportPath: expected a file inside the scan directory: ${reportPath}`, + { cause }, + ); + } + } +} + +/** Validate the complete aggregate and derive convergence from stable finding identities. */ export async function validateReducerArtifacts( input: { artifacts: DeepScanArtifacts; @@ -162,6 +270,8 @@ export async function validateReducerArtifacts( reducerId: string; previousReducerResultPath?: string; sources?: DeepReductionSources; + persistSourceCoverage?: boolean; + accepted?: boolean; }, expectedScanId?: string, ): Promise { @@ -178,7 +288,7 @@ export async function validateReducerArtifacts( await readJsonObject(resultPath), reducerId, expectedScanId, - parseDeepReduction, + (value) => parseDeepReduction(value, input.accepted), ); if (result.complete === false) throw new Error( @@ -192,7 +302,7 @@ export async function validateReducerArtifacts( await readJsonObject(previousReducerResultPath), "Previous successful reducer", result.scanId, - parseDeepReduction, + (value) => parseDeepReduction(value, true), ); } @@ -202,13 +312,20 @@ export async function validateReducerArtifacts( input.sources.discoveries, input.sources.previous, ); + } else { + validateRetainedFindings(result, [], previous); + } + await validateWriteupFiles(artifacts, result); + if (input.sources) { + const persisted = deepReductionForPersistence( + result, + input.persistSourceCoverage, + ); await saveScanDraftCheckpoint( { root: artifactDir, repoRoot: artifacts.scanDir, layout: "reducer" }, - result, + persisted, ); - await writeJsonAtomic(resultPath, result); - } else { - validateRetainedFindings(result, [], previous); + await writeJsonAtomic(resultPath, persisted); } const previousFindingIds = new Set( input.sources @@ -246,6 +363,11 @@ export function reconcileDeepReduction( })), }); const result = structuredClone(input); + result.sourceCoverage = aggregateSourceCoverage(discoveries, previous); + if (result.complete === false) + throw new Error( + "Deep reduction is only a checkpoint, not a complete result.", + ); for (const source of [ ...discoveries.map((discovery) => discovery.result), ...(previous ? [previous] : []), @@ -301,6 +423,134 @@ export function reconcileDeepReduction( return result; } +/** Keep independent reviews separate: matching labels do not resolve another pass's proof gap. */ +export function aggregateSourceCoverage( + discoveries: DeepReductionSources["discoveries"], + previous: DeepReductionInput | null, +): ScanDraftInput["coverage"] { + const sources = [ + ...(previous ? [previous.sourceCoverage ?? unknownSourceCoverage()] : []), + ...discoveries.map((source) => source.coverage ?? unknownSourceCoverage()), + ]; + const result: ScanDraftInput["coverage"] = { + surfaces: [], + explicitExclusions: [], + deferred: [], + reviews: [], + }; + for (const field of [ + "surfaces", + "explicitExclusions", + "deferred", + "openQuestions", + "reviews", + ]) { + const entries = sources.flatMap( + (source) => (source[field] as unknown[] | undefined) ?? [], + ); + if (entries.length || field !== "openQuestions") + result[field] = structuredClone(entries); + } + return result; +} + +function unknownSourceCoverage(): ScanDraftInput["coverage"] { + return { + surfaces: [], + explicitExclusions: [], + deferred: [], + }; +} + +/** Qualify worker-local IDs and receipt paths before combining accepted coverage. */ +export function projectDiscoveryCoverage( + coverage: ScanDraftInput["coverage"], + worker: { id: string; attempt?: number }, + artifactPrefix: string, +): ScanDraftInput["coverage"] { + const archivePrefix = `${posix.dirname(artifactPrefix)}/attempts/`; + const provenance = { + workerId: worker.id, + ...(worker.attempt === undefined ? {} : { attempt: worker.attempt }), + }; + const surfaces = coverage.surfaces; + const projectedSurfaceIds = workerCoverageIds(surfaces, worker.id); + const exclusionIds = workerCoverageIds( + coverage.explicitExclusions, + worker.id, + ); + const deferredIds = workerCoverageIds(coverage.deferred, worker.id); + const surfaceIds = new Map(); + for (const [index, surface] of surfaces.entries()) { + if (surface.id !== undefined && !surfaceIds.has(surface.id)) + surfaceIds.set(surface.id, projectedSurfaceIds[index]); + } + const project = >(item: Item) => { + const projected = structuredClone(item); + const sourceProvenance = + typeof projected.provenance === "object" && + projected.provenance !== null && + !Array.isArray(projected.provenance) + ? (projected.provenance as Record) + : {}; + // Preserve source descriptions while deriving identity only from the accepted owner. + for (const key of ["workerId", "attempt", "sourceId", "candidateId"]) + delete sourceProvenance[key]; + return { + ...projected, + workerId: worker.id, + provenance: { + ...sourceProvenance, + ...provenance, + ...(item.id === undefined ? {} : { sourceId: item.id }), + ...(item.candidateId === undefined + ? {} + : { candidateId: item.candidateId }), + }, + }; + }; + return { + reviews: [{ ...provenance }], + surfaces: surfaces.map((surface, index) => ({ + ...project(surface), + id: projectedSurfaceIds[index], + receiptRefs: ((surface.receiptRefs as string[] | undefined) ?? []).map( + (ref) => + ref.startsWith(archivePrefix) || ref.startsWith(`${artifactPrefix}/`) + ? ref + : `${artifactPrefix}/${ref}`, + ), + })), + explicitExclusions: coverage.explicitExclusions.map((item, index) => ({ + ...project(item), + id: exclusionIds[index], + })), + deferred: coverage.deferred.map((item, index) => ({ + ...project(item), + id: deferredIds[index], + ...(item.candidateId === undefined + ? {} + : { + candidateId: `${worker.id}-attempt-${worker.attempt ?? "unknown"}-candidate-${index + 1}`, + }), + ...(item.surfaceIds === undefined + ? {} + : { + surfaceIds: (item.surfaceIds as string[]).map( + (id) => surfaceIds.get(id) ?? id, + ), + }), + })), + ...(coverage.openQuestions === undefined + ? {} + : { + openQuestions: coverage.openQuestions.map((question) => + project(typeof question === "string" ? { question } : question), + ), + }), + }; +} + function findingSourceIds(finding: Record): string[] { const { sourceFindingIds, sourceFindings } = finding.provenance as { sourceFindingIds?: string[]; diff --git a/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts b/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts index d09cfbd3e7..9ff60f5014 100644 --- a/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts +++ b/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts @@ -1,4 +1,5 @@ /// +import { publishSelectedDeepScan } from "./finalization.js"; import { randomUUID } from "node:crypto"; import { promises as fs } from "node:fs"; import { basename, dirname, join } from "node:path"; @@ -8,11 +9,14 @@ import { ensureDeepScanDirectories, } from "./artifacts.js"; import { + aggregateSourceCoverage, + deepReductionToScanDraft, aggregateDiscoveryCoverage, validateDiscoveryArtifacts, validateReducerArtifacts, type DeepReductionInput, } from "./artifact-validation.js"; +import { readDeepReductionSources } from "../artifact-deep-reducer.js"; import { scanDraftInputSchema, type DeepScanPublication, @@ -21,6 +25,7 @@ import { } from "../artifact-scan-draft.js"; import type { DeepScanArtifacts } from "./artifacts.js"; import { DeepScanWorkerRunner } from "./worker-runner.js"; +import { isTransientPersistenceError } from "./store.js"; import type { AcceptedDiscovery, DedupOutcome, @@ -91,6 +96,8 @@ export interface CoordinatorOptions { signal: AbortSignal, publication: DeepScanPublication, ) => Promise; + /** Complete the enclosing scan after selected publication, before local waiters settle. */ + onFinalized?: (run: DeepScanRunState, signal: AbortSignal) => Promise; onStopped?: (run: DeepScanRunState) => Promise; } @@ -123,6 +130,7 @@ export class DeepScanCoordinator { private canceled = false; private externallyFailed = false; private failurePersisted = false; + private parentPublicationPending = false; private setupComplete = false; private discoveryDeadlineReached = false; private state: DeepScanRunState; @@ -175,7 +183,7 @@ export class DeepScanCoordinator { if (this.started) return; this.started = true; this.log({ event: "coordinator_started", scanId: this.state.scanId }); - this.scheduleDiscoveryDeadline(); + if (!this.state.finalizationInput) this.scheduleDiscoveryDeadline(); this.scheduleHeartbeat(); void this.run().catch((error: unknown) => { this.log({ @@ -304,7 +312,8 @@ export class DeepScanCoordinator { ) { // Terminal discovery has no active writers to stop. Keep its outcome // until the parent actually accepts cancellation. - if (this.state.status === "running") this.cancel(reason); + if (this.state.status === "running" || this.parentPublicationPending) + this.cancel(reason); await this.cancellationReady.promise; if (!this.failurePersisted) { try { @@ -359,21 +368,46 @@ export class DeepScanCoordinator { } private async run(): Promise { + let finalizationFailure: { error: unknown } | undefined; + let selectingFinalization = false; try { await ensureDeepScanDirectories(this.artifacts); if (this.canceled || this.externallyFailed) return; + if (this.state.finalizationInput) { + await this.completeSelectedFinalization(); + return; + } this.setupComplete = true; const schedulerResult = await this.runScheduler(); if (this.canceled || this.externallyFailed) return; + if (this.state.workflowVersion === "deep-security-scan/v2") { + if (!this.options.store.selectFinalization) + throw new Error( + "The Deep Scan store cannot select finalization input.", + ); + selectingFinalization = true; + this.state = await this.options.store.selectFinalization({ + scanId: this.state.scanId, + reason: schedulerResult.reason, + manifestPath: join(this.state.scanDir, "scan-manifest.json"), + resultPath: schedulerResult.resultPath, + omittedWorkerIds: schedulerResult.omittedWorkerIds, + }); + selectingFinalization = false; + await this.completeSelectedFinalization(); + return; + } const draft = schedulerResult.result - ? { - ...structuredClone(schedulerResult.result), - coverage: aggregateDiscoveryCoverage( - this.artifacts, - schedulerResult.discoveries, - ), - } + ? deepReductionToScanDraft({ + ...schedulerResult.result, + sourceCoverage: + schedulerResult.result.sourceCoverage ?? + aggregateDiscoveryCoverage( + this.artifacts, + schedulerResult.discoveries, + ), + }) : scanDraftInputSchema.parse({ scanId: this.state.scanId, findings: [], @@ -422,12 +456,33 @@ export class DeepScanCoordinator { reason: schedulerResult.reason, }); } catch (error) { + const cancellationPendingAtFailure = + this.cancellationPersistence !== undefined; if (this.canceled || this.externallyFailed) { return; } if (await this.stopAfterOwnershipChange(this.options.threadId, error)) { return; } + if ( + this.state.finalizationInput || + (selectingFinalization && isTransientPersistenceError(error)) + ) { + // A lost selection response may hide a committed input even when the + // ownership read also fails. Leave that selection attempt resumable. + // Publication can be retried from the committed input without model work. + this.log({ + event: "coordinator_publication_pending", + scanId: this.state.scanId, + reason: errorKind(error), + }); + if (cancellationPendingAtFailure) { + finalizationFailure = { error }; + } else if (this.stopLocally()) { + this.terminalResult.reject(error); + } + return; + } if ( this.abortController.signal.reason instanceof DeepScanNonRetryableError ) { @@ -571,11 +626,64 @@ export class DeepScanCoordinator { } if (this.canceled && !this.failurePersisted) this.state = { ...this.state, status: "canceled" }; - if (this.stopLocally()) - this.terminalResult.resolve(cloneState(this.state)); + if (this.stopLocally()) { + if (finalizationFailure && !this.canceled && !this.externallyFailed) + this.terminalResult.reject(finalizationFailure.error); + else this.terminalResult.resolve(cloneState(this.state)); + } } } + private async completeSelectedFinalization(): Promise { + this.state = await publishSelectedDeepScan({ + run: this.state, + artifacts: this.artifacts, + signal: this.publicationAbortController.signal, + publish: async (...args) => { + await this.options.onComplete?.(...args); + }, + finish: async (input) => { + try { + return await this.options.store.finish(input); + } catch (error) { + // The terminal mutation may commit even when every retry loses its + // response. Recover our committed run so parent finalization still runs. + const current = this.options.threadId + ? await this.options.store + .get(this.state.scanId, this.options.threadId) + .catch(() => undefined) + : undefined; + if ( + !this.publicationAbortController.signal.aborted && + current?.status === "succeeded" && + current.coordinatorGeneration === + this.state.coordinatorGeneration && + current.finalizationInput?.resultPath === + this.state.finalizationInput?.resultPath && + current.finalizationInput?.resultSha256 === + this.state.finalizationInput?.resultSha256 && + current.finalizationInput?.selectedAt === + this.state.finalizationInput?.selectedAt && + current.manifestPath === input.manifestPath && + current.terminalReason === input.reason + ) + return current; + throw error; + } + }, + }); + if (this.canceled || this.externallyFailed) return; + this.parentPublicationPending = true; + try { + await this.options.onFinalized?.( + cloneState(this.state), + this.publicationAbortController.signal, + ); + } finally { + this.parentPublicationPending = false; + } + if (this.canceled || this.externallyFailed) return; + } private stopLocally(): boolean { if (this.terminal) return false; this.terminal = true; @@ -674,7 +782,8 @@ export class DeepScanCoordinator { ): Promise { if (this.externallyFailed || this.terminal || !threadId) return this.externallyFailed; - let current = confirmedOwnershipChange(error, this.state.scanId)?.run; + const ownershipChange = confirmedOwnershipChange(error, this.state.scanId); + let current = ownershipChange?.run; try { current ??= await this.options.store.get(this.state.scanId, threadId); this.failurePersisted ||= current.status === "failed"; @@ -688,11 +797,30 @@ export class DeepScanCoordinator { } if (this.externallyFailed || this.terminal) return this.externallyFailed; const replacementConfirmed = - current.status === "running" && - current.coordinatorGeneration !== undefined && - this.state.coordinatorGeneration !== undefined && - current.coordinatorGeneration > this.state.coordinatorGeneration; - if (current.status === "running" && !replacementConfirmed) return false; + ownershipChange !== undefined || + (current.status === "running" && + current.coordinatorGeneration !== undefined && + this.state.coordinatorGeneration !== undefined && + current.coordinatorGeneration > this.state.coordinatorGeneration); + // finish-deep-scan commits before the enclosing scan's finalizer. Its own + // succeeded generation is still ours until that finalizer settles. + const completingSelectedParent = + current.status === "succeeded" && + this.state.finalizationInput !== undefined && + current.coordinatorGeneration === this.state.coordinatorGeneration; + if ( + !replacementConfirmed && + (current.status === "running" || completingSelectedParent) + ) { + // A selection response can be lost after its transaction commits. + if (current.finalizationInput) + this.state = { + ...this.state, + finalizationInput: current.finalizationInput, + terminalReason: current.terminalReason, + }; + return false; + } this.externallyFailed = true; this.state = current; @@ -800,9 +928,11 @@ export class DeepScanCoordinator { ); } if (reducerFailures >= errorLimit) { - const failure = (this.state.persistedWorkers ?? []).findLast( - (worker) => worker.kind === "dedup" && worker.status === "failed", - ); + const failure = [...(this.state.persistedWorkers ?? [])] + .reverse() + .find( + (worker) => worker.kind === "dedup" && worker.status === "failed", + ); throw reducerErrorLimitError( reducerFailures, errorLimit, @@ -843,14 +973,22 @@ export class DeepScanCoordinator { wakeScheduler = resolvePromise; }); } - return settlements.shift()!; + const settlement = settlements.shift(); + if (!settlement) + throw new Error("Deep Scan scheduler woke without a settled task."); + return settlement; }; - const reconcileRemainingDiscoveries = async (): Promise< - unknown | undefined - > => { - const results = await Promise.allSettled([...active.values()]); + const reconcileRemainingDiscoveries = async ( + succeededState: "buffered" | "omitted", + ): Promise => { + const entries = [...active.entries()]; + const results = await Promise.allSettled( + entries.map(([, promise]) => promise), + ); let firstFailure: unknown | undefined; - for (const result of results) { + for (const [index, result] of results.entries()) { + const workerId = entries[index]?.[0]; + if (workerId) active.delete(workerId); if (result.status === "rejected") { if (confirmedOwnershipChange(result.reason, this.state.scanId)) throw result.reason; @@ -859,9 +997,18 @@ export class DeepScanCoordinator { } const outcome = result.value; if (outcome.status === "failed") { - if (!outcome.replaceableFailureKind) firstFailure ??= outcome.error; + if (!outcome.replaceableFailureKind) { + firstFailure ??= outcome.error; + } } else if (outcome.status === "succeeded") { - omittedWorkerIds.push(outcome.worker.id); + accepted.set(outcome.worker.id, outcome.worker); + if (succeededState === "omitted") { + omittedWorkerIds.push(outcome.worker.id); + } else if ( + !buffer.some((worker) => worker.id === outcome.worker.id) + ) { + buffer.push(outcome.worker); + } } } return firstFailure; @@ -870,27 +1017,32 @@ export class DeepScanCoordinator { unknown | undefined > => { if (!reducer) return undefined; - const [result] = await Promise.allSettled([reducer]); - if (result.status === "rejected") return result.reason; + const pendingReducer = reducer; + reducer = undefined; + const [result] = await Promise.allSettled([pendingReducer]); + if (!result || result.status === "rejected") { + return result?.status === "rejected" ? result.reason : undefined; + } const outcome = result.value; - if ("status" in outcome) return outcome.error; + if ("status" in outcome) { + buffer = [...outcome.consumed, ...buffer].sort( + compareCompletionSequence, + ); + return outcome.error; + } if (!this.canceled && !this.externallyFailed) this.state = outcome.run; + previousReducerResultPath = outcome.resultPath; + latestResult = outcome.result; return undefined; }; const settleFailure = async (error: unknown): Promise => { this.abortController.abort(errorMessage(error)); const reducerFailure = await reconcileReducerSettlement(); - const discoveries = await Promise.allSettled([...active.values()]); + const discoveryFailure = await reconcileRemainingDiscoveries("buffered"); // SDK iterator cleanup can let a later generic rejection settle first. // Inspect every active worker's result before discarding confirmed ownership. - const failures = [ - error, - reducerFailure, - ...discoveries.flatMap((result) => - result.status === "rejected" ? [result.reason] : [], - ), - ]; + const failures = [error, reducerFailure, discoveryFailure]; throw ( failures.find((failure) => confirmedOwnershipChange(failure, this.state.scanId), @@ -953,6 +1105,7 @@ export class DeepScanCoordinator { label: `dedup-${String(reducerSequence).padStart(4, "0")}`, consumed, previousReducerResultPath, + previousSourceCoverage: latestResult?.sourceCoverage, }), ); observe(reducer); @@ -1029,7 +1182,9 @@ export class DeepScanCoordinator { reducer = undefined; if ("status" in outcome) { - buffer = [...outcome.consumed, ...buffer]; + buffer = [...outcome.consumed, ...buffer].sort( + compareCompletionSequence, + ); reducerFailures += 1; this.log({ event: "dedup_worker_replaced", @@ -1064,7 +1219,7 @@ export class DeepScanCoordinator { // Convergence cancels active workers, but their promises must settle before // the manifest records which results completed and which were canceled. - const lateFailure = await reconcileRemainingDiscoveries(); + const lateFailure = await reconcileRemainingDiscoveries("omitted"); // Once Deep reaches saturation, late worker errors cannot fail the scan. if (lateFailure && stopReason !== "saturated") throw lateFailure; @@ -1079,7 +1234,7 @@ export class DeepScanCoordinator { } return { reason: stopReason, - omittedWorkerIds: [...new Set(omittedWorkerIds)], + omittedWorkerIds: unique(omittedWorkerIds), result: latestResult, resultPath: previousReducerResultPath, discoveries: [...accepted.values()].filter((worker) => @@ -1093,21 +1248,39 @@ export class DeepScanCoordinator { for (const worker of this.state.persistedWorkers ?? []) { if (worker.kind !== "discovery" || worker.status !== "succeeded") continue; - if (!worker.resultManifestPath || !worker.completionSequence) { + // Migrated workers can have frozen merge inputs without an attempt record. + const claimedInput = this.state.persistedDedupInputs?.find( + (input) => + input.discoveryWorkerId === worker.id && + (input.attempt === undefined || input.attempt === worker.attempt) && + input.resultManifestPath, + ); + const resultPath = + worker.acceptedResultPath ?? + claimedInput?.resultManifestPath ?? + worker.resultManifestPath; + if (!resultPath || !worker.completionSequence) { throw new Error( `Accepted discovery ${worker.id} has incomplete persisted evidence.`, ); } const result = await validateDiscoveryArtifacts( this.artifacts, - worker.resultManifestPath, + resultPath, this.state.scanId, + this.state.workflowVersion === "deep-security-scan/v2", + true, + worker.artifactDir, ); recovered.push({ id: worker.id, - resultPath: worker.resultManifestPath, + label: basename(dirname(worker.promptPath)), + artifactDir: worker.artifactDir, + resultPath, result, completionSequence: worker.completionSequence, + attempt: worker.attempt, + ...(worker.threadId ? { threadId: worker.threadId } : {}), }); } return recovered.sort(compareCompletionSequence); @@ -1115,11 +1288,13 @@ export class DeepScanCoordinator { private async recoverCompletedReducers( discoveries: AcceptedDiscovery[], - ): Promise<{ resultPath?: string; result?: DeepReductionInput }> { - const discoveryIds = new Set(discoveries.map((worker) => worker.id)); + ): Promise<{ result?: DeepReductionInput; resultPath?: string }> { + const discoveriesById = new Map( + discoveries.map((worker) => [worker.id, worker]), + ); const inputs = this.state.persistedDedupInputs ?? []; - let resultPath: string | undefined; let latestResult: DeepReductionInput | undefined; + let latestResultPath: string | undefined; const completedReducers = (this.state.persistedWorkers ?? []) .filter( (worker) => worker.kind === "dedup" && worker.status === "succeeded", @@ -1140,33 +1315,80 @@ export class DeepScanCoordinator { : left.id.localeCompare(right.id); }); for (const worker of completedReducers) { - if (!worker.resultManifestPath) { + // A later merge claim can retain a legacy aggregate's accepted reference. + const resultPath = + worker.acceptedResultPath ?? + this.state.persistedMergeClaims?.find( + (claim) => + claim.previousWorkerId === worker.id && claim.previousResultPath, + )?.previousResultPath ?? + worker.resultManifestPath; + if (!resultPath) { throw new Error( `Completed reducer ${worker.id} has no persisted result manifest.`, ); } const consumed = inputs .filter((input) => input.dedupWorkerId === worker.id) - .map((input) => discoveryIds.has(input.discoveryWorkerId)); + .sort((left, right) => left.inputOrder - right.inputOrder) + .map((input) => { + const discovery = discoveriesById.get(input.discoveryWorkerId); + return ( + discovery && { + ...discovery, + resultPath: input.resultManifestPath ?? discovery.resultPath, + attempt: input.attempt ?? discovery.attempt, + } + ); + }); if (consumed.length === 0 || consumed.some((value) => !value)) { throw new Error( `Completed reducer ${worker.id} has incomplete persisted inputs.`, ); } + const accepted = consumed as AcceptedDiscovery[]; + const claim = this.state.persistedMergeClaims?.find( + (item) => item.workerId === worker.id, + ); const { result } = await validateReducerArtifacts( { artifacts: this.artifacts, artifactDir: worker.artifactDir, - resultPath: worker.resultManifestPath, + resultPath, reducerId: worker.id, - previousReducerResultPath: resultPath, + accepted: true, + previousReducerResultPath: claim + ? claim.previousResultPath + : latestResultPath, }, this.state.scanId, ); + if (result.sourceCoverage === undefined) { + const context = { + root: worker.artifactDir, + repoRoot: this.state.targetPath, + scanId: this.state.scanId, + layout: "reducer" as const, + deepReducer: { + scanRoot: this.artifacts.scanDir, + claimedWorkers: accepted.map((source) => ({ + id: source.id, + resultPath: source.resultPath, + artifactDir: source.artifactDir, + attempt: source.attempt, + })), + }, + }; + const sources = await readDeepReductionSources(context); + result.sourceCoverage = aggregateSourceCoverage( + sources.discoveries, + latestResult ?? null, + ); + } latestResult = result; - resultPath = worker.resultManifestPath; + latestResultPath = resultPath; } - return { resultPath, result: latestResult }; + return { result: latestResult, resultPath: latestResultPath }; } private reducerReady( @@ -1315,3 +1537,7 @@ function errorKind(error: unknown): string { if (!(error instanceof Error)) return typeof error; return errorNameWithCode(error); } + +function unique(values: string[]): string[] { + return [...new Set(values)]; +} diff --git a/plugins/codex-security/mcp-app/src/deep-scan/engine.ts b/plugins/codex-security/mcp-app/src/deep-scan/engine.ts index 9f619af1c1..a3a3a8bc0e 100644 --- a/plugins/codex-security/mcp-app/src/deep-scan/engine.ts +++ b/plugins/codex-security/mcp-app/src/deep-scan/engine.ts @@ -1,5 +1,12 @@ import { createScanArtifactContext } from "../artifact-context.js"; import { recordCodexSecurityScanDraftViaWorkbench } from "../artifact-scan-draft.js"; +import { createProviderProfile } from "../../../../../sdk/typescript/src/provider-profile.js"; +import { + loadDeepScanExecutionSettings, + restoredDeepScanWorkerSettings, + type DeepScanLegacySettingsContext, +} from "./recovery-settings.js"; +import type { CoordinatorOptions } from "./coordinator.js"; import { workbenchArgs } from "../workbench-arguments.js"; import { CodexSdkWorkerExecutor } from "./executor.js"; import type { DeepWorkerParentSandbox } from "./parent-sandbox.js"; @@ -26,6 +33,7 @@ export function startDeepScanEngine(options: { model?: string; reasoningEffort?: string; log?: DeepScanLogger; + onFinalized?: CoordinatorOptions["onFinalized"]; }) { const { run, runWorkbench, pluginRoot, threadId, handoffClaimToken } = options; @@ -34,6 +42,47 @@ export function startDeepScanEngine(options: { registry: options.registry, options: { store: options.store, + onFinalized: options.onFinalized, + prepareExecutor: async (owned) => + new CodexSdkWorkerExecutor({ + createProviderProfile, + parentTurn: options.parentTurn, + ...restoredDeepScanWorkerSettings( + await loadDeepScanExecutionSettings( + owned.scanDir, + owned, + async () => { + const context = await runWorkbench([ + "get-scan", + "--scan-id", + owned.scanId, + ]); + const recipe = context.recipe as + Pick | undefined; + const scan = context.scan as { + executionAttribution?: { + owner: DeepScanRunState["usageOwner"]; + }; + }; + return { + config: recipe?.config, + usageOwner: scan.executionAttribution?.owner, + }; + }, + ), + options.parentSandbox, + ), + artifactContext: { + pluginRoot, + scanRoot: owned.scanDir, + repoRoot: owned.targetPath, + scanId: owned.scanId, + scope: owned.scope, + includePaths: owned.includePaths, + filePaths: owned.filePaths, + pythonCommand: options.pythonCommand, + }, + }), executor: new CodexSdkWorkerExecutor({ model: options.model, reasoningEffort: options.reasoningEffort, @@ -41,6 +90,7 @@ export function startDeepScanEngine(options: { parentTurn: options.parentTurn, artifactContext: { pluginRoot, + scanRoot: run.scanDir, repoRoot: run.targetPath, scanId: run.scanId, scope: run.scope, @@ -60,6 +110,9 @@ export function startDeepScanEngine(options: { { requireRunning: true, requireClaim: true, + // V2 coordinators validate their durable selection before publishing. + requireCurrentTarget: + run.workflowVersion !== "deep-security-scan/v2", handoffClaimToken, pluginRoot, }, diff --git a/plugins/codex-security/mcp-app/src/deep-scan/executor.ts b/plugins/codex-security/mcp-app/src/deep-scan/executor.ts index 1d50c995c7..9bd2cf3cef 100644 --- a/plugins/codex-security/mcp-app/src/deep-scan/executor.ts +++ b/plugins/codex-security/mcp-app/src/deep-scan/executor.ts @@ -19,11 +19,9 @@ import { sep, win32, } from "node:path"; -import { - Codex, - type CyberAccessProgram, - type ThreadEvent, -} from "@openai/codex-sdk"; +import { createCodexClient } from "../../../../../sdk/typescript/src/codex-session.js"; +import type { CodexOptions } from "@openai/codex-sdk"; +import type { CyberAccessProgram, ThreadEvent } from "@openai/codex-sdk"; import { readCodexSessionTurn } from "../../../scripts/codex_session.mjs"; import { parse as parseToml } from "smol-toml"; import { @@ -53,11 +51,21 @@ import type { CodexWorkerResult, } from "./types.js"; +import type { + createProviderProfile, + ProviderProfile, +} from "../../../../../sdk/typescript/src/provider-profile.js"; + export interface CodexSdkWorkerModelSettings { + nativeProfileHome?: string; + createProviderProfile?: typeof createProviderProfile; + /** Resolved by the execution owner, including when reconstructing a scan. */ + codexOptions?: CodexOptions; model?: string; reasoningEffort?: string; artifactContext?: CodexSdkWorkerArtifactContext; parentSandbox?: DeepWorkerParentSandbox; + runtimeSettings?: CodexSdkWorkerRuntimeSettings; parentTurn?: DeepScanParentTurn; } @@ -67,20 +75,28 @@ export interface CodexSdkWorkerArtifactContext { repoRoot: string; scanId: string; scope?: string; + scanRoot?: string; includePaths?: readonly string[]; filePaths?: readonly string[]; pythonCommand?: string; } -interface CodexSdkWorkerRuntimeSettings { +export interface CodexSdkWorkerRuntimeSettings { environment?: Record; - config: Record; + config?: Record; preflightProviderOverrides?: string[]; nativeProfile?: string; + reasoningSummary?: string; + serviceTier?: string; + features?: { + api_key_cyber_access_programs?: boolean; + api_key_model_discovery?: boolean; + }; cyberAccessProgram?: CyberAccessProgram; } export class CodexSdkWorkerExecutor implements CodexWorkerExecutor { + private runtimeModelConfig?: Promise>; private runtimeSettings?: Promise; private parentCyberAccessProgram?: Promise; @@ -89,6 +105,7 @@ export class CodexSdkWorkerExecutor implements CodexWorkerExecutor { ) {} async run(request: CodexWorkerRequest): Promise { + let providerProfile: ProviderProfile | undefined; const includePaths = this.modelSettings.artifactContext?.includePaths; const scopeInput = JSON.stringify({ paths: includePaths, @@ -106,11 +123,16 @@ export class CodexSdkWorkerExecutor implements CodexWorkerExecutor { ); } const workerProfile = workerPermissionProfile(parentSandbox); + const resolved = this.modelSettings.codexOptions; const originalCwd = process.cwd(); - const childEnv = await snapshotWorkerEnvironment(); - // Snapshot the SDK's per-scan config once for this coordinator, including resumes. - const runtimeSettings = await (this.runtimeSettings ??= - workerRuntimeSettings(childEnv)); + const childEnv = await snapshotWorkerEnvironment(resolved?.env); + const runtimeSettings = await (this.runtimeSettings ??= (async () => ({ + ...(await workerRuntimeSettings( + childEnv, + this.modelSettings.nativeProfileHome, + )), + ...this.modelSettings.runtimeSettings, + }))()); for (const [name, value] of Object.entries( runtimeSettings.environment ?? {}, )) { @@ -125,38 +147,90 @@ export class CodexSdkWorkerExecutor implements CodexWorkerExecutor { // Keep host-bound inputs beside the prompt, outside the worker's output root. await writePrivateFile(includePathsFile, scopeInput); } - // Keep one native configuration for the policy check and the worker turn. - // Worker-owned tool and permission settings take precedence over inheritance. - const configOverrides = profileConfigOverrides({ - ...runtimeSettings.config, - responses_api_metadata: { - codex_security_surface: "plugin", - codex_security_command: "scan", - codex_security_scan_mode: "deep", - // Keep the host's integration/provider context as this coordinator - // launches or resumes workers with a different role. - ...(isRecord(runtimeSettings.config.responses_api_metadata) - ? runtimeSettings.config.responses_api_metadata - : {}), - codex_security_worker_role: request.kind, - }, + if (resolved?.apiKey !== undefined) + childEnv.CODEX_API_KEY = resolved.apiKey; + // Snapshot per-scan selections once; a reconstructed owner can supply them. + // Native account credentials continue to refresh in the selected home. + const modelConfig: NonNullable = { + ...(await (this.runtimeModelConfig ??= resolved?.config + ? Promise.resolve(resolved.config) + : workerModelConfig(childEnv))), + ...(this.modelSettings.model + ? { model: this.modelSettings.model } + : {}), + // The CLI can add effort levels before the pinned SDK widens ThreadOptions. ...(this.modelSettings.reasoningEffort ? { model_reasoning_effort: this.modelSettings.reasoningEffort } : {}), - mcp_servers: { - // A disabled server still needs a valid transport during native resolution. - "codex-security": { command: "node", enabled: false }, - ...this.compactArtifactServer(request, includePathsFile), - }, - ...workerSubagentConfig( - request.subagents, - runtimeSettings.config.features, - ), - approval_policy: "never", - default_permissions: DEEP_SCAN_WORKER_PERMISSION_PROFILE_ID, - [`permissions.${DEEP_SCAN_WORKER_PERMISSION_PROFILE_ID}`]: - workerProfile, - }); + }; + const workerConfig: Record = { + ...runtimeSettings.config, + ...modelConfig, + ...(modelConfig.model_reasoning_summary !== undefined || + runtimeSettings.reasoningSummary === undefined + ? {} + : { model_reasoning_summary: runtimeSettings.reasoningSummary }), + ...(modelConfig.service_tier !== undefined || + runtimeSettings.serviceTier === undefined + ? {} + : { service_tier: runtimeSettings.serviceTier }), + }; + // Use the same native configuration for preflight and the worker turn. + const configOverrides = [ + ...(resolved?.configOverrides ?? []), + ...profileConfigOverrides({ + ...workerConfig, + responses_api_metadata: { + codex_security_surface: "plugin", + codex_security_command: "scan", + codex_security_scan_mode: "deep", + ...(isRecord(workerConfig.responses_api_metadata) + ? workerConfig.responses_api_metadata + : {}), + codex_security_worker_role: request.kind, + }, + mcp_servers: { + ...(isRecord(workerConfig.mcp_servers) + ? workerConfig.mcp_servers + : {}), + // A disabled server still needs a valid transport during native resolution. + "codex-security": { command: "node", enabled: false }, + ...this.compactArtifactServer(request, includePathsFile), + }, + ...workerSubagentConfig( + request.subagents, + workerConfig, + runtimeSettings.features, + ), + approval_policy: "never", + default_permissions: DEEP_SCAN_WORKER_PERMISSION_PROFILE_ID, + [`permissions.${DEEP_SCAN_WORKER_PERMISSION_PROFILE_ID}`]: + workerProfile, + }), + ]; + const profileHome = this.modelSettings.nativeProfileHome; + const boundHome = + environmentVariable(childEnv, "CODEX_HOME", process.platform) || + join(homedir(), ".codex"); + if ( + runtimeSettings.nativeProfile !== undefined && + profileHome !== undefined && + this.modelSettings.createProviderProfile && + (await fs.realpath(profileHome)) !== (await fs.realpath(boundHome)) + ) { + // The account home is bound to the original scan; the current private + // provider profile belongs to this execution owner's environment. + const profileConfig = parseToml( + await fs.readFile( + join(profileHome, `${runtimeSettings.nativeProfile}.config.toml`), + "utf8", + ), + ); + providerProfile = await this.modelSettings.createProviderProfile( + boundHome, + profileConfig as Parameters[1], + ); + } const openAiApiKey = environmentVariable( childEnv, "OPENAI_API_KEY", @@ -168,7 +242,9 @@ export class CodexSdkWorkerExecutor implements CodexWorkerExecutor { process.platform, )?.trim(); const codexPath = resolveCodexPath( - childEnv, + resolved?.codexPathOverride === undefined + ? childEnv + : { ...childEnv, CODEX_CLI_PATH: resolved.codexPathOverride }, process.platform, process.arch, originalCwd, @@ -191,7 +267,10 @@ export class CodexSdkWorkerExecutor implements CodexWorkerExecutor { throw error; })) : undefined; - const workerFeatures = runtimeSettings.config.features; + const workerFeatures = { + ...(isRecord(workerConfig.features) ? workerConfig.features : {}), + ...runtimeSettings.features, + }; if ( parentProgram !== undefined && (!isRecord(workerFeatures) || @@ -204,11 +283,23 @@ export class CodexSdkWorkerExecutor implements CodexWorkerExecutor { const { useOpenAiApiKey } = await preflightDeepScanWorkerPermissionProfile({ ...preflightOptions, + configOverrides: [ + ...configOverrides, + ...(resolved?.baseUrl + ? profileConfigOverrides({ openai_base_url: resolved.baseUrl }) + : []), + ], expectedProfile: workerProfile, allowOpenAiApiKeyFallback: Boolean(openAiApiKey && !codexApiKey), }); const prompt = await fs.readFile(request.promptPath, "utf8"); + const { + config: _config, + configOverrides: _overrides, + ...resolvedOptions + } = resolved ?? {}; const codexOptions = { + ...resolvedOptions, codexPathOverride: executablePathForSpawn(codexPath), env: childEnv, // Codex exec reads CODEX_API_KEY; the SDK maps apiKey to that variable. @@ -219,10 +310,11 @@ export class CodexSdkWorkerExecutor implements CodexWorkerExecutor { }; const codex = runtimeSettings.nativeProfile === undefined - ? new Codex(codexOptions) + ? createCodexClient(codexOptions) : createCodexProfileClient({ ...codexOptions, - profileName: runtimeSettings.nativeProfile, + profileName: + providerProfile?.name ?? runtimeSettings.nativeProfile, }); const threadOptions = { ...(this.modelSettings.model @@ -233,7 +325,7 @@ export class CodexSdkWorkerExecutor implements CodexWorkerExecutor { workingDirectory: request.workingDirectory, } as const; const thread = request.resumeThreadId - ? codex.resumeThread(request.resumeThreadId, threadOptions) + ? codex.resumeThread!(request.resumeThreadId, threadOptions) : codex.startThread(threadOptions); const workerPrompt = request.resumeThreadId ? (request.continuationPrompt ?? prompt) @@ -262,28 +354,45 @@ export class CodexSdkWorkerExecutor implements CodexWorkerExecutor { events, stopOnCompletion: true, onEvent: async (event) => { - const item = event.type === "item.completed" ? event.item : event; - if (item.type === "error") { - const fallbackError = deepScanPermissionProfileFallbackError( - item.message, - ); + if ( + event.type === "thread.started" && + typeof event.thread_id === "string" + ) { + await request.onThreadStarted?.(event.thread_id); + } else if ( + event.type === "item.completed" && + isRecord(event.item) + ) { + const fallbackError = + event.item.type === "error" && + typeof event.item.message === "string" + ? deepScanPermissionProfileFallbackError(event.item.message) + : undefined; if (fallbackError) { controller.abort(fallbackError); throw fallbackError; } - } - if (event.type === "thread.started") { - await request.onThreadStarted?.(event.thread_id); - } else if (event.type === "item.completed") { appendItemDiagnostic(diagnostics, event.item); } else if (event.type === "turn.completed") { request.signal.removeEventListener("abort", forwardAbort); } else if (event.type === "turn.failed") { - const error = codexWorkerTurnError(event.error.message); + const error = codexWorkerTurnError( + (event.error as { message: string }).message, + ); if (error instanceof DeepScanNonRetryableError) request.onNonRetryableError?.(error); throw error; - } else if (event.type === "error") { + } else if ( + event.type === "error" && + typeof event.message === "string" + ) { + const fallbackError = deepScanPermissionProfileFallbackError( + event.message, + ); + if (fallbackError) { + controller.abort(fallbackError); + throw fallbackError; + } // Codex exec emits retry-in-progress notifications as error events. appendStreamDiagnostic(diagnostics, event.message); } @@ -307,8 +416,12 @@ export class CodexSdkWorkerExecutor implements CodexWorkerExecutor { } catch (error) { throw classifyCodexWorkerError(error); } finally { - if (includePathsFile !== undefined) - await fs.rm(includePathsFile, { force: true }); + try { + await providerProfile?.cleanup(); + } finally { + if (includePathsFile !== undefined) + await fs.rm(includePathsFile, { force: true }); + } } } @@ -380,14 +493,26 @@ export class CodexSdkWorkerExecutor implements CodexWorkerExecutor { } } -function workerSubagentConfig(subagents: number, inheritedFeatures: unknown) { +function workerSubagentConfig( + subagents: number, + config: Record, + inheritedFeatures: CodexSdkWorkerRuntimeSettings["features"], +) { return { // V1 counts children; V2 counts the root plus its children. Keeping its // feature disabled lets the model choose either runtime without rejecting // inherited agents.max_threads configuration. - ...(subagents > 0 ? { agents: { max_threads: subagents } } : {}), + ...(subagents > 0 + ? { + agents: { + ...(isRecord(config.agents) ? config.agents : {}), + max_threads: subagents, + }, + } + : {}), features: { - ...(isRecord(inheritedFeatures) ? inheritedFeatures : {}), + ...(isRecord(config.features) ? config.features : {}), + ...inheritedFeatures, multi_agent_v2: { enabled: false, max_concurrent_threads_per_session: subagents + 1, @@ -555,6 +680,84 @@ function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null; } +function workerModelSelection( + config: NonNullable, +): NonNullable { + return Object.fromEntries( + [ + "model", + "model_provider", + "model_reasoning_effort", + "model_reasoning_summary", + "service_tier", + "model_providers", + ] + .filter((key) => config[key] !== undefined) + .map((key) => [key, config[key]]), + ) as NonNullable; +} + +async function workerModelConfig( + environment: Record, +): Promise> { + const configPath = environmentVariable( + environment, + "CODEX_SECURITY_CONFIG_PATH", + process.platform, + ); + if (!configPath) return {}; + const config = parseToml(await fs.readFile(configPath, "utf8")); + const profiles = config.profiles; + const profile = + typeof config.profile === "string" && isRecord(profiles) + ? profiles[config.profile] + : undefined; + return workerModelSelection({ + ...config, + ...(isRecord(profile) ? profile : {}), + } as NonNullable); +} + +export function workerRuntimeSettingsFromConfig( + config: Record, +): CodexSdkWorkerRuntimeSettings { + const profiles = config.profiles; + const profile = + typeof config.profile === "string" && isRecord(profiles) + ? profiles[config.profile] + : undefined; + const summary = + isRecord(profile) && profile.model_reasoning_summary !== undefined + ? profile.model_reasoning_summary + : config.model_reasoning_summary; + const serviceTier = + isRecord(profile) && profile.service_tier !== undefined + ? profile.service_tier + : config.service_tier; + const settings: CodexSdkWorkerRuntimeSettings = { + ...(typeof summary === "string" ? { reasoningSummary: summary } : {}), + ...(typeof serviceTier === "string" ? { serviceTier } : {}), + }; + const security = config.codex_security; + if (isRecord(security) && typeof security.cyber_access_program === "string") { + settings.cyberAccessProgram = + security.cyber_access_program as CyberAccessProgram; + } + const features = config.features; + if (isRecord(features)) { + for (const name of [ + "api_key_cyber_access_programs", + "api_key_model_discovery", + ] as const) { + const value = features[name]; + if (typeof value === "boolean") { + (settings.features ??= {})[name] = value; + } + } + } + return settings; +} + function mergeConfigTables( base: Record, override: Record, @@ -578,6 +781,7 @@ function mergeConfigTables( async function workerRuntimeSettings( environment: Record, + nativeProfileHome?: string, ): Promise { const configPath = environmentVariable( environment, @@ -605,7 +809,7 @@ async function workerRuntimeSettings( selected[key] = mergeConfigTables(config[key], profile[key]); } } - const settings: CodexSdkWorkerRuntimeSettings = { config: selected }; + const settings = workerRuntimeSettingsFromConfig(config); const workerConfigPath = environmentVariable( environment, "CODEX_SECURITY_DEEP_SCAN_CONFIG_PATH", @@ -632,6 +836,7 @@ async function workerRuntimeSettings( } settings.nativeProfile = nativeProfile; const codexHome = + nativeProfileHome || environmentVariable(environment, "CODEX_HOME", process.platform) || join(homedir(), ".codex"); const nativeProfileConfig = parseToml( @@ -673,6 +878,9 @@ async function workerRuntimeSettings( ...(isRecord(workerConfig.features) ? workerConfig.features : {}), }, }; + settings.features = workerRuntimeSettingsFromConfig({ + features: settings.config.features, + }).features; // Profiles are resolved above; providers stay in their private native file. // Workers register their own tools and verified read-only permission profile. for (const key of [ @@ -694,9 +902,11 @@ async function workerRuntimeSettings( return settings; } -async function snapshotWorkerEnvironment(): Promise> { +async function snapshotWorkerEnvironment( + source: NodeJS.ProcessEnv = process.env, +): Promise> { const environment = Object.fromEntries( - Object.entries(process.env).filter( + Object.entries(source).filter( (entry): entry is [string, string] => entry[1] !== undefined, ), ) as Record; @@ -709,7 +919,7 @@ async function snapshotWorkerEnvironment(): Promise> { "CODEX_MANAGED_PACKAGE_ROOT", "LOCALAPPDATA", ]) { - const value = process.env[name]; + const value = environmentVariable(source, name, process.platform); for (const key of Object.keys(environment)) { if (key.toUpperCase() === name) delete environment[key]; } diff --git a/plugins/codex-security/mcp-app/src/deep-scan/finalization.ts b/plugins/codex-security/mcp-app/src/deep-scan/finalization.ts new file mode 100644 index 0000000000..5e2d61a216 --- /dev/null +++ b/plugins/codex-security/mcp-app/src/deep-scan/finalization.ts @@ -0,0 +1,190 @@ +import { + deepReductionToScanDraft, + parseDeepReduction, +} from "./artifact-validation.js"; +import { + createScanArtifactContext, + type RunArtifactWorkbench, +} from "../artifact-context.js"; +import { + recordCodexSecurityScanDraftViaWorkbench, + type DeepScanPublication, +} from "../artifact-scan-draft.js"; +import { WorkbenchDeepScanStore } from "./store.js"; +import { + createDeepScanArtifacts, + type DeepScanArtifacts, +} from "./artifacts.js"; +import { createHash } from "node:crypto"; +import { join } from "node:path"; +import { type ScanDraftInput } from "../artifact-scan-draft.js"; +import { readScanFile } from "../../../../../sdk/typescript/src/scan-files.js"; +import type { + DeepScanFinalizationInput, + DeepScanRunState, + DeepScanStore, +} from "./types.js"; + +export type { DeepScanFinalizationInput } from "./types.js"; + +/** Publish exactly the saved aggregate; the enclosing scan owns public completion. */ +export async function publishSelectedDeepScan(input: { + run: DeepScanRunState; + artifacts: DeepScanArtifacts; + signal: AbortSignal; + publish: ( + draft: ScanDraftInput, + signal: AbortSignal, + publication: DeepScanPublication, + ) => Promise; + finish: DeepScanStore["finish"]; +}): Promise { + const { run, artifacts, signal } = input; + const selection = selectedInput(run); + if (run.status === "succeeded") return run; + if (run.status !== "running") + throw new Error("Stopped Deep Scan finalization cannot become successful."); + signal.throwIfAborted(); + const draft = await readSelectedDeepScanDraft( + artifacts, + run.scanId, + selection, + signal, + ); + await input.publish(draft, signal, { + coordinatorGeneration: run.coordinatorGeneration, + resultPath: + selection.resultPath === null + ? null + : join(run.scanDir, selection.resultPath), + }); + signal.throwIfAborted(); + return input.finish({ + scanId: run.scanId, + reason: selection.terminalReason, + manifestPath: join(run.scanDir, "scan-manifest.json"), + omittedWorkerIds: selection.omittedWorkerIds, + }); +} + +/** Recreate the chosen draft without scheduling discovery or reducer work. */ +export async function readSelectedDeepScanDraft( + artifacts: DeepScanArtifacts, + scanId: string, + selection: DeepScanFinalizationInput, + signal?: AbortSignal, +): Promise { + if (selection.version !== 1) + throw new Error("Unsupported Deep Scan finalization input version."); + if (selection.resultPath === null) { + if ( + selection.terminalReason !== "capped" || + selection.resultSha256 !== null + ) { + throw new Error( + "An empty Deep Scan finalization requires the recorded discovery deadline.", + ); + } + return { + scanId, + findings: [], + coverage: { + surfaces: [], + explicitExclusions: [], + deferred: [ + { + reason: + "The configured discovery time limit elapsed before any source review completed.", + }, + ], + }, + }; + } + const contents = await readScanFile( + artifacts.scanDir, + selection.resultPath, + "Selected Deep Scan result", + signal, + ); + if ( + createHash("sha256").update(contents).digest("hex") !== + selection.resultSha256 + ) { + throw new Error( + "The selected Deep Scan finalization input changed after acceptance.", + ); + } + const stored = JSON.parse(contents.toString("utf8")); + const draft = deepReductionToScanDraft(parseDeepReduction(stored, true)); + if (draft.scanId !== scanId || draft.complete === false) { + throw new Error( + "Deep Scan finalization requires the selected complete result for this scan.", + ); + } + return draft; +} + +/** SDK recovery uses the installed plugin's publisher and the original parent scan. */ +export async function resumeSelectedDeepScan(input: { + scanId: string; + threadId: string; + pluginRoot: string; + runWorkbench: RunArtifactWorkbench; + signal: AbortSignal; + handoffClaimToken?: string; +}): Promise { + const store = new WorkbenchDeepScanStore(input.runWorkbench); + const run = await store.get(input.scanId, input.threadId); + selectedInput(run); + try { + await publishSelectedDeepScan({ + run, + artifacts: createDeepScanArtifacts(run.scanDir), + signal: input.signal, + publish: async (draft, signal, publication) => { + const context = await createScanArtifactContext( + input.scanId, + input.runWorkbench, + { + requireRunning: true, + requireClaim: true, + requireCurrentTarget: false, + handoffClaimToken: input.handoffClaimToken, + pluginRoot: input.pluginRoot, + }, + ); + await recordCodexSecurityScanDraftViaWorkbench( + context, + draft, + input.runWorkbench, + signal, + publication, + ); + }, + finish: (selection) => + store.finish({ + ...selection, + coordinatorGeneration: run.coordinatorGeneration, + }), + }); + } catch (error) { + // The original coordinator may publish while the SDK recovers its parent turn. + const committed = await store + .get(input.scanId, input.threadId) + .catch(() => null); + if (committed?.status !== "succeeded") throw error; + } +} + +function selectedInput(run: DeepScanRunState): DeepScanFinalizationInput { + const selection = run.finalizationInput; + if (!selection) + throw new Error("Deep Scan has no selected finalization input."); + if ( + run.workflowVersion !== "deep-security-scan/v2" || + selection.version !== 1 + ) { + throw new Error("Unsupported Deep Scan finalization input version."); + } + return selection; +} diff --git a/plugins/codex-security/mcp-app/src/deep-scan/recovery-settings.ts b/plugins/codex-security/mcp-app/src/deep-scan/recovery-settings.ts new file mode 100644 index 0000000000..02d0bc935a --- /dev/null +++ b/plugins/codex-security/mcp-app/src/deep-scan/recovery-settings.ts @@ -0,0 +1,522 @@ +import { promises as fs } from "node:fs"; +import { homedir } from "node:os"; +import { isAbsolute, join, sep, win32 } from "node:path"; +import type { CodexOptions } from "@openai/codex-sdk"; +import { parse as parseToml } from "smol-toml"; +import { scanPreflightCodexConfig } from "../../../../../sdk/typescript/src/preflight-config.js"; +import { + resolveCodexProfile, + type JsonObject, +} from "../../../../../sdk/typescript/src/config.js"; +import { readScanLogs } from "../../../../../sdk/typescript/src/scan-logs.js"; +import { + resolveCodexPath, + workerRuntimeSettingsFromConfig, + type CodexSdkWorkerRuntimeSettings, +} from "./executor.js"; +import type { DeepWorkerParentSandbox } from "./parent-sandbox.js"; +import type { DeepScanRunState } from "./types.js"; + +/** Credentials and arbitrary environment/configuration stay with Codex. */ +export interface DeepScanExecutionSettings { + codexPath: string; + codexHome: string; + model?: string; + modelProvider?: string; + reasoningEffort?: string; + reasoningSummary?: string; + serviceTier?: string; + /** The native snapshot recorded no request tier; serviceTier preserves its wire behavior. */ + nativeServiceTierAbsent?: true; + providerConfig?: JsonObject; + parentSandbox?: DeepWorkerParentSandbox; + runtimeSettings?: CodexSdkWorkerRuntimeSettings; +} + +export interface DeepScanLegacySettingsContext { + config?: JsonObject; + usageOwner?: DeepScanRunState["usageOwner"]; +} + +export interface DeepScanExecutionSettingsSnapshot { + version: number; + settings: DeepScanExecutionSettings; +} + +/** Existing runs can be joined without capturing a usable current runtime. */ +export async function beginDeepScanWithCapturedSettings( + capture: () => Promise, + begin: (settings: DeepScanExecutionSettings | null) => Promise, +): Promise { + let captureError: unknown; + const settings = await capture().catch((error: unknown) => { + captureError = error; + return null; + }); + try { + return await begin(settings); + } catch (error) { + if (settings !== null) throw error; + const detail = (failure: unknown) => + failure instanceof Error ? failure.message : String(failure); + throw new AggregateError( + [error, captureError], + `${detail(error)} Original execution settings could not be captured: ${detail(captureError)}`, + ); + } +} + +export async function captureDeepScanExecutionSettings( + original: Pick, + parentSandbox: DeepWorkerParentSandbox, + environment: NodeJS.ProcessEnv = process.env, + parent?: { threadId: string; startedAt?: string }, +): Promise { + const configuredHome = environment.CODEX_HOME || join(homedir(), ".codex"); + const codexHome = + !isAbsolute(configuredHome) || + (process.platform === "win32" && + ["\\", "/"].includes(win32.parse(configuredHome).root)) + ? await fs.realpath(configuredHome) + : configuredHome; + // Preserve symlink/.. traversal in an absolute home instead of joining lexically. + const configPath = + environment.CODEX_SECURITY_CONFIG_PATH ?? `${codexHome}${sep}config.toml`; + let config: JsonObject; + try { + config = parseToml(await fs.readFile(configPath, "utf8")) as JsonObject; + } catch (error) { + if ( + environment.CODEX_SECURITY_CONFIG_PATH || + (error as NodeJS.ErrnoException).code !== "ENOENT" + ) + throw error; + config = {}; + } + // Reuse the SDK projection: custom provider credentials belong in the native home. + const selected = scanPreflightCodexConfig(resolveCodexProfile(config)); + // A recovered scan can have a different continuation. Only its recorded owner + // establishes original history; null means that historical binding is missing. + const owner = + original.usageOwner === undefined ? parent : original.usageOwner; + const native = !owner?.threadId + ? {} + : await originalParentSettings(codexHome, { + ...owner, + threadId: owner.threadId, + startedAt: parent?.startedAt ?? owner.startedAt, + }); + return executionSettings({ + codexPath: resolveCodexPath( + environment, + process.platform, + process.arch, + process.cwd(), + ), + codexHome, + model: + original.model ?? (selected.model as string | undefined) ?? native.model, + reasoningEffort: + original.reasoningEffort ?? + (selected.model_reasoning_effort as string | undefined) ?? + native.reasoningEffort, + modelProvider: + (selected.model_provider as string | undefined) ?? native.modelProvider, + reasoningSummary: + (selected.model_reasoning_summary as string | undefined) ?? + native.reasoningSummary, + serviceTier: + (selected.service_tier as string | undefined) ?? native.serviceTier, + ...(selected.service_tier === undefined && native.nativeServiceTierAbsent + ? { nativeServiceTierAbsent: true as const } + : {}), + providerConfig: selected.model_providers as JsonObject | undefined, + runtimeSettings: workerRuntimeSettingsFromConfig(config), + parentSandbox, + }); +} + +async function originalParentSettings( + codexHome: string, + parent: { threadId: string; turnId?: string | null; startedAt?: string }, +): Promise> { + // Native config/read represents omitted selections as null. Recover recorded + // selections from the original parent; some native records omit the summary. + // History can be disabled or unavailable; configured selections still work. + try { + const log = await readScanLogs({ + scanId: parent.threadId, + threadId: parent.threadId, + executionThreadIds: [], + codexHome: await fs.realpath(codexHome), + allowMissingRoot: true, + }); + const settings: Partial = {}; + let applied: Partial | undefined; + let summaryIsCompatibilityOnly = false; + const cutoff = + parent.startedAt === undefined ? Infinity : Date.parse(parent.startedAt); + for (const entry of log.events) { + const event = entry.event as Record; + const timestamp = + typeof event.timestamp === "string" + ? Date.parse(event.timestamp) + : undefined; + if (timestamp !== undefined && timestamp > cutoff) continue; + const payload = event.payload; + if (!payload || typeof payload !== "object" || Array.isArray(payload)) + continue; + const context = payload as Record; + if ( + event.type === "event_msg" && + context.type === "thread_settings_applied" + ) { + if ( + typeof context.thread_id === "string" && + context.thread_id !== parent.threadId + ) + continue; + const snapshot = context.thread_settings; + if ( + !snapshot || + typeof snapshot !== "object" || + Array.isArray(snapshot) + ) + continue; + const value = snapshot as Record; + applied = { + model: typeof value.model === "string" ? value.model : undefined, + modelProvider: + typeof value.model_provider_id === "string" + ? value.model_provider_id + : undefined, + reasoningEffort: + typeof value.reasoning_effort === "string" + ? value.reasoning_effort + : undefined, + reasoningSummary: + typeof value.reasoning_summary === "string" + ? value.reasoning_summary + : undefined, + // A persisted native absent tier and explicit standard both omit the + // request tier. This does not infer a tier from missing history. + serviceTier: + typeof value.service_tier === "string" + ? value.service_tier + : value.service_tier === undefined + ? "default" + : undefined, + ...(value.service_tier === undefined + ? { nativeServiceTierAbsent: true as const } + : {}), + }; + } + if (event.type === "session_meta") { + if (typeof context.model_provider === "string") + settings.modelProvider = context.model_provider; + // Codex 0.133 replaced turn_context.summary with a compatibility default. + // Fresh threads need not have a thread_settings_applied record to replace it. + const version = + typeof context.cli_version === "string" + ? /^(\d+)\.(\d+)\./u.exec(context.cli_version) + : null; + summaryIsCompatibilityOnly = + version !== null && + (Number(version[1]) > 0 || Number(version[2]) >= 133); + } + if (event.type === "turn_context") { + if (parent.turnId && context.turn_id !== parent.turnId) continue; + if (typeof context.model === "string") settings.model = context.model; + if (typeof context.effort === "string") + settings.reasoningEffort = context.effort; + if (!summaryIsCompatibilityOnly && typeof context.summary === "string") + settings.reasoningSummary = context.summary; + } + } + // Applied snapshots contain native selected values. Newer turn-context + // summaries are only a compatibility field, not the active selection. + return { ...settings, ...applied }; + } catch { + return {}; + } +} + +/** Read recorded execution settings or original legacy facts. */ +export async function loadDeepScanExecutionSettings( + _scanDir: string, + original?: Pick< + DeepScanRunState, + | "model" + | "reasoningEffort" + | "usageOwner" + | "createdAt" + | "workflowVersion" + | "executionSettings" + >, + readLegacyContext?: () => Promise, + environment: NodeJS.ProcessEnv = process.env, +): Promise> { + // Only the workbench creation transaction records the launch selection. Scan + // artifacts are model-writable and cannot select a preflight executable/home. + const saved = original?.executionSettings; + let settings: DeepScanExecutionSettings; + if (saved) { + if (saved.version !== 1) { + throw new Error( + "This Deep Scan uses an unsupported execution settings version.", + ); + } + settings = executionSettings(saved.settings); + } else { + if ( + original?.workflowVersion === "deep-security-scan/v1" || + original?.workflowVersion === "deep-scan-mcp/v1" + ) { + // Legacy runs predate the binding. Their saved recipe and recorded owner + // can recover selections, but cannot establish an original executable or + // home. Leave those unknown and retain the existing native launch behavior. + const context = await readLegacyContext?.(); + const selected = scanPreflightCodexConfig( + resolveCodexProfile(context?.config ?? {}), + ); + const owner = original.usageOwner ?? context?.usageOwner; + const home = environment.CODEX_HOME || join(homedir(), ".codex"); + const native = !owner?.threadId + ? {} + : await originalParentSettings(home, { + ...owner, + threadId: owner.threadId, + startedAt: original.createdAt ?? owner.startedAt, + }); + return { + model: + original.model ?? + (selected.model as string | undefined) ?? + native.model, + reasoningEffort: + original.reasoningEffort ?? + (selected.model_reasoning_effort as string | undefined) ?? + native.reasoningEffort, + modelProvider: + (selected.model_provider as string | undefined) ?? + native.modelProvider, + reasoningSummary: + (selected.model_reasoning_summary as string | undefined) ?? + native.reasoningSummary, + serviceTier: + (selected.service_tier as string | undefined) ?? native.serviceTier, + ...(selected.service_tier === undefined && + native.nativeServiceTierAbsent + ? { nativeServiceTierAbsent: true as const } + : {}), + providerConfig: + selected.model_provider === "amazon-bedrock" + ? (selected.model_providers as JsonObject | undefined) + : undefined, + }; + } + throw new Error( + "This Deep Scan has no recorded original execution settings; its executable and Codex home cannot be recovered.", + ); + } + if ( + !original || + (settings.model !== undefined && + settings.reasoningEffort !== undefined && + settings.modelProvider !== undefined && + settings.reasoningSummary !== undefined && + settings.serviceTier !== undefined) + ) + return settings; + // Earlier snapshots can omit native selections. Recover only from the saved + // home and recorded owner; the continuation's current config is not history. + const owner = original.usageOwner; + const native = !owner?.threadId + ? {} + : await originalParentSettings(settings.codexHome, { + ...owner, + threadId: owner.threadId, + startedAt: original.createdAt, + }); + // History reads can outlive this coordinator. Project missing selections for + // its workers without overwriting a snapshot owned by a newer coordinator. + return executionSettings({ + ...settings, + model: settings.model ?? original.model ?? native.model, + reasoningEffort: + settings.reasoningEffort ?? + original.reasoningEffort ?? + native.reasoningEffort, + modelProvider: settings.modelProvider ?? native.modelProvider, + reasoningSummary: settings.reasoningSummary ?? native.reasoningSummary, + serviceTier: settings.serviceTier ?? native.serviceTier, + ...(settings.serviceTier === undefined && native.nativeServiceTierAbsent + ? { nativeServiceTierAbsent: true as const } + : {}), + }); +} + +export function restoredDeepScanWorkerSettings( + settings: Partial, + currentParentSandbox: DeepWorkerParentSandbox, + environment: () => NodeJS.ProcessEnv = () => process.env, +): { + nativeProfileHome: string; + codexOptions: CodexOptions; + model?: string; + reasoningEffort?: string; + parentSandbox: DeepWorkerParentSandbox; + runtimeSettings?: CodexSdkWorkerRuntimeSettings; +} { + const originalSandbox = settings.parentSandbox; + const depths = [ + originalSandbox?.globScanMaxDepth, + currentParentSandbox.globScanMaxDepth, + ].filter((depth): depth is number => depth !== undefined); + // Native depth caps limit deny-glob expansion, not allowed traversal. Keep + // the larger finite cap, or no cap when either known policy has uncapped globs. + const uncapped = [originalSandbox, currentParentSandbox].some( + (sandbox) => + sandbox?.globScanMaxDepth === undefined && + sandbox?.filesystemDenies.some((path) => + ["*", "?", "[", "]"].some((character) => path.includes(character)), + ), + ); + const config = scanPreflightCodexConfig({ + ...(settings.model === undefined ? {} : { model: settings.model }), + ...(settings.reasoningEffort === undefined + ? {} + : { model_reasoning_effort: settings.reasoningEffort }), + ...(settings.modelProvider === undefined + ? {} + : { model_provider: settings.modelProvider }), + ...(settings.reasoningSummary === undefined + ? {} + : { model_reasoning_summary: settings.reasoningSummary }), + ...(settings.serviceTier === undefined + ? {} + : { service_tier: settings.serviceTier }), + ...(settings.providerConfig === undefined + ? {} + : { model_providers: settings.providerConfig }), + }); + // Other provider definitions remain in the bound native home. Catalog + // defaults would override its customized endpoint or authentication key. + if (settings.modelProvider !== "amazon-bedrock") + delete config.model_providers; + return { + get nativeProfileHome() { + return environment().CODEX_HOME || join(homedir(), ".codex"); + }, + model: settings.model, + reasoningEffort: settings.reasoningEffort, + ...(settings.runtimeSettings === undefined + ? {} + : { runtimeSettings: settings.runtimeSettings }), + parentSandbox: { + filesystemDenies: [ + ...new Set([ + ...(originalSandbox?.filesystemDenies ?? []), + ...currentParentSandbox.filesystemDenies, + ]), + ], + ...((originalSandbox?.literalFilesystemDenies?.length ?? 0) || + currentParentSandbox.literalFilesystemDenies?.length + ? { + literalFilesystemDenies: [ + ...new Set([ + ...(originalSandbox?.literalFilesystemDenies ?? []), + ...(currentParentSandbox.literalFilesystemDenies ?? []), + ]), + ], + } + : {}), + ...(uncapped || depths.length === 0 + ? {} + : { globScanMaxDepth: Math.max(...depths) }), + }, + codexOptions: { + codexPathOverride: settings.codexPath, + // The executor reads this property for each launch. API keys can refresh; + // only the original account home and non-secret selections are bound. + get env() { + return Object.fromEntries( + Object.entries({ + ...environment(), + ...(settings.codexPath === undefined + ? {} + : { CODEX_CLI_PATH: settings.codexPath }), + ...(settings.codexHome === undefined + ? {} + : { CODEX_HOME: settings.codexHome }), + }).filter( + (entry): entry is [string, string] => entry[1] !== undefined, + ), + ); + }, + config: config as NonNullable, + }, + }; +} + +function executionSettings( + value: DeepScanExecutionSettings, +): DeepScanExecutionSettings { + // Only Bedrock's per-scan AWS selectors need persistence. Other provider + // definitions remain in the original native home. + const provider = + value.modelProvider === "amazon-bedrock" + ? (scanPreflightCodexConfig({ + ...(value.modelProvider === undefined + ? {} + : { model_provider: value.modelProvider }), + ...(value.providerConfig === undefined + ? {} + : { model_providers: value.providerConfig }), + }).model_providers as JsonObject | undefined) + : undefined; + const settings: DeepScanExecutionSettings = { + codexPath: value.codexPath, + codexHome: value.codexHome, + model: value.model, + modelProvider: value.modelProvider, + reasoningEffort: value.reasoningEffort, + reasoningSummary: value.reasoningSummary, + serviceTier: value.serviceTier, + ...(value.nativeServiceTierAbsent === true + ? { nativeServiceTierAbsent: true } + : {}), + ...(provider === undefined ? {} : { providerConfig: provider }), + ...(value.runtimeSettings === undefined + ? {} + : { runtimeSettings: structuredClone(value.runtimeSettings) }), + ...(value.parentSandbox === undefined + ? {} + : { + parentSandbox: { + filesystemDenies: [...value.parentSandbox.filesystemDenies], + ...(value.parentSandbox.literalFilesystemDenies === undefined + ? {} + : { + literalFilesystemDenies: [ + ...value.parentSandbox.literalFilesystemDenies, + ], + }), + ...(value.parentSandbox.globScanMaxDepth === undefined + ? {} + : { + globScanMaxDepth: value.parentSandbox.globScanMaxDepth, + }), + }, + }), + }; + if ( + typeof settings.codexPath !== "string" || + typeof settings.codexHome !== "string" + ) { + throw new Error( + "Deep Scan execution settings are missing the recorded executable or Codex home.", + ); + } + return settings; +} diff --git a/plugins/codex-security/mcp-app/src/deep-scan/registry.ts b/plugins/codex-security/mcp-app/src/deep-scan/registry.ts index 38eb9dc28d..165cba4657 100644 --- a/plugins/codex-security/mcp-app/src/deep-scan/registry.ts +++ b/plugins/codex-security/mcp-app/src/deep-scan/registry.ts @@ -2,7 +2,11 @@ import { setTimeout as delay } from "node:timers/promises"; import { DeepScanCoordinator } from "./coordinator.js"; import type { CoordinatorOptions, ParentScanState } from "./coordinator.js"; import { isTransientPersistenceError } from "./store.js"; -import type { DeepScanCoordinatorClaim, DeepScanRunState } from "./types.js"; +import type { + CodexWorkerExecutor, + DeepScanCoordinatorClaim, + DeepScanRunState, +} from "./types.js"; const COORDINATOR_LEASE_MS = 30_000; const COORDINATOR_POLL_MS = 1_000; @@ -12,6 +16,11 @@ export { DeepScanNonRetryableError, } from "./coordinator.js"; +export interface DeepScanCoordinatorStartOptions extends CoordinatorOptions { + /** Resolve persisted execution settings only after acquiring this run. */ + prepareExecutor?: (run: DeepScanRunState) => Promise; +} + /** Owns the live coordinators in this MCP server process. */ export class DeepScanCoordinatorRegistry { private readonly coordinators = new Map(); @@ -22,7 +31,8 @@ export class DeepScanCoordinatorRegistry { return this.coordinators.get(scanId); } - start(options: CoordinatorOptions): DeepScanCoordinator { + start(options: DeepScanCoordinatorStartOptions): DeepScanCoordinator { + requireSupportedDeepScan(options.run); const existing = this.coordinators.get(options.run.scanId); if (existing) return existing; if (this.closed) throw new Error("Deep Scan coordinator host has closed."); @@ -105,7 +115,10 @@ export class DeepScanRemoteCoordinator { private readonly input: { run: DeepScanRunState; registry: Pick; - options: Omit; + options: Omit< + DeepScanCoordinatorStartOptions, + "run" | "observeReplacement" + >; }, ) {} @@ -137,6 +150,7 @@ export class DeepScanRemoteCoordinator { } if (run !== undefined) { if (run.status !== "running") return run; + requireSupportedDeepScan(run); const heartbeat = run.updatedAt ? Date.parse(run.updatedAt) @@ -171,7 +185,11 @@ export class DeepScanRemoteCoordinator { } } if (claim?.acquired) { - const coordinator = registry.start({ ...options, run: claim.run }); + const coordinator = await startClaimedCoordinator( + registry, + options, + claim.run, + ); return deadline === undefined ? await coordinator.wait(signal) : await coordinator.wait( @@ -193,12 +211,15 @@ export class DeepScanRemoteCoordinator { } } -export async function startOrJoinDeepScanCoordinator( - input: ConstructorParameters[0], -): Promise<{ +export async function startOrJoinDeepScanCoordinator(input: { + run: DeepScanRunState; + registry: Pick; + options: Omit; +}): Promise<{ coordinator: DeepScanCoordinator | DeepScanRemoteCoordinator; joined: boolean; }> { + requireSupportedDeepScan(input.run); const existing = input.registry.get(input.run.scanId); if (existing) return { coordinator: existing, joined: true }; const threadId = input.options.threadId; @@ -223,11 +244,53 @@ export async function startOrJoinDeepScanCoordinator( }; } return { - coordinator: input.registry.start({ ...input.options, run: claim.run }), + coordinator: await startClaimedCoordinator( + input.registry, + input.options, + claim.run, + ), joined: false, }; } +async function startClaimedCoordinator( + registry: Pick, + options: Omit, + run: DeepScanRunState, +): Promise { + requireSupportedDeepScan(run); + const executor = + options.prepareExecutor && !run.finalizationInput + ? await options.prepareExecutor(run) + : options.executor; + return registry.start({ ...options, executor, run }); +} + +function requireSupportedDeepScan(run: DeepScanRunState): void { + if ( + run.finalizationInput !== undefined && + (run.workflowVersion !== "deep-security-scan/v2" || + run.finalizationInput.version !== 1) + ) { + throw new Error( + "This executor does not support this Deep Scan finalization input version.", + ); + } + // Missing versions are supported for older adapters that did not project them. + if ( + (run.schemaVersion !== undefined && run.schemaVersion !== 1) || + (run.workflowVersion !== undefined && + run.workflowVersion !== "deep-security-scan/v1" && + run.workflowVersion !== "deep-scan-mcp/v1" && + run.workflowVersion !== "deep-security-scan/v2") + ) { + throw new Error( + "This Deep Scan uses an unsupported workflow or schema version. " + + "Resume it with a compatible Codex Security release.", + ); + } +} + function remoteAbortError(reason: unknown): Error { const error = new Error("Deep Scan observation was aborted.", { cause: reason, diff --git a/plugins/codex-security/mcp-app/src/deep-scan/response.ts b/plugins/codex-security/mcp-app/src/deep-scan/response.ts index fbde85b767..60593fb8ae 100644 --- a/plugins/codex-security/mcp-app/src/deep-scan/response.ts +++ b/plugins/codex-security/mcp-app/src/deep-scan/response.ts @@ -1,3 +1,4 @@ +import type { DeepScanExecutionSettingsSnapshot } from "./recovery-settings.js"; import { isRecord } from "../record.js"; import { z } from "zod"; @@ -27,6 +28,7 @@ export const workerSchema = responseObject({ attempt: nonNegativeInteger, sdkThreadId: optionalString, resultManifestPath: optionalString, + acceptedResultPath: optionalString, completionSequence: optionalValue(positiveInteger), error: optionalString, }).transform( @@ -40,6 +42,36 @@ export const dedupInputSchema = z.object({ dedupWorkerId: requiredString, discoveryWorkerId: requiredString, inputOrder: nonNegativeInteger, + resultManifestPath: optionalString, + resultManifestSha256: optionalString, + attempt: optionalValue(positiveInteger), +}); + +export const finalizationInputSchema = z.object({ + version: positiveInteger, + resultPath: requiredString.nullable(), + resultSha256: requiredString.nullable(), + terminalReason: z.enum(["saturated", "capped"]), + omittedWorkerIds: z.array(requiredString), + selectedAt: requiredString, +}); +export const mergeClaimSchema = responseObject({ + workerId: requiredString, + previousWorkerId: optionalString, + previousResultPath: optionalString, + previousResultSha256: optionalString, +}); +const usageOwnerSchema = z.object({ + threadId: optionalString.transform((value) => value ?? null), + turnId: optionalString.transform((value) => value ?? null), + startedAt: requiredString, + dedicated: z.boolean().optional(), +}); +const committedMergeSchema = z.object({ + workerId: requiredString, + resultManifestPath: requiredString, + resultManifestSha256: requiredString, + newFindings: nonNegativeInteger, }); export const configSchema = z @@ -76,6 +108,30 @@ export const runSchema = z includePaths: z.array(z.string()).optional(), filePaths: z.array(z.string()).optional(), scanId: requiredString, + ownerThreadId: optionalString, + schemaVersion: optionalValue(positiveInteger), + workflowVersion: optionalString, + model: optionalString, + reasoningEffort: optionalString, + finalizationInput: finalizationInputSchema + .nullish() + .transform((value) => value ?? undefined), + usageOwner: usageOwnerSchema + .nullish() + .transform((value) => value ?? null), + executionSettings: z + .looseObject({}) + .nullish() + .transform((value) => + value == null + ? undefined + : (value as unknown as DeepScanExecutionSettingsSnapshot), + ), + mergeClaims: z + .array(mergeClaimSchema) + .nullish() + .transform((value) => value ?? []), + committedMerge: committedMergeSchema.optional(), coordinatorGeneration: optionalValue(positiveInteger), createdAt: optionalString, updatedAt: optionalString, @@ -101,10 +157,11 @@ export const runSchema = z .transform((value) => value ?? []), }), ) - .transform(({ workers, dedupInputs, ...run }) => ({ + .transform(({ workers, dedupInputs, mergeClaims, ...run }) => ({ ...run, persistedWorkers: workers, persistedDedupInputs: dedupInputs, + persistedMergeClaims: mergeClaims, })); function parseResponse( @@ -134,14 +191,16 @@ export function parseDeepScan(result: Record) { export function parseWorker(result: Record, workerId: string) { const deepScan = parseResponse(z.looseObject({}), result.deepScan); - if (!Array.isArray(deepScan.workers)) { + if (!deepScan.workerReceipt && !Array.isArray(deepScan.workers)) { throw new Error( "Codex Security workbench did not return deepScan.workers.", ); } - const value = deepScan.workers.find( - (candidate) => isRecord(candidate) && candidate.id === workerId, - ); + const value = + deepScan.workerReceipt ?? + (deepScan.workers as unknown[]).find( + (candidate) => isRecord(candidate) && candidate.id === workerId, + ); if (!value) { throw new Error( `Codex Security workbench did not return Deep Scan worker ${workerId}.`, diff --git a/plugins/codex-security/mcp-app/src/deep-scan/store.ts b/plugins/codex-security/mcp-app/src/deep-scan/store.ts index 9a0d273295..600bcff989 100644 --- a/plugins/codex-security/mcp-app/src/deep-scan/store.ts +++ b/plugins/codex-security/mcp-app/src/deep-scan/store.ts @@ -3,6 +3,7 @@ import { availableParallelism } from "node:os"; import { join } from "node:path"; import { setTimeout as delay } from "node:timers/promises"; import { writeJsonAtomic } from "./artifacts.js"; +import type { DeepScanExecutionSettings } from "./recovery-settings.js"; import { parseDeepScan, parseWorker } from "./response.js"; export { parseDeepScan }; @@ -26,9 +27,13 @@ type JsonObject = Record; export type WorkbenchRunner = ( args: string[], input?: string, + selectFinalization?: boolean, + withExecutionSettings?: boolean, + signal?: AbortSignal, + releaseCoordinator?: boolean, ) => Promise; -const WORKFLOW_VERSION = "deep-scan-mcp/v1"; +const WORKFLOW_VERSION = "deep-security-scan/v2"; const MAX_IDEMPOTENT_PERSISTENCE_ATTEMPTS = 3; const PERSISTENCE_RETRY_BASE_DELAY_MS = 100; @@ -123,6 +128,7 @@ export class WorkbenchDeepScanStore { { input: DeepScanCoordinatorLeaseInput; run: DeepScanRunState; + heartbeatWrite?: Promise; } >(); @@ -138,6 +144,7 @@ export class WorkbenchDeepScanStore { reasoningEffort?: string; threadId: string; scanRoot: string; + executionSettings?: DeepScanExecutionSettings | null; }): Promise { const userContext = input.userContext; const result = await this.enqueueWrite( @@ -146,7 +153,8 @@ export class WorkbenchDeepScanStore { scanId: input.scanId || undefined, targetPath: input.targetPath || undefined, scope: input.scope || undefined, - userContextStdin: Boolean(userContext), + userContextStdin: + Boolean(userContext) && input.executionSettings === undefined, claimToken: input.handoffClaimToken || undefined, model: input.model || undefined, reasoningEffort: input.reasoningEffort || undefined, @@ -154,7 +162,15 @@ export class WorkbenchDeepScanStore { availableParallelism: availableParallelism(), workflowVersion: WORKFLOW_VERSION, }), - userContext, + false, + input.executionSettings === undefined + ? userContext + : JSON.stringify({ + executionSettings: input.executionSettings, + userContext, + }), + false, + input.executionSettings !== undefined, ); const run = parseDeepScan(result); const startDisposition = result.startDisposition; @@ -163,19 +179,23 @@ export class WorkbenchDeepScanStore { "Codex Security workbench returned an invalid Deep Scan start disposition.", ); } - return run; + return { ...run, startDisposition }; } async get(scanId: string, threadId: string): Promise { const run = parseDeepScan( await this.runWorkbench( workbenchArgs("get-deep-scan", { scanId, threadId }), + undefined, + false, + true, ), ); const generation = this.coordinatorLeases.get(scanId)?.run.coordinatorGeneration; if ( - run.status !== "running" || + (run.status !== "running" && + !(run.status === "succeeded" && run.finalizationInput)) || (generation !== undefined && run.coordinatorGeneration !== undefined && run.coordinatorGeneration > generation) @@ -195,6 +215,10 @@ export class WorkbenchDeepScanStore { coordinatorGeneration: this.coordinatorGeneration(input.scanId), claimToken: input.handoffClaimToken || undefined, }), + false, + undefined, + false, + true, ); const run = parseDeepScan(result); const disposition = result.coordinatorDisposition; @@ -237,39 +261,97 @@ export class WorkbenchDeepScanStore { ); } const updatedAt = new Date().toISOString(); - await writeJsonAtomic( - join( - lease.run.scanDir, - "artifacts", - "deep_discovery", - `coordinator-heartbeat-${lease.run.coordinatorGeneration}.json`, + await this.writeCoordinatorHeartbeat(lease, updatedAt); + return { ...lease.run, updatedAt }; + } + + async releaseCoordinator(scanId: string): Promise { + const lease = this.coordinatorLeases.get(scanId); + if (!lease) return; + this.coordinatorLeases.delete(scanId); + await lease.heartbeatWrite; + await this.enqueueWrite( + [ + "claim-deep-scan-coordinator", + "--scan-id", + scanId, + "--thread-id", + lease.input.threadId, + "--coordinator-generation", + String(lease.run.coordinatorGeneration), + ...(lease.input.handoffClaimToken + ? ["--claim-token", lease.input.handoffClaimToken] + : []), + ], + false, + undefined, + false, + false, + true, + ); + } + + private writeCoordinatorHeartbeat( + lease: { run: DeepScanRunState; heartbeatWrite?: Promise }, + updatedAt: string, + ): Promise { + // Heartbeats bypass SQLite writes; only writes to this lease file must settle in order. + const { run } = lease; + const operation = (lease.heartbeatWrite ?? Promise.resolve()).then(() => + writeJsonAtomic( + join( + run.scanDir, + "artifacts", + "deep_discovery", + `coordinator-heartbeat-${run.coordinatorGeneration}.json`, + ), + { + coordinatorGeneration: run.coordinatorGeneration, + updatedAt, + }, ), - { coordinatorGeneration: lease.run.coordinatorGeneration, updatedAt }, ); - return { ...lease.run, updatedAt }; + lease.heartbeatWrite = operation.then( + () => undefined, + () => undefined, + ); + return operation; + } + + async cancel(scanId: string, threadId: string): Promise { + return this.enqueueWrite([ + "cancel-scan", + "--scan-id", + scanId, + "--thread-id", + threadId, + ]); } async updateWorker( update: DeepScanWorkerMutation, ): Promise { - const result = await this.enqueueWrite([ - ...workbenchArgs("upsert-deep-scan-worker", { - scanId: update.scanId, - workerId: update.id, - kind: update.kind, - status: update.status, - promptPath: update.promptPath, - artifactDir: update.artifactDir, - attempt: update.attempt, - coordinatorGeneration: this.coordinatorGeneration(update.scanId), - resultManifestPath: update.resultManifestPath || undefined, - sdkThreadId: update.threadId || undefined, - }), - ...(update.error ? [`--error-message=${update.error}`] : []), - ...(update.replaceableFailureKind - ? ["--replaceable-failure-kind", update.replaceableFailureKind] - : []), - ]); + const result = await this.enqueueWrite( + [ + ...workbenchArgs("upsert-deep-scan-worker", { + scanId: update.scanId, + workerId: update.id, + kind: update.kind, + status: update.status, + promptPath: update.promptPath, + artifactDir: update.artifactDir, + attempt: update.attempt, + coordinatorGeneration: this.coordinatorGeneration(update.scanId), + resultManifestPath: update.resultManifestPath || undefined, + sdkThreadId: update.threadId || undefined, + }), + ...(update.error ? [`--error-message=${update.error}`] : []), + ...(update.replaceableFailureKind + ? ["--replaceable-failure-kind", update.replaceableFailureKind] + : []), + ], + true, + ); return parseWorker(result, update.id); } @@ -279,16 +361,19 @@ export class WorkbenchDeepScanStore { workerIds: string[]; promptPath: string; artifactDir: string; - }): Promise { - await this.enqueueWrite( - workbenchArgs("claim-deep-scan-dedup", { - scanId: input.scanId, - workerId: input.id, - promptPath: input.promptPath, - artifactDir: input.artifactDir, - coordinatorGeneration: this.coordinatorGeneration(input.scanId), - inputWorkerId: input.workerIds, - }), + }): Promise { + return parseDeepScan( + await this.enqueueWrite( + workbenchArgs("claim-deep-scan-dedup", { + scanId: input.scanId, + workerId: input.id, + promptPath: input.promptPath, + artifactDir: input.artifactDir, + coordinatorGeneration: this.coordinatorGeneration(input.scanId), + inputWorkerId: input.workerIds, + }), + true, + ), ); } @@ -307,8 +392,45 @@ export class WorkbenchDeepScanStore { ); } + async selectFinalization(input: { + scanId: string; + coordinatorGeneration?: number; + reason: DeepScanTerminalReason; + manifestPath: string; + resultPath?: string; + omittedWorkerIds: string[]; + }): Promise { + return parseDeepScan( + await this.enqueueWrite( + [ + "finish-deep-scan", + "--scan-id", + input.scanId, + ...(input.coordinatorGeneration === undefined + ? this.coordinatorLeaseArgs(input.scanId) + : [ + "--coordinator-generation", + String(input.coordinatorGeneration), + ]), + "--terminal-reason", + input.reason, + "--manifest-path", + input.manifestPath, + ...input.omittedWorkerIds.flatMap((workerId) => [ + "--omitted-worker-id", + workerId, + ]), + ], + true, + JSON.stringify({ resultPath: input.resultPath ?? null }), + true, + ), + ); + } + async finish(input: { scanId: string; + coordinatorGeneration?: number; reason: DeepScanTerminalReason; manifestPath: string; stagedManifestPath?: string; @@ -318,7 +440,10 @@ export class WorkbenchDeepScanStore { await this.enqueueWrite( workbenchArgs("finish-deep-scan", { scanId: input.scanId, - coordinatorGeneration: this.coordinatorGeneration(input.scanId), + coordinatorGeneration: + input.coordinatorGeneration === undefined + ? this.coordinatorGeneration(input.scanId) + : input.coordinatorGeneration, terminalReason: input.reason, manifestPath: input.manifestPath, stagedManifestPath: input.stagedManifestPath || undefined, @@ -411,12 +536,27 @@ export class WorkbenchDeepScanStore { : ["--coordinator-generation", String(generation)]; } - private enqueueWrite(args: string[], input?: string): Promise { + private enqueueWrite( + args: string[], + retryTransientFailure = false, + input?: string, + selectFinalization = false, + withExecutionSettings = false, + releaseCoordinator = false, + ): Promise { const operation = this.writeTail.then(async () => { try { - return RETRYABLE_PERSISTENCE_COMMANDS.has(args[0]) - ? await this.runIdempotentPersistence(args) - : await this.runWorkbench(args, input); + return retryTransientFailure || + RETRYABLE_PERSISTENCE_COMMANDS.has(args[0]) + ? await this.runIdempotentPersistence(args, input, selectFinalization) + : await this.runWorkbench( + args, + input, + selectFinalization, + withExecutionSettings, + undefined, + releaseCoordinator, + ); } catch (error) { const scanId = argumentValue(args, "--scan-id"); const lease = scanId ? this.coordinatorLeases.get(scanId) : undefined; @@ -448,11 +588,15 @@ export class WorkbenchDeepScanStore { } /** Replay only existing, same-identity workbench mutations after transient failures. */ - private async runIdempotentPersistence(args: string[]): Promise { + private async runIdempotentPersistence( + args: string[], + input?: string, + selectFinalization = false, + ): Promise { const startedAt = Date.now(); for (let attempt = 1; ; attempt += 1) { try { - return await this.runWorkbench(args); + return await this.runWorkbench(args, input, selectFinalization); } catch (error) { if (!isTransientPersistenceError(error)) { throw error; diff --git a/plugins/codex-security/mcp-app/src/deep-scan/types.ts b/plugins/codex-security/mcp-app/src/deep-scan/types.ts index 542508b9bc..2ebde1107a 100644 --- a/plugins/codex-security/mcp-app/src/deep-scan/types.ts +++ b/plugins/codex-security/mcp-app/src/deep-scan/types.ts @@ -3,6 +3,8 @@ import type { WorkbenchDeepScanStore } from "./store.js"; import type { z } from "zod"; import type { configSchema, + finalizationInputSchema, + mergeClaimSchema, dedupInputSchema, runSchema, workerSchema, @@ -12,10 +14,20 @@ import type { export type DeepScanConfig = z.output; export type DeepScanRunState = Omit< z.output, - "persistedWorkers" | "persistedDedupInputs" + | "persistedWorkers" + | "persistedDedupInputs" + | "persistedMergeClaims" + | "finalizationInput" + | "usageOwner" + | "executionSettings" > & { + finalizationInput?: z.output["finalizationInput"]; + usageOwner?: z.output["usageOwner"]; + executionSettings?: z.output["executionSettings"]; + persistedMergeClaims?: PersistedDeepScanMergeClaim[]; persistedWorkers?: PersistedDeepScanWorker[]; persistedDedupInputs?: PersistedDeepScanDedupInput[]; + startDisposition?: "created" | "joined"; }; export type DeepScanTerminalReason = NonNullable< DeepScanRunState["terminalReason"] @@ -26,6 +38,18 @@ export type PersistedDeepScanWorker = z.output & { consecutiveErrors?: number; }; export type DeepScanWorkerKind = PersistedDeepScanWorker["kind"]; +export type DeepScanRunStatus = DeepScanRunState["status"]; +export type DeepScanWorkerStatus = PersistedDeepScanWorker["status"]; +export type DeepScanMergeState = PersistedDeepScanWorker["mergeState"]; +export type DeepScanFinalizationInput = z.output< + typeof finalizationInputSchema +>; +export type PersistedDeepScanMergeClaim = z.output; +export interface DeepScanCanonicalArtifacts { + inScopeFilesPath: string; + candidateLedgerPath: string; +} +export type DeepScanReducerArtifacts = DeepScanCanonicalArtifacts; export interface DeepScanCoordinatorClaim { run: DeepScanRunState; @@ -62,8 +86,13 @@ export interface DedupCommit { /** Durable operations implemented by the Python workbench. */ export type DeepScanStore = Omit< WorkbenchDeepScanStore, - "begin" | "coordinatorLeaseArgs" ->; + "begin" | "coordinatorLeaseArgs" | "selectFinalization" | "claimDedup" +> & { + selectFinalization?: WorkbenchDeepScanStore["selectFinalization"]; + claimDedup( + ...args: Parameters + ): Promise; +}; /** Host-bound worker artifact state; never populate this from model input. */ export interface CodexWorkerArtifactContext { diff --git a/plugins/codex-security/mcp-app/src/deep-scan/worker-runner.ts b/plugins/codex-security/mcp-app/src/deep-scan/worker-runner.ts index 4a36f4572b..ec549b5442 100644 --- a/plugins/codex-security/mcp-app/src/deep-scan/worker-runner.ts +++ b/plugins/codex-security/mcp-app/src/deep-scan/worker-runner.ts @@ -1,9 +1,13 @@ +import { + auditEvidence, + runAcceptedAudit, +} from "../../../../../sdk/typescript/src/accepted-audit.js"; import { promises as fs } from "node:fs"; import type { ScanDraftInput } from "../artifact-scan-draft.js"; import { dirname, join } from "node:path"; -import { getCodexSecurityDeepReducerInputs } from "../artifact-deep-reducer.js"; +import { readDeepReductionSources } from "../artifact-deep-reducer.js"; import { - validateDiscoveryArtifacts, + readDiscoveryAuditDraft, validateReducerArtifacts, } from "./artifact-validation.js"; import type { @@ -14,6 +18,7 @@ import { archiveDirectory, writePrivateFile } from "./artifacts.js"; import type { DeepScanArtifacts } from "./artifacts.js"; import { abortError, + errorNameWithCode, boundedDeepScanErrorMessage, classifyCodexWorkerError as asError, DeepScanNonRetryableError, @@ -36,9 +41,13 @@ import type { export interface AcceptedDiscovery { id: string; + label: string; + artifactDir: string; resultPath: string; result: ScanDraftInput; completionSequence: number; + attempt: number; + threadId?: string; } export type DiscoveryOutcome = @@ -51,8 +60,13 @@ export type DiscoveryOutcome = export interface SuccessfulDedupOutcome { type: "dedup"; + id: string; + consumed: AcceptedDiscovery[]; resultPath: string; result: DeepReductionInput; + newFindings: number; + attempt: number; + threadId?: string; run: DeepScanRunState; } @@ -71,6 +85,7 @@ export interface ReducerRequest { label: string; consumed: AcceptedDiscovery[]; previousReducerResultPath?: string; + previousSourceCoverage?: DeepReductionInput["sourceCoverage"]; } export interface DeepScanWorkerRunnerOptions { @@ -87,15 +102,22 @@ export interface DeepScanWorkerRunnerOptions { onNonRetryableError?: (error: Error) => void; } +interface WorkerAttemptEvidence { + attempt: number; + threadId?: string; +} + type WorkerAttemptOutcome = | { status: "succeeded"; attempt: number; threadId?: string } | { status: "failed"; error: Error; + attempt?: number; + threadId?: string; replaceableFailureKind?: DeepScanReplaceableFailureKind; consecutiveErrors?: number; } - | { status: "canceled" }; + | { status: "canceled"; attempt?: number; threadId?: string }; /** Owns prompt rendering, retries, validation, and persistence for each worker. */ export class DeepScanWorkerRunner { @@ -155,11 +177,13 @@ export class DeepScanWorkerRunner { artifactContext: { root: artifactDir, layout: "worker" }, subagents: run.config.subagents, validate: async () => { - discoveryResult = await validateDiscoveryArtifacts( + const draft = await readDiscoveryAuditDraft( artifacts, resultPath, run.scanId, ); + discoveryResult = draft.complete !== false ? draft : undefined; + return draft; }, }); if (outcome.status === "succeeded" && this.options.signal.aborted) { @@ -222,9 +246,13 @@ export class DeepScanWorkerRunner { status: "succeeded", worker: { id: workerId, - resultPath, + label: workerLabel, + artifactDir, + resultPath: persisted.acceptedResultPath ?? resultPath, result: discoveryResult!, completionSequence: persisted.completionSequence, + attempt: outcome.attempt, + threadId: outcome.threadId, }, }; } @@ -233,9 +261,9 @@ export class DeepScanWorkerRunner { const { id: reducerId, label: reducerLabel, - consumed, - previousReducerResultPath, + previousSourceCoverage, } = request; + let { consumed, previousReducerResultPath } = request; const { artifacts, run } = this.options; const reducerRoot = join(artifacts.dedupRoot, reducerLabel); const artifactDir = join(reducerRoot, "output"); @@ -246,13 +274,37 @@ export class DeepScanWorkerRunner { const workerIds = consumed.map((worker) => worker.id); const basePrompt = renderDedupPrompt(reducerLabel, workerIds); await writePrivateFile(promptPath, basePrompt); - await this.options.store.claimDedup({ + const claimed = await this.options.store.claimDedup({ id: reducerId, scanId: run.scanId, workerIds, promptPath, artifactDir, }); + const claim = claimed?.persistedMergeClaims?.find( + (item) => item.workerId === reducerId, + ); + if (claim) { + previousReducerResultPath = claim.previousResultPath; + const inputs = + claimed?.persistedDedupInputs?.filter( + (item) => item.dedupWorkerId === reducerId, + ) ?? []; + consumed = inputs + .sort((a, b) => a.inputOrder - b.inputOrder) + .map((item) => { + const discovery = consumed.find( + (worker) => worker.id === item.discoveryWorkerId, + ); + if (!discovery || !item.resultManifestPath) + throw new Error("The reducer claim is missing an accepted input."); + return { + ...discovery, + resultPath: item.resultManifestPath, + attempt: item.attempt ?? discovery.attempt, + }; + }); + } this.options.log({ event: "dedup_claimed", scanId: run.scanId, @@ -260,6 +312,9 @@ export class DeepScanWorkerRunner { count: consumed.length, }); + const persistSourceCoverage = + "workflowVersion" in run && + run.workflowVersion === "deep-security-scan/v2"; const artifactContext = { root: artifactDir, repoRoot: run.targetPath, @@ -267,17 +322,23 @@ export class DeepScanWorkerRunner { layout: "reducer" as const, deepReducer: { scanRoot: artifacts.scanDir, + persistSourceCoverage, claimedWorkers: consumed.map((worker) => ({ id: worker.id, resultPath: worker.resultPath, + artifactDir: worker.artifactDir, + attempt: worker.attempt, })), previousReducerResultPath, }, }; // Snapshot inputs before execution: direct file output has the same // conservation checks as the MCP writer without rereading consumed sources. - const sources = await getCodexSecurityDeepReducerInputs(artifactContext); - let reducerValidation!: ReducerArtifactValidation; + const sources = await readDeepReductionSources(artifactContext); + if (sources.previous && previousSourceCoverage !== undefined) { + sources.previous.sourceCoverage = structuredClone(previousSourceCoverage); + } + let reducerValidation: ReducerArtifactValidation | undefined; let outcome = await this.runWorkerWithRetries({ workerId: reducerId, kind: "dedup", @@ -295,6 +356,7 @@ export class DeepScanWorkerRunner { reducerId, previousReducerResultPath, sources, + persistSourceCoverage, }, run.scanId, ); @@ -324,25 +386,43 @@ export class DeepScanWorkerRunner { }; } if (outcome.status === "canceled") throw abortError(); - + if (!reducerValidation) { + throw new Error( + `${reducerId} completed without validated reducer artifacts.`, + ); + } const commit = { id: reducerId, scanId: run.scanId, newFindings: reducerValidation.newFindings, resultManifestPath: resultPath, }; - const committed = await this.options.store.commitDedup(commit); + const committed = await this.replayStoreMutation( + "dedup_commit_replay", + reducerId, + async () => await this.options.store.commitDedup(commit), + ); + const accepted = committed.committedMerge; + const acceptedPath = accepted?.resultManifestPath ?? resultPath; + // V1 checkpoints omit host-only coverage; retain the validated projection. + const acceptedResult = reducerValidation.result; + const newFindings = accepted?.newFindings ?? reducerValidation.newFindings; this.options.log({ event: "dedup_committed", scanId: run.scanId, workerId: reducerId, count: consumed.length, - newFindings: reducerValidation.newFindings, + newFindings, }); return { type: "dedup", - resultPath, - result: reducerValidation.result, + id: reducerId, + consumed, + resultPath: acceptedPath, + result: acceptedResult, + newFindings, + attempt: outcome.attempt, + threadId: outcome.threadId, run: committed, }; } @@ -355,7 +435,7 @@ export class DeepScanWorkerRunner { artifactDir: string; artifactContext?: CodexWorkerArtifactContext; subagents: number; - validate: () => Promise; + validate: () => Promise; }): Promise { const { run, signal } = this.options; const maximumAttempts = this.options.retryDelaysMs.length + 1; @@ -389,54 +469,77 @@ export class DeepScanWorkerRunner { attempt, }); try { - const result = await this.options.executor.run({ - kind: input.kind, - promptPath: executionPromptPath, - // Discovery workers write only to their isolated directory. Reducers - // own shared scan artifacts; the target remains read-only. - workingDirectory: - input.kind === "discovery" - ? input.artifactDir - : join(run.scanDir, "artifacts"), - subagents: input.subagents, - signal, - onNonRetryableError: this.options.onNonRetryableError, - resumeThreadId: resumableThreadId, - continuationPrompt: resumableThreadId - ? (continuationPrompt ?? - transientExecutionContinuation(input.kind, attempt)) - : undefined, - artifactContext: input.artifactContext, - onThreadStarted: async (threadId) => { - activeThreadId = threadId; - lastThreadId = threadId; - await this.options.store.updateWorker({ - ...baseMutation, - threadId, - }); - this.options.log({ - event: "worker_thread_started", - scanId: run.scanId, - workerId: input.workerId, - kind: input.kind, - attempt, - threadId, - }); - }, - }); - if (signal.aborted) { - return await this.cancelAttempt(input, attempt, activeThreadId); - } - validationStarted = true; - try { - await input.validate(); - } catch (validationError) { - throw withWorkerDiagnostics(validationError, result.diagnostics); + const execute = () => + this.options.executor.run({ + kind: input.kind, + promptPath: executionPromptPath, + // Discovery workers write only to their isolated directory. Reducers + // own shared scan artifacts; the target remains read-only. + workingDirectory: + input.kind === "discovery" + ? input.artifactDir + : join(run.scanDir, "artifacts"), + subagents: input.subagents, + signal, + onNonRetryableError: this.options.onNonRetryableError, + resumeThreadId: resumableThreadId, + continuationPrompt: resumableThreadId + ? (continuationPrompt ?? + transientExecutionContinuation(input.kind, attempt)) + : undefined, + artifactContext: input.artifactContext, + onThreadStarted: async (threadId) => { + activeThreadId = threadId; + lastThreadId = threadId; + await this.options.store.updateWorker({ + ...baseMutation, + threadId, + }); + this.options.log({ + event: "worker_thread_started", + scanId: run.scanId, + workerId: input.workerId, + kind: input.kind, + attempt, + threadId, + }); + }, + }); + const accept = async (result: Awaited>) => { + validationStarted = true; + let accepted: ScanDraftInput | void; + try { + accepted = await input.validate(); + } catch (validationError) { + throw withWorkerDiagnostics(validationError, result.diagnostics); + } + validationCompleted = accepted?.complete !== false; + return accepted === undefined ? {} : auditEvidence(accepted); + }; + // Reducers keep their aggregate contract; discovery uses the shared audit. + const audit = + input.kind === "discovery" + ? await runAcceptedAudit({ signal, execute, accept }) + : undefined; + let result: Awaited>; + if (audit) { + if (audit.status === "checkpoint") { + throw withWorkerDiagnostics( + new Error( + "Standard scan worker wrote only a checkpoint; its audit is not complete.", + ), + audit.execution.diagnostics, + ); + } + result = audit.execution; + } else { + result = await execute(); + if (signal.aborted) + return await this.cancelAttempt(input, attempt, activeThreadId); + await accept(result); } - validationCompleted = true; - if (signal.aborted) { + if (signal.aborted) return await this.cancelAttempt(input, attempt, activeThreadId); - } this.options.log({ event: "worker_succeeded", scanId: run.scanId, @@ -489,6 +592,8 @@ export class DeepScanWorkerRunner { ...(persistedFailure.consecutiveErrors === undefined ? {} : { consecutiveErrors: persistedFailure.consecutiveErrors }), + attempt, + threadId: activeThreadId, }; } await this.options.store.updateWorker({ @@ -563,14 +668,14 @@ export class DeepScanWorkerRunner { ); } - private async cancelAttempt( + private async persistWorkerCancellation( input: Pick< Parameters[0], "workerId" | "kind" | "promptPath" | "artifactDir" >, attempt: number, threadId: string | undefined, - ): Promise { + ): Promise { const coordinatorShutdown = this.options.signal.reason === "mcp_transport_closed"; await this.options.store.updateWorker({ @@ -586,7 +691,56 @@ export class DeepScanWorkerRunner { ? { error: "coordinator_shutdown: mcp_transport_closed" } : {}), }); - return { status: "canceled" }; + } + + /** Replay idempotent SQLite commits when their process response is ambiguous. */ + private async replayStoreMutation( + event: string, + workerId: string, + operation: () => Promise, + ): Promise { + try { + return await operation(); + } catch (firstError) { + // The workbench store already replays these writes before reporting exhaustion. + if ( + firstError instanceof Error && + firstError.name === "DeepScanPersistenceError" + ) + throw firstError; + this.options.log({ + event, + scanId: this.options.run.scanId, + workerId, + reason: errorNameWithCode(asError(firstError)), + }); + try { + return await operation(); + } catch (replayError) { + throw new Error( + `Deep Scan persistence replay failed: ${asError(replayError).message}`, + { cause: firstError }, + ); + } + } + } + + private async cancelAttempt( + input: { + workerId: string; + kind: "discovery" | "dedup"; + promptPath: string; + artifactDir: string; + }, + attempt: number, + threadId: string | undefined, + ): Promise { + await this.persistWorkerCancellation(input, attempt, threadId); + return { + status: "canceled", + attempt, + threadId, + }; } } diff --git a/plugins/codex-security/mcp-app/src/workbench-client.ts b/plugins/codex-security/mcp-app/src/workbench-client.ts index 980634cc27..a8e2e7c01a 100644 --- a/plugins/codex-security/mcp-app/src/workbench-client.ts +++ b/plugins/codex-security/mcp-app/src/workbench-client.ts @@ -31,12 +31,25 @@ interface WorkbenchOptions { export async function runWorkbench( args: string[], input?: string | Buffer, - options: WorkbenchOptions = {}, + selectFinalization: boolean | WorkbenchOptions = false, + withExecutionSettings = false, + signal?: AbortSignal, + releaseCoordinator = false, + isolatedPython = false, ): Promise { let pythonCommand: string | undefined; try { pythonCommand = await resolvePythonCommand(); - return await executeWorkbench(pythonCommand, args, input, options); + return await executeWorkbench( + pythonCommand, + args, + input, + selectFinalization, + withExecutionSettings, + signal, + releaseCoordinator, + isolatedPython, + ); } catch (error) { const launchError = pythonCommand ? missingPythonHelperMessage(error, pythonCommand) @@ -55,8 +68,29 @@ async function executeWorkbench( pythonCommand: string, args: string[], input?: string | Buffer, - options: WorkbenchOptions = {}, + selectFinalization: boolean | WorkbenchOptions = false, + withExecutionSettings = false, + signal?: AbortSignal, + releaseCoordinator = false, + isolatedPython = false, ): Promise { + if (typeof selectFinalization === "object") { + isolatedPython = selectFinalization.isolatedPython ?? isolatedPython; + selectFinalization = false; + } + const internalInvocation = releaseCoordinator + ? "release_coordinator=True" + : selectFinalization + ? "select_finalization=True" + : withExecutionSettings + ? "with_execution_settings=True" + : undefined; + const pythonSource = internalInvocation + ? WORKBENCH_PYTHON.replace( + "workbench_db.main()", + `workbench_db.main(${internalInvocation})`, + ) + : WORKBENCH_PYTHON; const timeout = [ "begin-deep-scan", "cancel-scan", @@ -89,13 +123,14 @@ async function executeWorkbench( const execution = execFileAsync( pythonCommand, [ - ...(options.isolatedPython ? ["-I", "-X", "utf8"] : []), + ...(isolatedPython ? ["-I", "-X", "utf8"] : []), "-B", "-c", - WORKBENCH_PYTHON, + pythonSource, workbenchScriptPath(), ], { + signal, cwd: PLUGIN_ROOT, windowsHide: true, env: process.env, diff --git a/plugins/codex-security/mcp-app/src/workbench/migrations.ts b/plugins/codex-security/mcp-app/src/workbench/migrations.ts index 86da3ab781..f039e310d0 100644 --- a/plugins/codex-security/mcp-app/src/workbench/migrations.ts +++ b/plugins/codex-security/mcp-app/src/workbench/migrations.ts @@ -227,6 +227,21 @@ export function applyMigrations( if ([2, 12, 13, 26, 28, 31, 32, 42, 47, 50].includes(item.version)) repairAdditive(database, item.version); else if (item.version === 11) repairDeepScan(database); + else if (item.version >= 53 && item.version <= 58) { + // Restore recorded extensions when the base Deep tables were recreated. + // Data updates from an already applied migration must not run again. + for (const statement of item.statements) { + if (statement.startsWith("ALTER TABLE ")) + addColumn(database, statement); + else if (statement.startsWith("CREATE TABLE ")) + database.exec( + statement.replace( + "CREATE TABLE ", + "CREATE TABLE IF NOT EXISTS ", + ), + ); + } + } } else { database.exec(item.statements.join("\n")); if (item.version === 38) migrateWorkflowResults(database); diff --git a/plugins/codex-security/mcp-app/tests/assertions.ts b/plugins/codex-security/mcp-app/tests/assertions.ts index d9460ea384..ce4cc55781 100644 --- a/plugins/codex-security/mcp-app/tests/assertions.ts +++ b/plugins/codex-security/mcp-app/tests/assertions.ts @@ -26,6 +26,61 @@ export function assertFlagPair( assert.equal(args[index + 1], value); } +export function assertReadOnlyWorkerPolicy(args: readonly string[]) { + assert.equal(args.includes("--sandbox"), false); + assert.equal(args.includes("--add-dir"), false); + assert.deepEqual( + args.filter((arg: string) => arg.startsWith("approval_policy=")), + ['approval_policy="never"'], + ); + assert.equal( + args.some((arg: string) => arg.includes("network_access")), + false, + ); + const override = workerPermissionProfileOverride(args); + assertConfigOverrides(args, { + "permissions.codex_security_deep_scan_worker.extends": ":read-only", + "permissions.codex_security_deep_scan_worker.filesystem.:root": "read", + "permissions.codex_security_deep_scan_worker.network.enabled": false, + }); + assert.equal(override.includes('"write"'), false); +} + +export function workerPermissionProfileOverride(args: readonly string[]) { + assert.deepEqual( + args.filter((arg: string) => arg.startsWith("default_permissions=")), + ['default_permissions="codex_security_deep_scan_worker"'], + ); + const overrides = args.filter((arg: string) => + arg.startsWith("permissions.codex_security_deep_scan_worker="), + ); + assert.equal(overrides.length, 1); + return overrides[0]; +} + +export function assertWorkerSubagentPolicy( + args: readonly string[], + subagents: number, +) { + assertConfigOverrides(args, { + "features.multi_agent_v2.enabled": false, + "features.multi_agent_v2.max_concurrent_threads_per_session": subagents + 1, + "features.multi_agent": undefined, + "features.code_mode.excluded_tool_namespaces": undefined, + ...(subagents === 0 + ? { + "agents.max_threads": undefined, + "features.enable_fanout": false, + "features.code_mode.enabled": undefined, + } + : { + "agents.max_threads": subagents, + "features.enable_fanout": undefined, + }), + }); + assert.equal(args.includes("features.multi_agent_v2.enabled=true"), false); +} + export function nativeConfigOverrides(args: readonly string[]) { return args.flatMap((arg, index) => arg === "--config" || arg === "-c" ? [args[index + 1]] : [], diff --git a/plugins/codex-security/mcp-app/tests/build-server.ts b/plugins/codex-security/mcp-app/tests/build-server.ts index c916ec0751..4d3fc2070b 100644 --- a/plugins/codex-security/mcp-app/tests/build-server.ts +++ b/plugins/codex-security/mcp-app/tests/build-server.ts @@ -11,6 +11,7 @@ export function buildServer(outfile: string, options: BuildOptions = {}) { external: ["fsevents"], format: "cjs", loader: { ".md": "text" }, + nodePaths: [path.join(applicationRoot, "node_modules")], logLevel: "silent", outfile, platform: "node", diff --git a/plugins/codex-security/mcp-app/tests/clock_workbench.py b/plugins/codex-security/mcp-app/tests/clock_workbench.py index 54c6e44758..93ecf539e3 100644 --- a/plugins/codex-security/mcp-app/tests/clock_workbench.py +++ b/plugins/codex-security/mcp-app/tests/clock_workbench.py @@ -5,11 +5,17 @@ import sys from pathlib import Path -source = Path(sys.argv[1]) -sys.argv = [str(source), *sys.argv[2:]] -sys.path.insert(0, str(source.parent)) -api = runpy.run_path(str(source), run_name="test_workbench") -instant = os.environ["TEST_WORKBENCH_NOW"] -api["main"].__globals__["now"] = lambda: instant -api["_WORKBENCH_DB_CONTEXT"].now = lambda: instant -api["main"]() + +def main(*, select_finalization: bool = False) -> None: + source = Path(sys.argv[1]) + sys.argv = [str(source), *sys.argv[2:]] + sys.path.insert(0, str(source.parent)) + api = runpy.run_path(str(source), run_name="test_workbench") + instant = os.environ["TEST_WORKBENCH_NOW"] + api["main"].__globals__["now"] = lambda: instant + api["_WORKBENCH_DB_CONTEXT"].now = lambda: instant + api["main"](select_finalization=select_finalization) + + +if __name__ == "__main__": + main() diff --git a/plugins/codex-security/mcp-app/tests/deep_scan_cancellation_cases.ts b/plugins/codex-security/mcp-app/tests/deep_scan_cancellation_cases.ts new file mode 100644 index 0000000000..a8c0058418 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/deep_scan_cancellation_cases.ts @@ -0,0 +1,179 @@ +import assert from "node:assert/strict"; +import childProcess, { + type SpawnOptions, + type ChildProcess, +} from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { mkdir, readFile, readdir, writeFile } from "node:fs/promises"; +import { syncBuiltinESMExports } from "node:module"; +import path from "node:path"; +import { assertFlagPair } from "./assertions.ts"; + +export async function testWorkerCancellation({ + CodexSdkWorkerExecutor, + fakeCodexFixture, + trustedParentSandbox, + restoreEnv, +}: { + CodexSdkWorkerExecutor: typeof import("../src/deep-scan/executor.js").CodexSdkWorkerExecutor; + fakeCodexFixture: typeof import("./test_deep_scan_executor.ts").fakeCodexFixture; + trustedParentSandbox: import("../src/deep-scan/parent-sandbox.js").DeepWorkerParentSandbox; + restoreEnv: (name: string, value: string | undefined) => void; +}) { + const fixture = await fakeCodexFixture(); + const previousPath = process.env.CODEX_CLI_PATH; + const originalSpawn = childProcess.spawn; + let worker: ChildProcess | undefined; + let workerSignal: AbortSignal | undefined; + let cancelDuringCleanup: (() => void) | undefined; + let timeout: NodeJS.Timeout | undefined; + childProcess.spawn = (( + command: string, + args: readonly string[] = [], + options: SpawnOptions = {}, + ) => { + const child = originalSpawn( + command, + command === process.execPath || + command === path.toNamespacedPath(process.execPath) + ? [fixture.executablePath, ...args] + : args, + options, + ); + if (args[0] === "exec") { + assertFlagPair(args, "--thread-source", "security_scan"); + worker = child; + workerSignal = options.signal; + const kill = child.kill; + child.kill = function (...args: Parameters) { + cancelDuringCleanup?.(); + return kill.apply(this, args); + }; + } + return child; + }) as typeof childProcess.spawn; + syncBuiltinESMExports(); + process.env.CODEX_CLI_PATH = process.execPath; + try { + const promptPath = path.join(fixture.root, "prompt.md"); + const workingDirectory = path.join(fixture.root, "artifacts"); + await mkdir(workingDirectory); + const executor = new CodexSdkWorkerExecutor({ + parentSandbox: trustedParentSandbox, + artifactContext: { + pluginRoot: fixture.root, + repoRoot: fixture.root, + scanId: randomUUID(), + includePaths: ["api"], + }, + }); + const run = (controller: AbortController, onThreadStarted?: () => void) => + executor.run({ + kind: "discovery", + promptPath, + workingDirectory, + subagents: 0, + signal: controller.signal, + onThreadStarted, + artifactContext: { root: workingDirectory, layout: "worker" }, + }); + const preAborted = new AbortController(); + const beforeStartup = new Error( + "coordinator canceled before worker startup", + ); + preAborted.abort(beforeStartup); + await assert.rejects( + run(preAborted), + (error: Error) => error === beforeStartup, + ); + assert.equal(worker, undefined); + assert.equal( + (await readdir(fixture.root)).some((name) => name.startsWith("scope-")), + false, + ); + await assert.rejects(readFile(fixture.preflightMarkerPath), { + code: "ENOENT", + }); + await assert.rejects(readFile(fixture.markerPath), { code: "ENOENT" }); + + for (const prompt of [ + "BLOCK_AFTER_START", + "COMPLETE_THEN_HANG", + "FAIL_THEN_HANG", + ]) { + await writeFile(promptPath, `${prompt}\n`); + const controller = new AbortController(); + const cancellation = new Error( + "coordinator canceled its remaining workers", + ); + // Exercise cancellation inside the real SDK's iterator cleanup, before kill returns. + cancelDuringCleanup = + prompt === "COMPLETE_THEN_HANG" + ? () => controller.abort(cancellation) + : undefined; + const deadline = new Promise((_, reject) => { + timeout = setTimeout( + () => reject(new Error(`${prompt} worker did not settle`)), + 10_000, + ); + }); + const execution = Promise.race([ + run(controller, () => { + if (prompt === "BLOCK_AFTER_START") controller.abort(cancellation); + }), + deadline, + ]); + if (prompt === "BLOCK_AFTER_START") { + await assert.rejects( + execution, + (error: Error) => + error?.name === "AbortError" || + /abort|SIGTERM/i.test(error?.message ?? ""), + ); + assert(workerSignal); + assert.equal(workerSignal.aborted, true); + assert.equal(workerSignal.reason, cancellation); + } else { + if (prompt === "FAIL_THEN_HANG") { + await assert.rejects(execution, /fixture worker failed/); + controller.abort(cancellation); + } else { + const result = await execution; + assert.equal(result.threadId, "fixture-thread-id"); + } + assert.equal(controller.signal.aborted, true); + assert(workerSignal); + assert.equal(workerSignal.aborted, false); + } + assert.equal( + (await readdir(fixture.root)).some((name) => name.startsWith("scope-")), + false, + ); + assert.notEqual(workerSignal, controller.signal); + const currentWorker = worker as ChildProcess | undefined; + assert(currentWorker); + assert.equal(currentWorker.killed, true); + if ( + currentWorker.exitCode === null && + currentWorker.signalCode === null + ) { + await Promise.race([ + new Promise((resolve) => + worker!.once("close", () => resolve()), + ), + deadline, + ]); + } + clearTimeout(timeout); + cancelDuringCleanup = undefined; + } + } finally { + clearTimeout(timeout); + cancelDuringCleanup = undefined; + if (worker && worker.exitCode === null && worker.signalCode === null) + worker.kill("SIGKILL"); + childProcess.spawn = originalSpawn; + syncBuiltinESMExports(); + restoreEnv("CODEX_CLI_PATH", previousPath); + } +} diff --git a/plugins/codex-security/mcp-app/tests/deep_scan_coordinator_fixture.ts b/plugins/codex-security/mcp-app/tests/deep_scan_coordinator_fixture.ts index 48736a0b79..8ad3e85a80 100644 --- a/plugins/codex-security/mcp-app/tests/deep_scan_coordinator_fixture.ts +++ b/plugins/codex-security/mcp-app/tests/deep_scan_coordinator_fixture.ts @@ -90,10 +90,11 @@ export async function fixtureRun( export type TestWorker = PersistedDeepScanWorker & Pick; export type StoreInput = Parameters< - DeepScanStore[Method] + NonNullable >[0]; export class FakeStore { + declare selectFinalization: DeepScanStore["selectFinalization"]; run: DeepScanRunState; declare failProgressAt: number | undefined; constructor(run: DeepScanRunState) { diff --git a/plugins/codex-security/mcp-app/tests/deep_scan_coverage_fixture.mjs b/plugins/codex-security/mcp-app/tests/deep_scan_coverage_fixture.mjs new file mode 100644 index 0000000000..2010574d15 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/deep_scan_coverage_fixture.mjs @@ -0,0 +1,682 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { promisify } from "node:util"; +import { build } from "esbuild"; + +const pluginRoot = fileURLToPath(new URL("../../", import.meta.url)); +const exec = promisify(execFile); +const bundled = await build({ + bundle: true, + stdin: { + contents: [ + 'export { DeepScanCoordinator } from "./src/deep-scan/coordinator.ts";', + 'export { createDeepScanArtifacts } from "./src/deep-scan/artifacts.ts";', + 'export { validateDiscoveryArtifacts } from "./src/deep-scan/artifact-validation.ts";', + 'export { WorkbenchDeepScanStore } from "./src/deep-scan/store.ts";', + 'export { createScanArtifactContext } from "./src/artifact-context.ts";', + 'export { recordCodexSecurityScanDraftViaWorkbench, recordCodexSecurityWorkerScanDraft, saveScanDraftCheckpoint } from "./src/artifact-scan-draft.ts";', + 'export { recordCodexSecurityDeepReduction, getCodexSecurityDeepReducerInputs } from "./src/artifact-deep-reducer.ts";', + ].join("\n"), + resolveDir: path.join(pluginRoot, "mcp-app"), + }, + format: "esm", + platform: "node", + loader: { ".md": "text" }, + write: false, +}); +export async function publishCoverageFixture( + root, + completeness, + { + resume = false, + continueAfterResume = false, + immutableInputs = false, + materialFindings = false, + discardMutableResults = false, + legacyAttempts = false, + splitSeededReducers = false, + selectedRecovery = false, + sourceProvenance, + discoveryReceiptRetry, + } = {}, +) { + const runtimePath = path.join(root, "fixture-runtime.mjs"); + await writeFile(runtimePath, bundled.outputFiles[0].contents); + const { + DeepScanCoordinator, + createDeepScanArtifacts, + validateDiscoveryArtifacts, + WorkbenchDeepScanStore, + createScanArtifactContext, + recordCodexSecurityScanDraftViaWorkbench, + recordCodexSecurityWorkerScanDraft, + recordCodexSecurityDeepReduction, + getCodexSecurityDeepReducerInputs, + saveScanDraftCheckpoint, + } = await import(pathToFileURL(runtimePath).href); + const targetPath = path.join(root, "target"); + const codexHome = path.join(root, "codex-home"); + const scanRoot = path.join(root, "scans"); + const threadId = "coverage-fixture-owner"; + const statuses = + completeness === "partial" + ? ["partial", "complete", "unknown"] + : completeness === "unknown" + ? ["unknown", "complete"] + : ["complete"]; + await mkdir(scanRoot, { mode: 0o700 }); + await mkdir(targetPath, { recursive: true }); + await mkdir(path.join(codexHome, "codex-security"), { recursive: true }); + await writeFile(path.join(targetPath, "source.py"), "# Synthetic source\n"); + await writeFile( + path.join(codexHome, "codex-security", "config.toml"), + `[deep_scan]\nworkers = 1\nsubagents = 0\nstop_after_no_new = ${statuses.length}\nmax_discovery_runs = ${statuses.length}\n`, + ); + const runWorkbench = async (args, input, selectFinalization = false) => { + const script = path.join(pluginRoot, "scripts", "workbench_db.py"); + const pythonArgs = selectFinalization + ? [ + "-c", + "import runpy, sys; script = sys.argv.pop(1); runpy.run_path(script)['main'](select_finalization=True)", + script, + ...args, + ] + : [script, ...args]; + const execution = exec(process.env.PYTHON || "python3", pythonArgs, { + env: { + ...process.env, + CODEX_HOME: codexHome, + CODEX_SECURITY_STATE_DIR: path.join(root, "state"), + }, + }); + if (input !== undefined) execution.child.stdin.end(input); + const { stdout } = await execution; + return JSON.parse(stdout); + }; + const store = new WorkbenchDeepScanStore(runWorkbench); + let run = await store.begin({ targetPath, scope: ".", threadId, scanRoot }); + assert.equal( + run.workflowVersion, + "deep-security-scan/v2", + "new scans use persisted finalization", + ); + if (selectedRecovery) { + ({ run } = await store.claimCoordinator({ scanId: run.scanId, threadId })); + } else { + // Seed an existing v1 run for legacy direct publication and in-memory coverage recovery. + await exec(process.env.PYTHON || "python3", [ + "-c", + [ + "import sqlite3, sys", + "with sqlite3.connect(sys.argv[1]) as db:", + " db.execute(\"UPDATE deep_scan_runs SET workflow_version = 'deep-scan-mcp/v1' WHERE scan_id = ?\", (sys.argv[2],))", + ].join("\n"), + path.join(root, "state", "workbench.sqlite3"), + run.scanId, + ]); + run = await store.get(run.scanId, threadId); + assert.equal(run.workflowVersion, "deep-scan-mcp/v1"); + } + const context = await createScanArtifactContext(run.scanId, runWorkbench, { + requireRunning: true, + }); + const rawSources = new Map(); + const writeReduction = async (context) => { + const inputs = await getCodexSecurityDeepReducerInputs(context); + const sources = [ + ...(inputs.previous?.findings ?? []), + ...inputs.discoveries.flatMap((source) => source.result.findings), + ]; + const findings = []; + if (sources.length) { + const finding = structuredClone(sources[0]); + finding.provenance.sourceFindingIds = [ + ...new Set( + sources.flatMap((source) => source.provenance.sourceFindingIds), + ), + ]; + delete finding.provenance.sourceFindings; + findings.push(finding); + } + await recordCodexSecurityDeepReduction(context, { + scanId: run.scanId, + findings, + }); + }; + const writeDiscovery = async (artifactDir, index) => { + const status = statuses[index]; + const pending = completeness === "partial" && status !== "complete"; + const coverage = { + completeness: status, + surfaces: [ + { + id: "shared-surface", + label: "Archive route", + disposition: pending ? "needs_follow_up" : "no_issue_found", + receiptRefs: ["artifacts/review.md"], + }, + ], + explicitExclusions: [ + { pattern: "vendor/", reason: "External dependency." }, + ], + deferred: pending + ? [ + { + id: "same-id", + candidateId: "candidate-1", + reason: + index === 0 + ? "Verify entry boundaries." + : "Verify symbolic links.", + paths: ["source.py"], + surfaceIds: ["shared-surface"], + }, + ] + : [], + openQuestions: pending + ? [{ question: `Deployment question ${index + 1}.` }] + : [], + }; + if (sourceProvenance !== undefined) { + for (const field of [ + "surfaces", + "explicitExclusions", + "deferred", + "openQuestions", + ]) { + for (const item of coverage[field]) + item.provenance = structuredClone(sourceProvenance); + } + } + await mkdir(path.join(artifactDir, "artifacts"), { recursive: true }); + await writeFile( + path.join(artifactDir, "artifacts", "review.md"), + "Synthetic review evidence.\n", + ); + const resultPath = path.join(artifactDir, "result.json"); + const findings = + materialFindings && index < 2 + ? [ + { + ruleId: "archive-extraction", + identity: { anchor: "archive-destination" }, + title: "Archive entries can escape the destination", + summary: + "Archive extraction requires both entry containment and symbolic-link handling.", + severity: { level: "high" }, + confidence: { + level: "high", + rationale: "Synthetic accepted source evidence.", + }, + taxonomy: { category: "path-traversal", cwe: ["CWE-22"] }, + locations: [{ path: "source.py", startLine: 1, endLine: 1 }], + remediation: materialRemediations[index], + remediationTests: [materialRemediationTests[index]], + provenance: { source: "local_plugin" }, + }, + ] + : []; + const bytes = JSON.stringify({ + scanId: run.scanId, + complete: true, + findings, + coverage, + }); + await writeFile(resultPath, bytes); + rawSources.set(resultPath, bytes); + if (immutableInputs) { + await saveScanDraftCheckpoint( + { root: artifactDir, repoRoot: targetPath, layout: "worker" }, + JSON.parse(bytes), + ); + const head = JSON.parse( + await readFile(path.join(artifactDir, "checkpoint-head.json"), "utf8"), + ); + const acceptedPath = path.join( + artifactDir, + "checkpoints", + head.checkpoint, + ); + rawSources.set(acceptedPath, await readFile(acceptedPath, "utf8")); + return acceptedPath; + } + return resultPath; + }; + if (resume) { + const workers = []; + const seeded = continueAfterResume ? statuses.slice(0, -1) : statuses; + for (const index of seeded.keys()) { + const workerRoot = path.join( + run.scanDir, + "artifacts", + "deep_discovery", + "workers", + `discovery-${String(index + 1).padStart(4, "0")}`, + ); + const artifactDir = path.join(workerRoot, "output"); + const worker = { + id: randomUUID(), + scanId: run.scanId, + kind: "discovery", + promptPath: path.join(workerRoot, "prompt.md"), + artifactDir, + attempt: index === 0 ? 2 : 1, + }; + const writtenPath = await writeDiscovery(artifactDir, index); + const resultManifestPath = discardMutableResults + ? path.join(artifactDir, "result.json") + : writtenPath; + await writeFile(worker.promptPath, "Synthetic discovery prompt.\n"); + for (const status of ["queued", "running", "succeeded"]) { + await store.updateWorker({ + ...worker, + status, + ...(status === "succeeded" ? { resultManifestPath } : {}), + }); + } + workers.push({ ...worker, resultPath: resultManifestPath }); + } + const batches = splitSeededReducers + ? [workers.slice(0, 2), workers.slice(2)] + : [workers]; + let lastReducerId; + let lastReducerReference; + for (const [index, batch] of batches.entries()) { + const label = `dedup-${String(index + 1).padStart(4, "0")}`; + const artifactDir = path.join( + run.scanDir, + "artifacts", + "deep_discovery", + "dedup", + label, + "output", + ); + const promptPath = path.join(path.dirname(artifactDir), "prompt.md"); + await mkdir(artifactDir, { recursive: true }); + await writeFile(promptPath, "Synthetic reducer prompt.\n"); + const id = randomUUID(); + const claimed = await store.claimDedup({ + id, + scanId: run.scanId, + workerIds: batch.map((worker) => worker.id), + artifactDir, + promptPath, + }); + if (discardMutableResults) { + await store.updateWorker({ + id, + scanId: run.scanId, + kind: "dedup", + status: "running", + artifactDir, + promptPath, + attempt: 1, + }); + } + const resultManifestPath = path.join(artifactDir, "result.json"); + // Legacy accepted reducers omitted coverage entirely. + if (materialFindings) { + await writeReduction({ + root: artifactDir, + repoRoot: targetPath, + scanId: run.scanId, + layout: "reducer", + deepReducer: { + scanRoot: run.scanDir, + claimedWorkers: batch.map((worker) => { + const input = claimed.persistedDedupInputs.find( + (input) => + input.dedupWorkerId === id && + input.discoveryWorkerId === worker.id, + ); + return { + ...worker, + resultPath: input.resultManifestPath ?? worker.resultPath, + attempt: input.attempt ?? worker.attempt, + }; + }), + persistSourceCoverage: selectedRecovery, + previousReducerResultPath: claimed.persistedMergeClaims?.find( + (claim) => claim.workerId === id, + )?.previousResultPath, + }, + }); + } else { + await writeFile( + resultManifestPath, + JSON.stringify({ scanId: run.scanId, findings: [] }), + ); + } + rawSources.set( + resultManifestPath, + await readFile(resultManifestPath, "utf8"), + ); + const committed = await store.commitDedup({ + id, + scanId: run.scanId, + newFindings: materialFindings && index === 0 ? 1 : 0, + resultManifestPath, + }); + lastReducerReference = committed.committedMerge.resultManifestPath; + lastReducerId = id; + } + if (legacyAttempts) { + // Migrated discoveries and prior reducers can have frozen claims without attempt rows. + await exec(process.env.PYTHON || "python3", [ + "-c", + [ + "import sqlite3, sys", + "with sqlite3.connect(sys.argv[1]) as db:", + ' db.execute("DELETE FROM deep_scan_attempts WHERE worker_id != ?", (sys.argv[2],))', + ].join("\n"), + path.join(root, "state", "workbench.sqlite3"), + lastReducerId, + ]); + } + run = await store.get(run.scanId, threadId); + if (discardMutableResults) { + for (const worker of run.persistedWorkers) { + const acceptedPath = + worker.acceptedResultPath ?? + run.persistedDedupInputs.find( + (input) => input.discoveryWorkerId === worker.id, + )?.resultManifestPath ?? + run.persistedMergeClaims.find( + (claim) => claim.previousWorkerId === worker.id, + )?.previousResultPath; + assert.ok(acceptedPath, "the real store retains an accepted reference"); + assert.notEqual(acceptedPath, worker.resultManifestPath); + rawSources.set(acceptedPath, await readFile(acceptedPath, "utf8")); + rawSources.delete(worker.resultManifestPath); + await rm(worker.resultManifestPath); + } + } + if (selectedRecovery) { + run = await store.selectFinalization({ + scanId: run.scanId, + reason: "capped", + manifestPath: path.join(run.scanDir, "scan-manifest.json"), + resultPath: lastReducerReference, + omittedWorkerIds: [], + }); + } + } + let discoveryCalls = 0; + let receiptRetryStarted = false; + const retryReceipts = new Map(); + const executor = { + async run(request) { + assert.equal( + resume && !continueAfterResume, + false, + "accepted legacy sources should resume without new model work", + ); + const thread = request.resumeThreadId ?? randomUUID(); + await request.onThreadStarted?.(thread); + if (request.kind === "discovery") { + discoveryCalls++; + const index = + Number( + path + .basename(path.dirname(request.artifactContext.root)) + .split("-") + .at(-1), + ) - 1; + if (discoveryReceiptRetry && !receiptRetryStarted) { + receiptRetryStarted = true; + const artifactDir = request.artifactContext.root; + await mkdir(path.join(artifactDir, "artifacts"), { recursive: true }); + await writeFile( + path.join(artifactDir, "artifacts", "old-review.md"), + "Archived review evidence.\n", + ); + await recordCodexSecurityWorkerScanDraft( + { + ...request.artifactContext, + repoRoot: targetPath, + scanId: run.scanId, + }, + { + scanId: run.scanId, + complete: false, + findings: [], + coverage: { + completeness: "partial", + surfaces: [ + { + id: "archived-surface", + label: "Archived boundary", + disposition: "no_issue_found", + receiptRefs: ["artifacts/old-review.md"], + }, + ], + explicitExclusions: [], + deferred: [], + }, + }, + ); + // A returned partial draft fails validation and triggers the real archive/retry path. + return { threadId: thread, finalResponse: "Checkpoint recorded." }; + } + if (!discoveryReceiptRetry && index === 0 && !request.resumeThreadId) + return { + threadId: thread, + finalResponse: "Continue the unfinished audit.", + }; + await writeDiscovery(request.artifactContext.root, index); + if (discoveryReceiptRetry) { + const artifactDir = request.artifactContext.root; + const archived = path.join( + path.dirname(artifactDir), + "attempts", + "attempt-01", + "artifacts", + "old-review.md", + ); + retryReceipts.set(archived, "Archived review evidence.\n"); + if (discoveryReceiptRetry === "collision") { + const current = path.join( + artifactDir, + "artifacts", + "old-review.md", + ); + await writeFile(current, "New retry evidence.\n"); + retryReceipts.set(current, "New retry evidence.\n"); + } + const resultPath = path.join(artifactDir, "result.json"); + const result = JSON.parse(await readFile(resultPath, "utf8")); + await recordCodexSecurityWorkerScanDraft( + { + ...request.artifactContext, + repoRoot: targetPath, + scanId: run.scanId, + }, + result, + ); + rawSources.set(resultPath, await readFile(resultPath, "utf8")); + } + } else { + if (immutableInputs) { + const current = await store.get(run.scanId, threadId); + for (const claimed of request.artifactContext.deepReducer + .claimedWorkers) { + const accepted = current.persistedWorkers.find( + (worker) => worker.id === claimed.id, + ); + assert.equal( + claimed.resultPath, + accepted.acceptedResultPath ?? accepted.resultManifestPath, + "the reducer uses the exact accepted input", + ); + assert.equal( + claimed.artifactDir, + accepted.artifactDir, + "receipts retain their original output owner", + ); + } + } + await writeReduction({ + ...request.artifactContext, + repoRoot: targetPath, + scanId: run.scanId, + }); + } + return { threadId: thread, finalResponse: "Audit finished." }; + }, + }; + let publicationCalls = 0; + const options = { + run, + store, + executor, + pluginRoot, + retryDelaysMs: [1], + onComplete: async (draft, signal, publication) => { + publicationCalls++; + if (selectedRecovery && publicationCalls === 1) + throw new Error("Synthetic selected publication failure"); + await recordCodexSecurityScanDraftViaWorkbench( + context, + draft, + runWorkbench, + signal, + selectedRecovery ? publication : undefined, + ); + }, + }; + const coordinator = new DeepScanCoordinator(options); + coordinator.start(); + let terminal; + if (selectedRecovery) { + await assert.rejects( + coordinator.wait(undefined, 30_000), + /Synthetic selected publication failure/, + ); + const pending = await store.get(run.scanId, threadId); + assert.equal(pending.status, "running"); + assert.deepEqual(pending.finalizationInput, run.finalizationInput); + const worker = pending.persistedWorkers.find( + (worker) => worker.kind === "discovery", + ); + const rejected = { + scanId: run.scanId, + complete: false, + findings: [], + coverage: { + completeness: "complete", + surfaces: [], + explicitExclusions: [], + deferred: [], + }, + }; + await saveScanDraftCheckpoint( + { root: worker.artifactDir, repoRoot: targetPath, layout: "worker" }, + rejected, + ); + const replacement = path.join(worker.artifactDir, "result.json"); + await writeFile(replacement, JSON.stringify(rejected)); + await assert.rejects( + validateDiscoveryArtifacts( + createDeepScanArtifacts(run.scanDir), + replacement, + run.scanId, + ), + /only a checkpoint/, + ); + const headPath = path.join(worker.artifactDir, "checkpoint-head.json"); + const head = JSON.parse(await readFile(headPath, "utf8")); + for (const file of [ + replacement, + headPath, + path.join(worker.artifactDir, "checkpoints", head.checkpoint), + ]) { + rawSources.set(file, await readFile(file, "utf8")); + } + const restarted = new DeepScanCoordinator({ ...options, run: pending }); + restarted.start(); + terminal = await restarted.wait(undefined, 30_000); + assert.deepEqual(terminal.finalizationInput, run.finalizationInput); + assert.equal(publicationCalls, 2); + } else { + terminal = await coordinator.wait(undefined, 30_000); + } + assert.equal(terminal?.status, "succeeded", terminal?.error); + assert.equal( + terminal.noNewStreak, + materialFindings + ? resume && !continueAfterResume && !splitSeededReducers + ? 0 + : 1 + : statuses.length, + "source coverage must not change stopping policy", + ); + assert.equal( + discoveryCalls, + resume ? (continueAfterResume ? 1 : 0) : statuses.length + 1, + ); + const accepted = await store.get(run.scanId, threadId); + for (const worker of accepted.persistedWorkers.filter( + (worker) => worker.kind === "dedup", + )) { + const resultPath = + worker.acceptedResultPath ?? + accepted.persistedMergeClaims.find( + (claim) => claim.previousWorkerId === worker.id, + )?.previousResultPath ?? + worker.resultManifestPath; + const result = JSON.parse(await readFile(resultPath, "utf8")); + assert.equal( + Object.hasOwn(result, "sourceCoverage"), + selectedRecovery, + "coverage persistence follows the accepted workflow version", + ); + if (!rawSources.has(worker.resultManifestPath)) { + for (const name of await readdir( + path.join(worker.artifactDir, "checkpoints"), + )) { + const checkpoint = JSON.parse( + await readFile( + path.join(worker.artifactDir, "checkpoints", name), + "utf8", + ), + ); + assert.equal( + Object.hasOwn(checkpoint, "sourceCoverage"), + selectedRecovery, + "checkpoint coverage follows the accepted workflow version", + ); + } + } + } + await runWorkbench(["complete-scan", "--scan-id", run.scanId]); + for (const [file, bytes] of rawSources) + assert.equal(await readFile(file, "utf8"), bytes); + for (const [file, bytes] of retryReceipts) + assert.equal(await readFile(file, "utf8"), bytes); + return { scanDir: run.scanDir, threadId, terminal }; +} + +export const materialRemediations = [ + "Check the destination before writing the archive entry.", + "Reject symbolic links before opening the destination.", +]; +export const materialRemediationTests = [ + "Reject an archive entry outside the destination.", + "Reject a symbolic link inside the destination.", +]; + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + const result = await publishCoverageFixture( + process.argv[2], + process.argv[3], + { + resume: process.argv[4] === "true", + continueAfterResume: process.argv[5] === "true", + }, + ); + process.stdout.write(JSON.stringify(result)); +} diff --git a/plugins/codex-security/mcp-app/tests/deep_scan_home_case.ts b/plugins/codex-security/mcp-app/tests/deep_scan_home_case.ts new file mode 100644 index 0000000000..544112ae87 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/deep_scan_home_case.ts @@ -0,0 +1,188 @@ +import assert from "node:assert/strict"; +import { mkdir, realpath, symlink, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { assertConfigOverrides } from "./assertions.ts"; +import { readJson } from "./support/json.ts"; +import type { DeepWorkerParentSandbox } from "../src/deep-scan/parent-sandbox.js"; + +type Fixture = { root: string; executablePath: string; markerPath: string }; + +export async function testHome( + Executor: typeof import("../src/deep-scan/executor.js").CodexSdkWorkerExecutor, + capture: typeof import("../src/deep-scan/recovery-settings.js").captureDeepScanExecutionSettings, + restore: typeof import("../src/deep-scan/recovery-settings.js").restoredDeepScanWorkerSettings, + load: typeof import("../src/deep-scan/recovery-settings.js").loadDeepScanExecutionSettings, + withFixture: ( + action: ( + fixture: Fixture, + promptPath: string, + workingDirectory: string, + ) => Promise, + ) => Promise, + sandbox: DeepWorkerParentSandbox, +) { + return withFixture(async (fixture, promptPath, workingDirectory) => { + const home = path.join(fixture.root, "selected-home"); + const child = path.join(home, "child"); + const link = path.join(fixture.root, "home-link"); + await mkdir(child, { recursive: true }); + await symlink(child, link, "dir"); + await writeFile( + path.join(home, "config.toml"), + 'model_reasoning_summary = "none"\n', + ); + await writeFile( + path.join(fixture.root, "config.toml"), + 'model_reasoning_summary = "detailed"\n', + ); + await writeFile(promptPath, "fixture captured home settings\n"); + + for (const configuredHome of [ + `${path.relative(process.cwd(), link)}/..`, + `${link}/..`, + ]) { + const captured = await capture({}, sandbox, { + CODEX_CLI_PATH: fixture.executablePath, + CODEX_HOME: configuredHome, + }); + assert.equal(captured.reasoningSummary, "none"); + assert.equal(await realpath(captured.codexHome), await realpath(home)); + for (const saved of [captured, JSON.parse(JSON.stringify(captured))]) { + const executor = new Executor( + restore(saved, sandbox, () => ({ + ...process.env, + CODEX_HOME: fixture.root, + })), + ); + for (const kind of ["discovery", "dedup"] as const) { + for (const resumeThreadId of [undefined, "fixture-home-resume"]) { + await executor.run({ + promptPath, + workingDirectory, + kind, + resumeThreadId, + subagents: 0, + signal: new AbortController().signal, + }); + const invocation = await readJson(fixture.markerPath); + assert.equal( + await realpath(invocation.codexHome), + await realpath(home), + ); + assertConfigOverrides(invocation.argv, { + model_reasoning_summary: "none", + }); + assert.equal( + invocation.argv.includes("resume"), + resumeThreadId !== undefined, + ); + } + } + } + } + + await writeFile(path.join(home, "config.toml"), ""); + const sessions = path.join(home, "sessions"); + await mkdir(sessions); + const owner = { + threadId: "fixture-history-owner", + turnId: "history-turn", + startedAt: "2026-01-01T00:01:00Z", + }; + await writeFile( + path.join(sessions, "history.jsonl"), + [ + { + type: "session_meta", + timestamp: "2026-01-01T00:00:00Z", + payload: { + id: owner.threadId, + cli_version: "0.162.0", + model_provider: "openai", + }, + }, + { + type: "event_msg", + timestamp: "2026-01-01T00:00:01Z", + payload: { + type: "thread_settings_applied", + thread_id: owner.threadId, + thread_settings: { + model: "fixture-history-model", + model_provider_id: "openai", + reasoning_effort: "high", + reasoning_summary: "concise", + service_tier: "flex", + }, + }, + }, + ] + .map((value) => JSON.stringify(value)) + .join("\n") + "\n", + ); + const captured = await capture({ usageOwner: owner }, sandbox, { + CODEX_CLI_PATH: fixture.executablePath, + CODEX_HOME: `${link}/..`, + }); + const history = { + model: "fixture-history-model", + modelProvider: "openai", + reasoningEffort: "high", + reasoningSummary: "concise", + serviceTier: "flex", + }; + for (const [key, value] of Object.entries(history)) + assert.equal(captured[key as keyof typeof captured], value); + const snapshot = JSON.parse( + JSON.stringify({ + version: 1, + settings: { + codexPath: captured.codexPath, + codexHome: captured.codexHome, + parentSandbox: sandbox, + }, + }), + ); + const recovered = await load(fixture.root, { + usageOwner: owner, + createdAt: owner.startedAt, + executionSettings: snapshot, + }); + for (const saved of [captured, recovered]) { + const executor = new Executor( + restore(saved, sandbox, () => ({ + ...process.env, + CODEX_HOME: fixture.root, + })), + ); + for (const kind of ["discovery", "dedup"] as const) { + for (const resumeThreadId of [undefined, "fixture-history-resume"]) { + await executor.run({ + promptPath, + workingDirectory, + kind, + resumeThreadId, + subagents: 0, + signal: new AbortController().signal, + }); + const invocation = await readJson(fixture.markerPath); + assert.equal( + await realpath(invocation.codexHome), + await realpath(home), + ); + assertConfigOverrides(invocation.argv, { + model: history.model, + model_provider: history.modelProvider, + model_reasoning_effort: history.reasoningEffort, + model_reasoning_summary: history.reasoningSummary, + service_tier: history.serviceTier, + }); + assert.equal( + invocation.argv.includes("resume"), + resumeThreadId !== undefined, + ); + } + } + } + }); +} diff --git a/plugins/codex-security/mcp-app/tests/deep_scan_lifecycle_cases.ts b/plugins/codex-security/mcp-app/tests/deep_scan_lifecycle_cases.ts index 58f7e856f9..3bf1635043 100644 --- a/plugins/codex-security/mcp-app/tests/deep_scan_lifecycle_cases.ts +++ b/plugins/codex-security/mcp-app/tests/deep_scan_lifecycle_cases.ts @@ -40,6 +40,8 @@ export async function testDeepScanLifecycle({ }; const errors = []; for (const test of [ + resumedSelectedCancellationWaitsForCleanup, + resumedSelectedFailureWaitsForCleanup, lateCancellationWaitsForPersistence, failedParentCancellationStopsActiveWorker, canceledPublicationWaitsForHeartbeat, @@ -63,6 +65,109 @@ export async function testDeepScanLifecycle({ if (errors.length) throw new AggregateError(errors, "Deep Scan lifecycle regressions"); + async function resumedSelectedCancellationWaitsForCleanup( + failPublication = false, + ) { + const fixture = await fixtureRun(config); + fixture.run.workflowVersion = "deep-security-scan/v2"; + fixture.run.finalizationInput = { + version: 1, + resultPath: null, + resultSha256: null, + terminalReason: "capped", + omittedWorkerIds: [], + selectedAt: "2026-01-01T00:00:00Z", + }; + const store = new FakeStore(fixture.run); + const publishing = Promise.withResolvers(); + const publishRelease = Promise.withResolvers(); + const admission = Promise.withResolvers(); + const admissionRelease = Promise.withResolvers(); + const publicationReturned = Promise.withResolvers(); + const preserving = Promise.withResolvers(); + const preserveRelease = Promise.withResolvers(); + store.finish = async () => { + store.run.status = "succeeded"; + return structuredClone(store.run); + }; + const coordinator = createCoordinator(fixture, store, new FakeExecutor(), { + threadId: "fixture-owner", + onComplete: async (_draft: unknown, signal: AbortSignal) => { + publishing.resolve(); + await publishRelease.promise; + signal.throwIfAborted(); + publicationReturned.resolve(); + if (failPublication) + throw new Error("synthetic selected publication failure"); + }, + onStopped: async () => { + preserving.resolve(); + await preserveRelease.promise; + }, + }); + let settledStatus: string | undefined; + let cancellationSettled = false; + const settled = coordinator.settled().then( + (result: DeepScanRunState) => { + settledStatus = result.status; + return result; + }, + (error: unknown) => { + settledStatus = "rejected"; + throw error; + }, + ); + void settled.catch(() => {}); + coordinator.start(); + await publishing.promise; + const cancellation = coordinator + .cancelAfterPersistence( + "cancel resumed selection", + async () => { + store.run.status = "canceled"; + }, + async () => { + admission.resolve(); + await admissionRelease.promise; + return { status: "running" }; + }, + ) + .then((result: DeepScanRunState) => { + cancellationSettled = true; + return result; + }); + void cancellation.catch(() => {}); + try { + await admission.promise; + publishRelease.resolve(); + await publicationReturned.promise; + await new Promise(setImmediate); + admissionRelease.resolve(); + await preserving.promise; + assert.equal( + settledStatus, + undefined, + "terminal waiters include stopped-result publication", + ); + assert.equal( + cancellationSettled, + false, + "cancellation joins common cleanup", + ); + } finally { + publishRelease.resolve(); + admissionRelease.resolve(); + preserveRelease.resolve(); + await Promise.allSettled([settled, cancellation]); + } + assert.equal((await settled).status, "canceled"); + assert.equal((await cancellation).status, "canceled"); + } + + async function resumedSelectedFailureWaitsForCleanup() { + await resumedSelectedCancellationWaitsForCleanup(true); + } + async function failedParentCancellationStopsActiveWorker() { for (const terminalReadFails of [false, true]) { const fixture = await fixtureRun(config); @@ -231,6 +336,9 @@ with tempfile.TemporaryDirectory(prefix="deep-scan-completion-contract-") as roo ), }, loader: { ".md": "text" }, + banner: { + js: `import { createRequire as fixtureCreateRequire } from "node:module"; const require = fixtureCreateRequire(${JSON.stringify(new URL("../server.ts", import.meta.url).href)});`, + }, }); const server = createCodexSecurityServer(); const cancel = diff --git a/plugins/codex-security/mcp-app/tests/deep_scan_private_profile_cases.ts b/plugins/codex-security/mcp-app/tests/deep_scan_private_profile_cases.ts new file mode 100644 index 0000000000..abeae8285d --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/deep_scan_private_profile_cases.ts @@ -0,0 +1,262 @@ +import assert from "node:assert/strict"; +import childProcess, { type SpawnOptions } from "node:child_process"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { syncBuiltinESMExports } from "node:module"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { parse as parseToml, stringify as stringifyToml } from "smol-toml"; +import { assertConfigOverrides } from "./assertions.ts"; +import { importModule } from "./import-module.ts"; +import { readJson } from "./support/json.ts"; +import type { CodexWorkerRequest } from "../src/deep-scan/types.js"; +import type { fakeCodexFixture as createFixture } from "./test_deep_scan_executor.ts"; + +interface Dependencies { + CodexSdkWorkerExecutor: typeof import("../src/deep-scan/executor.js").CodexSdkWorkerExecutor; + captureDeepScanExecutionSettings: typeof import("../src/deep-scan/recovery-settings.js").captureDeepScanExecutionSettings; + restoredDeepScanWorkerSettings: typeof import("../src/deep-scan/recovery-settings.js").restoredDeepScanWorkerSettings; + fakeCodexFixture: typeof createFixture; + workerRequest: ( + promptPath: string, + workingDirectory: string, + options?: Partial, + ) => CodexWorkerRequest; + emptyWorkerPermissionProfile: NonNullable< + Parameters[0] + >; +} + +export async function testRestoredPrivateProfileHomes({ + CodexSdkWorkerExecutor, + captureDeepScanExecutionSettings, + restoredDeepScanWorkerSettings, + fakeCodexFixture, + workerRequest, + emptyWorkerPermissionProfile, +}: Dependencies) { + const providerSource = new URL( + "../../../../sdk/typescript/src/provider-profile.ts", + import.meta.url, + ); + const { createProviderProfile } = await importModule({ + entryPoints: [fileURLToPath(providerSource)], + define: { "import.meta.url": JSON.stringify(providerSource.href) }, + banner: { + js: `import { createRequire as profileRequire } from "node:module"; const require = profileRequire(${JSON.stringify(providerSource.href)});`, + }, + }); + const fixture = await fakeCodexFixture(); + const owners = []; + for (const name of ["same", "first", "second"]) { + const originalHome = path.join(fixture.root, name, "original"); + const currentHome = + name === "same" ? originalHome : path.join(fixture.root, name, "current"); + await mkdir(originalHome, { recursive: true }); + await mkdir(currentHome, { recursive: true }); + const provider = + name === "first" + ? "openrouter" + : name === "second" + ? "fireworks" + : "fixture-provider"; + const config = { + model: "fixture-model", + model_provider: provider, + model_providers: { + [provider]: { + name: "Fixture", + base_url: `https://${name}.example.invalid/v1`, + env_key: `SYNTHETIC_${name.toUpperCase()}_KEY`, + wire_api: "responses", + requires_openai_auth: true, + http_headers: { Authorization: `synthetic-${name}-profile-header` }, + }, + }, + }; + const configPath = path.join(originalHome, "config.toml"); + await writeFile(configPath, stringifyToml(config)); + const sandbox = { + filesystemDenies: [...new Set([originalHome, currentHome])], + }; + const saved = await captureDeepScanExecutionSettings( + { model: config.model }, + sandbox, + { + CODEX_HOME: originalHome, + CODEX_CLI_PATH: process.execPath, + }, + ); + const current = await createProviderProfile(currentHome, config); + const workerConfigPath = path.join(currentHome, "workers.toml"); + await writeFile( + workerConfigPath, + stringifyToml({ worker_runtime: { native_profile: current.name } }), + ); + const promptPath = path.join(fixture.root, name, "prompt.md"); + await writeFile( + promptPath, + "NULL_USAGE CAPTURE_SYNTHETIC_OPENAI_AUTH fixture worker\n", + ); + const markerPath = path.join(fixture.root, name, "invocation.json"); + const environment = { + ...process.env, + CODEX_HOME: currentHome, + CODEX_SECURITY_CONFIG_PATH: configPath, + CODEX_SECURITY_DEEP_SCAN_CONFIG_PATH: workerConfigPath, + CODEX_CLI_PATH: process.execPath, + CODEX_API_KEY: `synthetic-${name}-worker-key`, + FAKE_CODEX_MARKER: markerPath, + FAKE_CODEX_PREFLIGHT_MARKER: path.join( + fixture.root, + name, + "preflight.json", + ), + FAKE_CODEX_PREFLIGHT_PROFILE: JSON.stringify({ + extends: ":read-only", + filesystem: Object.fromEntries([ + [":root", "read"], + ...sandbox.filesystemDenies.map((value) => [value, "deny"]), + ]), + network: { enabled: false }, + }), + }; + const recreated: { + name: string; + path: string; + cleanup(): Promise; + }[] = []; + const executor = new CodexSdkWorkerExecutor({ + ...restoredDeepScanWorkerSettings(saved, sandbox, () => environment), + createProviderProfile: async ( + home: string, + profileConfig: Record, + ) => { + const profile = await createProviderProfile(home, profileConfig); + recreated.push(profile); + return profile; + }, + }); + owners.push({ + name, + provider, + config, + originalHome, + currentHome, + current, + promptPath, + markerPath, + environment, + recreated, + executor, + }); + } + const originalSpawn = childProcess.spawn; + childProcess.spawn = (( + command: string, + args: readonly string[] = [], + options: SpawnOptions = {}, + ) => + originalSpawn( + command, + command === process.execPath || + command === path.win32.toNamespacedPath(process.execPath) + ? [fixture.executablePath, ...args] + : args, + options, + )) as typeof childProcess.spawn; + syncBuiltinESMExports(); + try { + for (const kind of ["discovery", "dedup"] as const) { + const launches: PromiseSettledResult[] = await Promise.allSettled( + owners.map(async (owner) => { + const request = workerRequest(owner.promptPath, fixture.root, { + kind, + ...(kind === "dedup" + ? { resumeThreadId: `fixture-resumed-${owner.name}` } + : {}), + }); + const result = await owner.executor.run(request); + assert.equal( + result.threadId, + kind === "dedup" + ? `fixture-resumed-${owner.name}` + : "fixture-thread-id", + ); + const child = await readJson(owner.markerPath); + assert.equal(child.codexHome, owner.originalHome); + const profile = parseToml(child.profileContents); + assert.deepEqual( + JSON.parse(JSON.stringify(profile.model_providers)), + owner.config.model_providers, + ); + const preflight = await readJson( + owner.environment.FAKE_CODEX_PREFLIGHT_MARKER, + ); + assertConfigOverrides(child.argv, { + [`model_providers.${owner.provider}.base_url`]: undefined, + [`model_providers.${owner.provider}.env_key`]: undefined, + }); + assertConfigOverrides(preflight.argv, { + [`model_providers.${owner.provider}.base_url`]: undefined, + [`model_providers.${owner.provider}.env_key`]: undefined, + }); + assert.match( + child.profileContents, + new RegExp(`synthetic-${owner.name}-profile-header`), + ); + assert.equal( + child.argv + .join(" ") + .includes(`synthetic-${owner.name}-profile-header`), + false, + ); + assert.equal( + child.openaiAuthentication.CODEX_API_KEY, + `synthetic-${owner.name}-worker-key`, + ); + for (const profile of owner.recreated) + await assert.rejects(readFile(profile.path), { code: "ENOENT" }); + await readFile(owner.current.path); + }), + ); + assert.deepEqual( + launches.flatMap((launch) => + launch.status === "rejected" ? [String(launch.reason)] : [], + ), + [], + `native provider selections survive ${kind} child launches`, + ); + } + assert.equal(owners[0].recreated.length, 0); + assert.equal( + new Set( + owners.flatMap((owner) => + owner.recreated.map((profile) => profile.name), + ), + ).size, + 4, + ); + const owner = owners[1]; + const acceptedProfile = owner.environment.FAKE_CODEX_PREFLIGHT_PROFILE; + owner.environment.FAKE_CODEX_PREFLIGHT_PROFILE = JSON.stringify({ + ...emptyWorkerPermissionProfile, + network: { enabled: true }, + }); + await assert.rejects( + owner.executor.run(workerRequest(owner.promptPath, fixture.root)), + ); + for (const profile of owner.recreated) + await assert.rejects(readFile(profile.path), { code: "ENOENT" }); + owner.environment.FAKE_CODEX_PREFLIGHT_PROFILE = acceptedProfile; + await writeFile(owner.promptPath, "CONFIG_ERROR\n"); + await assert.rejects( + owner.executor.run(workerRequest(owner.promptPath, fixture.root)), + ); + for (const profile of owner.recreated) + await assert.rejects(readFile(profile.path), { code: "ENOENT" }); + } finally { + childProcess.spawn = originalSpawn; + syncBuiltinESMExports(); + await Promise.all(owners.map((owner) => owner.current.cleanup())); + } +} diff --git a/plugins/codex-security/mcp-app/tests/deep_scan_reconstructed_cases.ts b/plugins/codex-security/mcp-app/tests/deep_scan_reconstructed_cases.ts new file mode 100644 index 0000000000..92a429cb19 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/deep_scan_reconstructed_cases.ts @@ -0,0 +1,613 @@ +import assert from "node:assert/strict"; +import childProcess, { spawnSync, type SpawnOptions } from "node:child_process"; +import { + copyFile, + mkdir, + readFile, + realpath, + rm, + symlink, + writeFile, +} from "node:fs/promises"; +import { syncBuiltinESMExports } from "node:module"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { parse as parseToml } from "smol-toml"; +import { + assertConfigOverrides, + assertFlagPair, + assertReadOnlyWorkerPolicy, + assertWorkerSubagentPolicy, + workerPermissionProfileOverride, +} from "./assertions.ts"; +import type { DeepWorkerParentSandbox } from "../src/deep-scan/parent-sandbox.js"; +import type { fakeCodexFixture as createFixture } from "./test_deep_scan_executor.ts"; + +interface Dependencies { + CodexSdkWorkerExecutor: typeof import("../src/deep-scan/executor.js").CodexSdkWorkerExecutor; + WorkbenchDeepScanStore: typeof import("../src/deep-scan/store.js").WorkbenchDeepScanStore; + captureDeepScanExecutionSettings: typeof import("../src/deep-scan/recovery-settings.js").captureDeepScanExecutionSettings; + loadDeepScanExecutionSettings: typeof import("../src/deep-scan/recovery-settings.js").loadDeepScanExecutionSettings; + restoredDeepScanWorkerSettings: typeof import("../src/deep-scan/recovery-settings.js").restoredDeepScanWorkerSettings; + fakeCodexFixture: typeof createFixture; + trustedParentSandboxWithDenials: DeepWorkerParentSandbox; + deniedWorkerPermissionProfile: NonNullable< + Parameters[0] + >; + restoreEnv: (name: string, value: string | undefined) => void; +} + +export async function testReconstructedWorkers({ + CodexSdkWorkerExecutor, + WorkbenchDeepScanStore, + captureDeepScanExecutionSettings, + loadDeepScanExecutionSettings, + restoredDeepScanWorkerSettings, + fakeCodexFixture, + trustedParentSandboxWithDenials, + deniedWorkerPermissionProfile, + restoreEnv, +}: Dependencies) { + const launchFailures: string[] = []; + const previousMarker = process.env.FAKE_CODEX_MARKER; + const originalSpawn = childProcess.spawn; + const scans: Awaited>[] = []; + async function prepareReconstructedScan(name: string) { + const uncappedSandbox = { + filesystemDenies: trustedParentSandboxWithDenials.filesystemDenies, + literalFilesystemDenies: + trustedParentSandboxWithDenials.literalFilesystemDenies, + }; + const currentParentSandbox = + name === "first" ? uncappedSandbox : trustedParentSandboxWithDenials; + const filesystem = { ...deniedWorkerPermissionProfile.filesystem }; + delete filesystem["glob_scan_max_depth"]; + const fixture = await fakeCodexFixture({ + ...deniedWorkerPermissionProfile, + filesystem, + }); + const codexHome = path.join(fixture.root, "home"); + const configPath = path.join(fixture.root, "scan config.toml"); + const promptPath = path.join(fixture.root, "prompt.md"); + await mkdir(codexHome); + const config: Record = { + model: `fixture-${name}-inherited`, + model_provider: name === "first" ? "openrouter" : "amazon-bedrock", + model_reasoning_effort: "medium", + model_reasoning_summary: "concise", + service_tier: name === "first" ? "default" : "fast", + }; + await writeFile( + configPath, + (name === "second" + ? 'model_provider = "openai"\nprofile = "cloud.production"\n' + : "") + + Object.entries(config) + .filter( + ([key]) => + (name !== "second" || key !== "model_provider") && + (name !== "first" || + ![ + "model_provider", + "model_reasoning_summary", + "service_tier", + ].includes(key)), + ) + .map(([key, value]) => `${key} = ${JSON.stringify(value)}\n`) + .join("") + + (name === "second" + ? '[profiles."cloud.production"]\nmodel_provider = "amazon-bedrock"\n[model_providers.amazon-bedrock.aws]\nregion = "us-west-2"\nprofile = "fixture-profile"\n' + : "") + + `[codex_security]\ncyber_access_program = "${name === "first" ? "daybreak_blue" : "standard"}"\n[features]\napi_key_cyber_access_programs = ${name === "first"}\napi_key_model_discovery = ${name !== "first"}\n`, + ); + const providerKeys = + name === "first" + ? ["OPENROUTER_API_KEY"] + : ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]; + await writeFile( + promptPath, + "CAPTURE_SYNTHETIC_OPENAI_AUTH CAPTURE_SYNTHETIC_PROVIDER_AUTH NULL_USAGE\n", + ); + const executable = path.join( + fixture.root, + process.platform === "win32" ? "node.exe" : "node", + ); + // Keep dynamically linked Node beside its libraries on Unix. Each scan + // still selects a distinct executable path at the spawn boundary. + if (process.platform === "win32") + await copyFile(process.execPath, executable); + else await symlink(process.execPath, executable); + const codexOptions = { + codexPathOverride: executable, + baseUrl: `https://${name}.example.invalid/v1`, + env: { + PATH: path.dirname(process.execPath), + ...(process.env.SystemRoot + ? { SystemRoot: process.env.SystemRoot } + : {}), + CODEX_HOME: codexHome, + CODEX_SECURITY_CONFIG_PATH: configPath, + CODEX_API_KEY: `synthetic-${name}-credential`, + FAKE_CODEX_MARKER: fixture.markerPath, + FAKE_CODEX_PROVIDER_ENV_KEYS: JSON.stringify(providerKeys), + FAKE_CODEX_SCAN_VALUE: name, + }, + }; + const settings = { + codexOptions, + model: `fixture-${name}-override`, + reasoningEffort: "ultra", + usageOwner: { + threadId: `fixture-${name}-owner`, + turnId: "original-turn", + startedAt: "2026-01-01T00:00:00Z", + }, + parentSandbox: + name === "first" ? trustedParentSandboxWithDenials : uncappedSandbox, + }; + await mkdir(path.join(codexHome, "sessions")); + await writeFile( + path.join(codexHome, "sessions", "owner.jsonl"), + [ + { + type: "session_meta", + timestamp: "2026-01-01T00:00:00Z", + payload: { + id: `fixture-${name}-owner`, + model_provider: config.model_provider, + }, + }, + { + type: "event_msg", + timestamp: "2026-01-01T00:00:00Z", + payload: { + type: "thread_settings_applied", + thread_id: `fixture-${name}-owner`, + thread_settings: { + model: "native-parent-model", + model_provider_id: config.model_provider, + reasoning_effort: "medium", + reasoning_summary: config.model_reasoning_summary, + }, + }, + }, + { + type: "turn_context", + timestamp: "2026-01-01T00:00:01Z", + payload: { + turn_id: "original-turn", + model: "native-parent-model", + effort: "medium", + summary: "none", + }, + }, + { + type: "turn_context", + timestamp: "2026-01-01T00:02:00Z", + payload: { + turn_id: "later-turn", + model: "later-parent-model", + effort: "low", + summary: "detailed", + }, + }, + ] + .map((value) => JSON.stringify(value)) + .join("\n") + "\n", + ); + await writeFile( + path.join(codexHome, "sessions", "observer.jsonl"), + JSON.stringify({ + type: "session_meta", + timestamp: "2026-01-01T00:00:00Z", + payload: { + id: `fixture-${name}-observer`, + model_provider: "observer-provider", + }, + }) + "\n", + ); + const saved = await captureDeepScanExecutionSettings( + settings, + settings.parentSandbox, + { ...codexOptions.env, CODEX_CLI_PATH: executable }, + { + threadId: `fixture-${name}-observer`, + startedAt: "2026-01-01T00:01:00Z", + }, + ); + assert.equal( + saved.nativeServiceTierAbsent, + name === "first" ? true : undefined, + ); + const targetPath = path.join(fixture.root, "target"); + await mkdir(targetPath); + const workbenchPath = fileURLToPath( + new URL("../../scripts/workbench_db.py", import.meta.url), + ); + const runWorkbench = async ( + args: string[], + input?: string, + _selectFinalization?: boolean, + withExecutionSettings?: boolean, + ) => { + const pythonArgs = withExecutionSettings + ? [ + "-c", + "import runpy, sys; script = sys.argv.pop(1); runpy.run_path(script)['main'](with_execution_settings=True)", + workbenchPath, + ...args, + ] + : [workbenchPath, ...args]; + const result = spawnSync( + process.env.PYTHON?.trim() || "python3", + pythonArgs, + { + env: { + ...process.env, + CODEX_HOME: codexHome, + CODEX_SECURITY_STATE_DIR: path.join(fixture.root, "state"), + }, + input, + encoding: "utf8", + timeout: 30_000, + }, + ); + assert.equal(result.status, 0, result.stderr); + return JSON.parse(result.stdout); + }; + const store = new WorkbenchDeepScanStore(runWorkbench); + const beginInput = { + targetPath, + threadId: settings.usageOwner.threadId, + model: settings.model, + reasoningEffort: settings.reasoningEffort, + scanRoot: path.join(fixture.root, "scans"), + }; + const run: import("../src/deep-scan/types.js").DeepScanRunState = + await store.begin({ ...beginInput, executionSettings: saved }); + const recordedScanDir = run.scanDir; + const snapshotPath = path.join( + recordedScanDir, + "artifacts", + "deep_discovery", + "execution-settings.json", + ); + assert.equal(run.workflowVersion, "deep-security-scan/v2"); + assert.deepEqual(run.executionSettings, { version: 1, settings: saved }); + const snapshot = await readFile(snapshotPath, "utf8"); + const expectedProvider = + name === "first" + ? undefined + : { + "amazon-bedrock": { + aws: { region: "us-west-2", profile: "fixture-profile" }, + }, + }; + assert.deepEqual( + JSON.parse(snapshot).settings.providerConfig, + expectedProvider, + "recorded provider selections need no persisted catalog definitions", + ); + const claim = await store.claimCoordinator({ + scanId: run.scanId, + threadId: beginInput.threadId, + }); + assert.equal(claim.acquired, true); + assert.deepEqual( + await loadDeepScanExecutionSettings(recordedScanDir, claim.run), + saved, + ); + assert.deepEqual( + (await store.get(run.scanId, beginInput.threadId)).executionSettings, + run.executionSettings, + ); + const observer = await new WorkbenchDeepScanStore(runWorkbench).begin({ + ...beginInput, + executionSettings: null, + model: "observer-model", + reasoningEffort: "low", + }); + assert.equal(observer.startDisposition, "joined"); + assert.equal(await readFile(snapshotPath, "utf8"), snapshot); + assert.deepEqual(observer.executionSettings, run.executionSettings); + assert.equal(observer.model, settings.model); + assert.equal(snapshot.includes("synthetic-"), false); + const runtimeEnvironment: NodeJS.ProcessEnv = { ...codexOptions.env }; + const restored = restoredDeepScanWorkerSettings( + saved, + currentParentSandbox, + () => runtimeEnvironment, + ); + restored.codexOptions.baseUrl = codexOptions.baseUrl; + return { + name, + fixture, + run, + readRun: async (): Promise< + import("../src/deep-scan/types.js").DeepScanRunState + > => + ( + await new WorkbenchDeepScanStore(runWorkbench).claimCoordinator({ + scanId: run.scanId, + threadId: beginInput.threadId, + }) + ).run, + recordedScanDir, + currentParentSandbox, + config, + configPath, + promptPath, + settings, + runtimeEnvironment, + snapshotPath, + snapshot, + providerKeys, + expectedProvider, + executor: new CodexSdkWorkerExecutor(restored), + }; + } + try { + for (const name of ["first", "second"]) + scans.push(await prepareReconstructedScan(name)); + childProcess.spawn = (( + command: string, + args: readonly string[] = [], + options: SpawnOptions = {}, + ) => { + const scan = scans.find( + (scan) => options?.env?.FAKE_CODEX_MARKER === scan.fixture.markerPath, + ); + return originalSpawn( + command, + scan ? [scan.fixture.executablePath, ...args] : args, + options, + ); + }) as unknown as typeof childProcess.spawn; + syncBuiltinESMExports(); + + for (const phase of [ + "fresh", + "resume", + "reconstructed-fresh", + "reconstructed", + "incomplete", + ]) { + if (phase.startsWith("reconstructed") || phase === "incomplete") { + for (const scan of scans) { + scan.run = await scan.readRun(); + if (phase === "reconstructed-fresh") await rm(scan.configPath); + if (phase.startsWith("reconstructed")) { + const rewritten = JSON.parse(scan.snapshot); + rewritten.settings.codexPath = process.execPath; + rewritten.settings.codexHome = scans.find( + (other) => other !== scan, + )!.settings.codexOptions.env.CODEX_HOME; + await writeFile(scan.snapshotPath, JSON.stringify(rewritten)); + } + if (phase === "incomplete") { + const saved = JSON.parse(scan.snapshot); + for (const key of ["model", "reasoningEffort", "reasoningSummary"]) + delete saved.settings[key]; + if (scan.name === "first") delete saved.settings.modelProvider; + if (scan.name === "first") delete saved.settings.serviceTier; + await writeFile(scan.snapshotPath, JSON.stringify(saved)); + // Emulate an older trusted record with missing optional selections. + scan.run.executionSettings = saved; + } + const snapshotBeforeRead = await readFile(scan.snapshotPath, "utf8"); + const recorded = await loadDeepScanExecutionSettings( + scan.recordedScanDir, + { + ...scan.run, + ...scan.settings, + createdAt: "2026-01-01T00:01:00Z", + }, + ); + const restored = restoredDeepScanWorkerSettings( + recorded, + scan.currentParentSandbox, + () => scan.runtimeEnvironment, + ); + restored.codexOptions.baseUrl = scan.settings.codexOptions.baseUrl; + scan.executor = new CodexSdkWorkerExecutor(restored); + assert.equal( + await readFile(scan.snapshotPath, "utf8"), + snapshotBeforeRead, + "restoring original worker selections must not rewrite saved settings", + ); + } + } + for (const scan of scans) { + scan.runtimeEnvironment.CODEX_API_KEY = `synthetic-${scan.name}-${phase}`; + for (const key of scan.providerKeys) + scan.runtimeEnvironment[key] = + `synthetic-${scan.name}-${phase}-${key}`; + scan.runtimeEnvironment.FAKE_CODEX_SCAN_VALUE = `${scan.name}-${phase}`; + scan.runtimeEnvironment.CODEX_HOME = path.join( + scan.fixture.root, + "observer-home", + ); + scan.runtimeEnvironment.CODEX_CLI_PATH = path.join( + scan.fixture.root, + "observer-codex", + ); + } + for (const kind of ["discovery", "dedup"] as const) { + const launches = await Promise.allSettled( + scans.map(async (scan) => { + const resumeThreadId = ["fresh", "reconstructed-fresh"].includes( + phase, + ) + ? undefined + : `fixture-${scan.name}-resumed`; + const result = await scan.executor.run({ + kind, + promptPath: scan.promptPath, + workingDirectory: scan.fixture.root, + subagents: scan.name === "first" ? 0 : 2, + resumeThreadId, + continuationPrompt: + "CAPTURE_SYNTHETIC_OPENAI_AUTH CAPTURE_SYNTHETIC_PROVIDER_AUTH NULL_USAGE continuation", + signal: new AbortController().signal, + }); + assert.equal( + result.threadId, + resumeThreadId ?? "fixture-thread-id", + ); + const child = JSON.parse( + await readFile(scan.fixture.markerPath, "utf8"), + ); + const preflight = JSON.parse( + await readFile(scan.fixture.preflightMarkerPath, "utf8"), + ); + assert.equal( + await realpath(child.executable), + await realpath(scan.settings.codexOptions.codexPathOverride), + ); + assert.equal( + child.codexCliPath, + scan.settings.codexOptions.codexPathOverride, + ); + assert.equal( + child.codexHome, + scan.settings.codexOptions.env.CODEX_HOME, + ); + assert.equal(preflight.codexHome, child.codexHome); + assert.equal(child.scanValue, `${scan.name}-${phase}`); + assert.equal( + child.configPath, + scan.runtimeEnvironment.CODEX_SECURITY_CONFIG_PATH, + ); + assert.deepEqual(child.openaiAuthentication, { + CODEX_API_KEY: `synthetic-${scan.name}-${phase}`, + }); + assert.deepEqual( + child.providerAuthentication, + Object.fromEntries( + scan.providerKeys.map((key) => [ + key, + `synthetic-${scan.name}-${phase}-${key}`, + ]), + ), + ); + assertFlagPair(child.argv, "--model", scan.settings.model); + assertFlagPair( + child.argv, + "--cyber-access-program", + scan.name === "first" ? "daybreak_blue" : "standard", + ); + assertConfigOverrides(child.argv, { + "features.api_key_cyber_access_programs": scan.name === "first", + "features.api_key_model_discovery": scan.name !== "first", + }); + for (const key of [ + "model_provider", + "model_reasoning_summary", + "service_tier", + ]) { + const override = `${key}=${JSON.stringify(scan.config[key])}`; + assert.equal(child.argv.includes(override), true, override); + assert.equal(preflight.argv.includes(override), true, override); + } + assert.equal( + child.argv.includes('model_reasoning_effort="ultra"'), + true, + ); + assert.equal( + preflight.argv.includes('model_reasoning_effort="ultra"'), + true, + ); + assert.equal( + preflight.argv.includes( + `model=${JSON.stringify(scan.settings.model)}`, + ), + true, + ); + const baseUrl = `openai_base_url=${JSON.stringify(scan.settings.codexOptions.baseUrl)}`; + assert.equal(child.argv.includes(baseUrl), true); + assert.equal(preflight.argv.includes(baseUrl), true); + for (const launch of [child, preflight]) { + const provider = launch.argv.filter((argument: string) => + /^model_providers[.=]/u.test(argument), + ); + if (scan.expectedProvider) { + const providers = parseToml(provider.join("\n")) + .model_providers as Record>; + assert.deepEqual( + JSON.parse(JSON.stringify(providers)), + scan.expectedProvider, + ); + } else { + assert.deepEqual( + provider, + [], + "restored external providers inherit the recorded native home", + ); + } + assert.equal( + workerPermissionProfileOverride(launch.argv).includes( + "glob_scan_max_depth", + ), + false, + "a resumed bounded cap must not truncate an original uncapped deny glob, in either order", + ); + } + assertReadOnlyWorkerPolicy(child.argv); + assertWorkerSubagentPolicy( + child.argv, + scan.name === "first" ? 0 : 2, + ); + assert.equal( + ( + parseToml(workerPermissionProfileOverride(child.argv)) + .permissions as Record< + string, + { filesystem: Record } + > + ).codex_security_deep_scan_worker.filesystem["/repo/.env"], + "deny", + ); + assert.equal( + child.argv.includes("resume"), + resumeThreadId !== undefined, + ); + assert.equal( + child.stdin.includes("continuation"), + resumeThreadId !== undefined, + ); + }), + ); + for (const [index, launch] of launches.entries()) { + if (launch.status === "rejected") { + launchFailures.push( + `${scans[index].name}/${phase}/${kind}: ${launch.reason.message}`, + ); + } + } + } + if (phase === "fresh") { + for (const scan of scans) { + const replacementConfig = path.join( + scan.fixture.root, + "replacement config.toml", + ); + await writeFile( + replacementConfig, + 'model_provider = "changed-provider"\nmodel_reasoning_summary = "detailed"\n[codex_security]\ncyber_access_program = "daybreak_red"\n[features]\napi_key_cyber_access_programs = false\napi_key_model_discovery = false\n', + ); + scan.runtimeEnvironment.CODEX_SECURITY_CONFIG_PATH = + replacementConfig; + } + } + } + assert.deepEqual( + launchFailures, + [], + "every actual preflight and worker must retain the original launch selection", + ); + } finally { + childProcess.spawn = originalSpawn; + syncBuiltinESMExports(); + restoreEnv("FAKE_CODEX_MARKER", previousMarker); + } +} diff --git a/plugins/codex-security/mcp-app/tests/deep_scan_selected_finalization_cases.ts b/plugins/codex-security/mcp-app/tests/deep_scan_selected_finalization_cases.ts new file mode 100644 index 0000000000..7d5aca11af --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/deep_scan_selected_finalization_cases.ts @@ -0,0 +1,220 @@ +import assert from "node:assert/strict"; +import type { + DeepScanRunState, + DeepScanLogEvent, +} from "../src/deep-scan/types.js"; +import { + fixtureRun, + FakeStore, + DeepScanCoordinator, +} from "./deep_scan_coordinator_fixture.ts"; + +export async function testSelectedFinalizationOwner() { + await testCancellationDuringSelectedParentPublication(); + for (const failure of [ + undefined, + "draft", + "finish", + "parent", + "replacement", + ] as const) { + const fixture = await fixtureRun({ + workers: 1, + subagents: 0, + stopAfterNoNew: 1, + maxDiscoveryRuns: 1, + }); + fixture.run.workflowVersion = "deep-security-scan/v2"; + fixture.run.coordinatorGeneration = 3; + fixture.run.finalizationInput = { + version: 1, + resultPath: null, + resultSha256: null, + terminalReason: "capped", + omittedWorkerIds: [], + selectedAt: "2026-01-01T00:00:00Z", + }; + const store = new FakeStore(fixture.run); + const finishStarted = Promise.withResolvers(); + const releaseFinish = Promise.withResolvers(); + const parentStarted = Promise.withResolvers(); + const releaseParent = Promise.withResolvers(); + const calls: string[] = []; + const events: DeepScanLogEvent[] = []; + store.finish = async () => { + calls.push("finish"); + finishStarted.resolve(); + await releaseFinish.promise; + if (failure === "finish") + throw new Error("injected selected finish failure"); + if (failure === "replacement") { + store.run = { + ...store.run, + status: "succeeded", + coordinatorGeneration: 4, + }; + throw new Error("Deep Scan coordinator generation is stale."); + } + store.run = { + ...store.run, + status: "succeeded", + terminalReason: "capped", + }; + return structuredClone(store.run); + }; + const coordinator = new DeepScanCoordinator({ + run: fixture.run, + store, + pluginRoot: fixture.pluginRoot, + executor: { + run: async () => + assert.fail("Selected finalization cannot start model work"), + }, + threadId: "selected-owner", + log: (event: DeepScanLogEvent) => events.push(event), + onComplete: async () => { + calls.push("draft"); + if (failure === "draft") + throw new Error("injected selected draft failure"); + }, + onFinalized: async (run: DeepScanRunState) => { + calls.push("parent"); + assert.equal(run.status, "succeeded"); + assert.equal(run.coordinatorGeneration, 3); + parentStarted.resolve(); + await releaseParent.promise; + if (failure === "parent") + throw new Error("injected selected parent failure"); + }, + }); + // Rejections stay observed even when the user detaches their only waiter. + const settled = coordinator.settled(); + settled.catch(() => {}); + const observer = new AbortController(); + coordinator.start(); + const detached = coordinator.wait(observer.signal); + observer.abort(); + await assert.rejects(detached, { name: "AbortError" }); + if (failure !== "draft") { + await finishStarted.promise; + assert.deepEqual(calls, ["draft", "finish"]); + assert.equal(await coordinator.wait(undefined, 10), undefined); + releaseFinish.resolve(); + } + if (failure === undefined || failure === "parent") { + await parentStarted.promise; + assert.equal( + await coordinator.wait(undefined, 10), + undefined, + "wait includes enclosing completion", + ); + releaseParent.resolve(); + } + if (failure && failure !== "replacement") { + await assert.rejects( + settled, + new RegExp(`injected selected ${failure} failure`), + ); + assert.equal( + store.failureMessages.length, + 0, + "publication errors retain the original selected stop", + ); + assert.equal( + events.some( + (event) => event.event === "coordinator_publication_pending", + ), + true, + ); + assert.equal( + store.run.status, + failure === "parent" ? "succeeded" : "running", + ); + } else { + assert.equal((await settled).status, "succeeded"); + } + assert.deepEqual( + calls, + failure === "draft" + ? ["draft"] + : failure === "finish" || failure === "replacement" + ? ["draft", "finish"] + : ["draft", "finish", "parent"], + ); + assert.deepEqual( + store.run.finalizationInput, + fixture.run.finalizationInput, + ); + } +} + +async function testCancellationDuringSelectedParentPublication() { + const fixture = await fixtureRun({ + workers: 1, + subagents: 0, + stopAfterNoNew: 1, + maxDiscoveryRuns: 1, + }); + fixture.run.workflowVersion = "deep-security-scan/v2"; + fixture.run.finalizationInput = { + version: 1, + resultPath: null, + resultSha256: null, + terminalReason: "capped", + omittedWorkerIds: [], + selectedAt: "2026-01-01T00:00:00Z", + }; + const store = new FakeStore(fixture.run); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + let publicationSignal: AbortSignal | undefined; + let completed = false; + let canceled = false; + const coordinator = new DeepScanCoordinator({ + run: fixture.run, + store, + pluginRoot: fixture.pluginRoot, + threadId: "selected-owner", + executor: { + run: async () => + assert.fail("Selected finalization cannot launch a worker"), + }, + onComplete: async () => {}, + onFinalized: async (_run: DeepScanRunState, signal: AbortSignal) => { + publicationSignal = signal; + entered.resolve(); + await release.promise; + signal.throwIfAborted(); + completed = true; + }, + }); + coordinator.start(); + const settled = coordinator.settled(); + void settled.catch(() => {}); + await entered.promise; + const cancellation = coordinator.cancelAfterPersistence( + "cancel selected parent", + async () => { + if (completed) throw new Error("Only a running scan can be canceled."); + canceled = true; + store.run.status = "canceled"; + }, + async () => ({ status: completed ? "complete" : "running" }), + ); + void cancellation.catch(() => {}); + try { + await new Promise(setImmediate); + assert.equal( + publicationSignal?.aborted, + true, + "cancel aborts pending parent publication after reducer success", + ); + } finally { + release.resolve(); + await Promise.allSettled([settled, cancellation]); + } + assert.equal((await cancellation).status, "canceled"); + assert.equal((await settled).status, "canceled"); + assert.equal(canceled, true); + assert.equal(completed, false); +} diff --git a/plugins/codex-security/mcp-app/tests/fixtures/accepted_source_bank.mjs b/plugins/codex-security/mcp-app/tests/fixtures/accepted_source_bank.mjs new file mode 100644 index 0000000000..0fed15b231 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/fixtures/accepted_source_bank.mjs @@ -0,0 +1,180 @@ +// Synthetic accepted artifacts. Expected fixes are independent of display identities. +export const scanId = "7fc17317-9594-49e0-b06a-d72fd7e14bba"; +export const bankVersion = "accepted-source-bank/v1"; + +export const fixes = { + owner: "Check document ownership before returning document contents.", + path: "Resolve archive paths and reject entries outside the extraction directory.", + sql: "Bind the search term as a SQL parameter.", + html: "HTML-encode the search term before rendering the response.", +}; + +function finding(id, remediation, extra = {}) { + return { + ruleId: "synthetic." + id, + identity: { anchor: id }, + title: "Request boundary " + id, + summary: "A request-controlled value crosses an unchecked boundary.", + severity: { level: "high" }, + confidence: { + level: "high", + rationale: "A synthetic local test reaches the sink.", + }, + taxonomy: { category: "input-validation", cwe: ["CWE-20"] }, + locations: [{ path: "src/routes.py", startLine: 12, endLine: 14 }], + remediation, + provenance: { source: "local_plugin" }, + ...extra, + }; +} + +function coverage(completeness = "complete", deferred = []) { + return { completeness, surfaces: [], explicitExclusions: [], deferred }; +} + +function worker(id, findings, scanCoverage = coverage()) { + return { + id, + result: { scanId, complete: true, findings, coverage: scanCoverage }, + }; +} + +export const workers = [ + worker( + "worker-owner", + [ + finding("owner", fixes.owner, { + summary: + "An authenticated user can read another user's document by changing its ID.", + validation: { + summary: "The ownership check is absent; authentication is required.", + }, + }), + ], + coverage("partial", [ + { + candidateId: "candidate-1", + reason: "Check the alternate document handler.", + paths: ["src/alternate.py"], + }, + ]), + ), + worker( + "worker-path", + [finding("path", fixes.path)], + coverage("partial", [ + { + candidateId: "candidate-1", + reason: "Check symlink extraction separately.", + paths: ["src/archive.py"], + }, + ]), + ), + worker("worker-owner-duplicate", [ + finding("owner-copy", fixes.owner, { + summary: "Document contents may be reachable without authentication.", + severity: { level: "critical" }, + confidence: { + level: "low", + rationale: "Authentication middleware was not examined.", + }, + validation: { + summary: + "The unauthenticated claim is untested; ownership check is absent.", + }, + }), + ]), + worker("worker-bundled", [ + finding("search-bundle", fixes.sql + " " + fixes.html, { + summary: + "The search route interpolates the query into SQL and separately into HTML.", + locations: [{ path: "src/search.py", startLine: 5, endLine: 9 }], + }), + ]), +]; + +export const sourceGroups = { + "worker-owner:0": "owner", + "worker-path:0": "path", + "worker-owner-duplicate:0": "owner", + "worker-bundled:0": "search-bundle", +}; + +export const sourceFixes = { + "worker-owner:0": ["owner"], + "worker-path:0": ["path"], + "worker-owner-duplicate:0": ["owner"], + "worker-bundled:0": ["sql", "html"], +}; + +// This history is distinct from the immutable accepted terminal bank above. +// A newer rejection is authoritative for this logical worker's final result. +export const rejectionHistory = { + workerId: "worker-rejected", + earlier: worker("worker-rejected", [finding("safe-query", fixes.sql)]).result, + latest: worker("worker-rejected", [], { + ...coverage(), + surfaces: [ + { + label: "Search SQL", + disposition: "rejected", + notes: + "The driver binds parameters; the earlier interpolation claim was disproved.", + }, + ], + }).result, +}; + +export function permutations(items) { + if (!items.length) return [[]]; + return items.flatMap((item, index) => + permutations(items.filter((_, i) => i !== index)).map((rest) => [ + item, + ...rest, + ]), + ); +} + +export function partitions(items) { + if (!items.length) return [[]]; + return items.flatMap((_, index) => + partitions(items.slice(index + 1)).map((rest) => [ + items.slice(0, index + 1), + ...rest, + ]), + ); +} + +export function originals(inputs) { + const sources = new Map(); + for (const current of inputs.previous?.findings ?? []) { + for (const source of current.provenance.sourceFindings ?? []) + sources.set(source.id, source.finding); + } + for (const discovery of inputs.discoveries) { + discovery.result.findings.forEach((value, i) => + sources.set(`${discovery.workerId}:${i}`, value), + ); + } + return sources; +} + +// Scripted proposals isolate host reconciliation from model variability. +export function proposal(inputs, groupFor = (id) => sourceGroups[id]) { + const grouped = new Map(); + for (const [id, value] of originals(inputs)) { + const group = groupFor(id); + if (!grouped.has(group)) grouped.set(group, { value, refs: [] }); + grouped.get(group).refs.push(id); + } + return { + scanId, + complete: true, + findings: [...grouped].map(([group, { value, refs }]) => ({ + ...structuredClone(value), + ruleId: "synthetic." + group, + identity: { anchor: group }, + provenance: { source: "local_plugin", sourceFindingIds: refs }, + })), + }; +} diff --git a/plugins/codex-security/mcp-app/tests/fixtures/accepted_source_replay.mjs b/plugins/codex-security/mcp-app/tests/fixtures/accepted_source_replay.mjs new file mode 100644 index 0000000000..26b6a61632 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/fixtures/accepted_source_replay.mjs @@ -0,0 +1,105 @@ +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { build } from "esbuild"; +import { proposal, scanId, workers } from "./accepted_source_bank.mjs"; + +export async function loadReducer() { + const bundled = await build({ + bundle: true, + entryPoints: [ + fileURLToPath( + new URL("../../src/artifact-deep-reducer.ts", import.meta.url), + ), + ], + format: "esm", + platform: "node", + write: false, + }); + return import( + "data:text/javascript;base64," + + Buffer.from(bundled.outputFiles[0].contents).toString("base64") + ); +} + +export async function prepareReplay(root, bank = workers) { + const workerRoot = path.join(root, "artifacts", "deep_discovery", "workers"); + const records = new Map(); + for (const [i, worker] of bank.entries()) { + const output = path.join(workerRoot, worker.id, "output"); + await mkdir(output, { recursive: true }); + const resultPath = path.join(output, "result.json"); + await writeFile(resultPath, JSON.stringify(worker.result) + "\n"); + records.set(worker.id, { + id: worker.id, + resultPath, + completionSequence: i + 1, + }); + } + return records; +} + +export async function runBatch( + reducer, + root, + records, + ids, + index, + previous, + propose = proposal, +) { + const output = path.join( + root, + "artifacts", + "deep_discovery", + "dedup", + `dedup-${index}`, + "output", + ); + await mkdir(output, { recursive: true }); + const context = { + root: output, + repoRoot: root, + scanId, + layout: "reducer", + deepReducer: { + scanRoot: root, + claimedWorkers: ids.map((id) => records.get(id)), + ...(previous ? { previousReducerResultPath: previous } : {}), + }, + }; + const inputs = await reducer.getCodexSecurityDeepReducerInputs(context); + const submitted = propose(inputs); + const receipt = await reducer.recordCodexSecurityDeepReduction( + context, + submitted, + ); + const resultPath = path.join(output, "result.json"); + return { + inputs, + submitted, + receipt, + resultPath, + result: JSON.parse(await readFile(resultPath, "utf8")), + }; +} + +export async function replay(reducer, root, batches, propose = proposal) { + const records = await prepareReplay(root); + const steps = []; + let previous; + for (const [index, batch] of batches.entries()) { + const step = await runBatch( + reducer, + root, + records, + batch, + index, + previous, + propose, + ); + steps.push(step); + previous = step.resultPath; + } + return { steps, result: steps.at(-1).result }; +} diff --git a/plugins/codex-security/mcp-app/tests/import-module.ts b/plugins/codex-security/mcp-app/tests/import-module.ts index efdb75f1fd..a5b0567eb5 100644 --- a/plugins/codex-security/mcp-app/tests/import-module.ts +++ b/plugins/codex-security/mcp-app/tests/import-module.ts @@ -1,10 +1,12 @@ import { build, type BuildOptions } from "esbuild"; +import path from "node:path"; export async function importModule(options: BuildOptions) { const result = await build({ bundle: true, format: "esm", platform: "node", + nodePaths: [path.resolve(import.meta.dirname, "../node_modules")], write: false, ...options, }); diff --git a/plugins/codex-security/mcp-app/tests/legacy-accepted-finding.ts b/plugins/codex-security/mcp-app/tests/legacy-accepted-finding.ts new file mode 100644 index 0000000000..2084e6d8fc --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/legacy-accepted-finding.ts @@ -0,0 +1,70 @@ +import { finding } from "./scan-draft-fixture.ts"; + +/** Finding fields accepted before the current producer contract was narrowed. */ +export function legacyAcceptedFinding( + base = finding("historical-checkpoint", "fixture.py"), +) { + const referenced = { + summary: "Synthetic source reaches the response.", + evidenceRefs: ["legacy-snippet"], + evidence_refs: ["canonical-snippet"], + }; + return { + ...base, + codeEvidence: [ + { + id: "canonical-snippet", + label: "Response", + path: "fixture.py", + startLine: 1, + code: "return value", + explanation: "The synthetic value reaches the response.", + }, + ], + code_evidence: [{ id: "legacy-snippet", code: "return value" }], + rootCause: { + ...referenced, + code: "return value", + language: "python", + }, + root_cause: { ...referenced, code: "return value", language: "python" }, + validation: { ...referenced, status: "confirmed" }, + attackPath: { + ...referenced, + dataflow: { ...referenced }, + dataFlow: { ...referenced }, + data_flow: { ...referenced }, + reachability: { ...referenced }, + }, + }; +} + +/** Older discovery readers admitted these bytes without rewriting the file. */ +export function legacyRawDiscoveryFinding( + base = finding("historical-checkpoint", "fixture.py"), +): ReturnType & Record { + const raw: ReturnType & Record = + legacyAcceptedFinding(base); + const attackPath = raw.attackPath as Record; + for (const detail of [ + raw.rootCause, + raw.root_cause, + raw.validation, + attackPath, + attackPath.dataflow, + attackPath.dataFlow, + attackPath.data_flow, + attackPath.reachability, + ]) { + const section = detail as Record; + for (const field of ["evidenceRefs", "evidence_refs"]) + section[field] = (section[field] as string[])[0]; + } + (raw.code_evidence as unknown[]).push( + { id: "legacy-snippet", code: "duplicate()" }, + { id: "canonical-snippet", code: "duplicate()" }, + { id: "invalid", code: " " }, + null, + ); + return raw; +} diff --git a/plugins/codex-security/mcp-app/tests/scan-draft-recovery-fixture.ts b/plugins/codex-security/mcp-app/tests/scan-draft-recovery-fixture.ts index e73d4f97f2..cda5467e1a 100644 --- a/plugins/codex-security/mcp-app/tests/scan-draft-recovery-fixture.ts +++ b/plugins/codex-security/mcp-app/tests/scan-draft-recovery-fixture.ts @@ -1,4 +1,3 @@ -import { createHash } from "node:crypto"; import { temporaryDirectory } from "./support/temporary-directories.ts"; import { readJson } from "./support/json.ts"; import assert from "node:assert/strict"; @@ -8,76 +7,15 @@ import path from "node:path"; import type { TestContext } from "node:test"; import type { ScanDraftInput } from "../src/artifact-scan-draft.js"; import type { ArtifactContext } from "../src/artifact-io.js"; -import { importModule, importSource } from "./import-module.ts"; +import { importSource } from "./import-module.ts"; export const draftApi = await importSource("../src/artifact-scan-draft.ts", { absWorkingDir: import.meta.dirname, }); -const { - artifactDestination, - replaceArtifactJson, - replaceArtifactText, - saveThreatModelDocument, -} = await importModule({ - stdin: { - contents: - 'export * from "./artifact-io.ts"; export * from "./threat-model-document.ts";', - resolveDir: path.join(import.meta.dirname, "../src"), - }, -}); - -// Unit publication boundary; real lock/binding behavior is covered by the workbench integration suite. -export async function recordCodexSecurityScanDraft( - context: ArtifactContext, - input: unknown, -) { - return draftApi.recordCodexSecurityScanDraftViaWorkbench( - context, - input, - async (args: string[]) => { - const draft = await readJson(args[args.indexOf("--draft-path") + 1]); - const rawContents = await fs.readFile( - args[args.indexOf("--checkpoint-path") + 1], - "utf8", - ); - const checkpoint = JSON.parse(rawContents); - const rawName = - createHash("sha256").update(rawContents).digest("hex") + ".json"; - await replaceArtifactText( - await artifactDestination( - context, - ["checkpoints", rawName], - "raw checkpoint", - ), - rawContents, - ); - const { complete, scope, threatModel } = draft.manifest.scan; - await draftApi.saveScanDraftCheckpoint( - { ...context, layout: "worker" }, - { - scanId: checkpoint.scanId, - ...(complete === undefined ? {} : { complete }), - ...(scope === undefined ? {} : { scope }), - ...(threatModel === undefined ? {} : { threatModel }), - findings: draft.findings.findings, - coverage: draft.coverage, - }, - ); - for (const [key, name] of [ - ["findings", "findings.json"], - ["coverage", "coverage.json"], - ["manifest", "scan-manifest.json"], - ]) { - await replaceArtifactJson( - await artifactDestination(context, [name], "scan draft"), - draft[key], - ); - } - const warning = await saveThreatModelDocument(context, threatModel); - return warning === undefined ? {} : { warnings: [warning] }; - }, - ); -} +export const { recordCodexSecurityScanDraft } = await importSource( + "./scan-draft-writer-fixture.ts", + { absWorkingDir: import.meta.dirname }, +); export const scanId = "7b95abf2-dc04-47a9-9950-53b5c2057f49"; export const claimToken = "19bfba38-0913-4bd7-86ef-134e9a4d9a42"; diff --git a/plugins/codex-security/mcp-app/tests/scan-draft-writer-fixture.ts b/plugins/codex-security/mcp-app/tests/scan-draft-writer-fixture.ts new file mode 100644 index 0000000000..46d28a1c69 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/scan-draft-writer-fixture.ts @@ -0,0 +1,69 @@ +import { createHash } from "node:crypto"; +import { promises as fs } from "node:fs"; +import type { ArtifactContext } from "../src/artifact-io.js"; +import { + artifactDestination, + replaceArtifactJson, + replaceArtifactText, +} from "../src/artifact-io.ts"; +import { + recordCodexSecurityScanDraftViaWorkbench, + type ScanDraftInput, + saveScanDraftCheckpoint, +} from "../src/artifact-scan-draft.ts"; +import { saveThreatModelDocument } from "../src/threat-model-document.ts"; +import { readJson } from "./support/json.ts"; + +// Unit publication boundary; real lock/binding behavior is covered by the workbench integration suite. +export async function recordCodexSecurityScanDraft( + context: ArtifactContext, + input: ScanDraftInput, +) { + return recordCodexSecurityScanDraftViaWorkbench( + context, + input, + async (args: string[]) => { + const draft = await readJson(args[args.indexOf("--draft-path") + 1]); + const rawContents = await fs.readFile( + args[args.indexOf("--checkpoint-path") + 1], + "utf8", + ); + const checkpoint = JSON.parse(rawContents); + const rawName = + createHash("sha256").update(rawContents).digest("hex") + ".json"; + await replaceArtifactText( + await artifactDestination( + context, + ["checkpoints", rawName], + "raw checkpoint", + ), + rawContents, + ); + const { complete, scope, threatModel } = draft.manifest.scan; + const canonicalCheckpoint = { + scanId: checkpoint.scanId, + ...(complete === undefined ? {} : { complete }), + ...(scope === undefined ? {} : { scope }), + ...(threatModel === undefined ? {} : { threatModel }), + findings: draft.findings.findings, + coverage: draft.coverage, + }; + await saveScanDraftCheckpoint( + { ...context, layout: "worker" }, + canonicalCheckpoint, + ); + for (const [key, name] of [ + ["findings", "findings.json"], + ["coverage", "coverage.json"], + ["manifest", "scan-manifest.json"], + ]) { + await replaceArtifactJson( + await artifactDestination(context, [name], "scan draft"), + draft[key], + ); + } + const warning = await saveThreatModelDocument(context, threatModel); + return warning === undefined ? {} : { warnings: [warning] }; + }, + ); +} diff --git a/plugins/codex-security/mcp-app/tests/support/fake-codex-fixture.ts b/plugins/codex-security/mcp-app/tests/support/fake-codex-fixture.ts index 5fde2ee743..23de6c1e41 100644 --- a/plugins/codex-security/mcp-app/tests/support/fake-codex-fixture.ts +++ b/plugins/codex-security/mcp-app/tests/support/fake-codex-fixture.ts @@ -42,7 +42,11 @@ export const emptyWorkerPermissionProfile = { export async function createFakeCodexFixture( temporaryDirectories: ReturnType, - preflightProfile = emptyWorkerPermissionProfile, + preflightProfile: { + extends: string; + filesystem: Record>; + network: { enabled: boolean }; + } = emptyWorkerPermissionProfile, preflightAllowed = true, accountResult: { account: { type: string } | null; diff --git a/plugins/codex-security/mcp-app/tests/support/fake-codex-source.ts b/plugins/codex-security/mcp-app/tests/support/fake-codex-source.ts index 1d1bacc8ea..330d7c6732 100644 --- a/plugins/codex-security/mcp-app/tests/support/fake-codex-source.ts +++ b/plugins/codex-security/mcp-app/tests/support/fake-codex-source.ts @@ -63,6 +63,7 @@ let scopeInput; const profileIndex = process.argv.indexOf('--profile'); const profileContents = profileIndex === -1 ? undefined : readFileSync(join(process.env.CODEX_HOME, process.argv[profileIndex + 1] + '.config.toml'), 'utf8'); const openaiAuthentication = stdin.includes('CAPTURE_SYNTHETIC_OPENAI_AUTH') ? { OPENAI_API_KEY: process.env.OPENAI_API_KEY, CODEX_API_KEY: process.env.CODEX_API_KEY } : undefined; +const providerAuthentication = stdin.includes('CAPTURE_SYNTHETIC_PROVIDER_AUTH') ? Object.fromEntries(JSON.parse(process.env.FAKE_CODEX_PROVIDER_ENV_KEYS).map((name) => [name, process.env[name]])) : undefined; const bedrockAuthentication = stdin.includes('CAPTURE_SYNTHETIC_BEDROCK_AUTH') ? Object.fromEntries(JSON.parse(process.env.FAKE_CODEX_BEDROCK_ENV_KEYS).map((name) => [name, process.env[name]])) : undefined; const runtimeEnvironment = Object.fromEntries(['PATH', 'HOME', 'PYTHON', 'PYTHONUTF8', 'LD_LIBRARY_PATH', 'DYLD_LIBRARY_PATH', 'DYLD_FALLBACK_LIBRARY_PATH', 'CODEX_SECURITY_STATE_DIR', 'RUNNER_TRACKING_ID'].map(name => [name, process.env[name]])); const toolProbe = stdin.includes('CAPTURE_SYNTHETIC_RG') ? spawnSync('rg', ['--version'], { encoding: 'utf8' }) : undefined; @@ -73,7 +74,7 @@ const pythonRuntime = pythonProbe ? JSON.parse(pythonProbe.stdout) : undefined; const knowledgePath = stdin.includes('synthetic worker configuration fixture') ? process.env.CODEX_SECURITY_KNOWLEDGE_BASE : undefined; const knowledgeDocuments = knowledgePath === undefined ? undefined : Object.fromEntries(readdirSync(knowledgePath).map(name => [name, readFileSync(join(knowledgePath, name), 'utf8')])); const deepConfig = process.env.CODEX_SECURITY_DEEP_SCAN_CONFIG_PATH ? readFileSync(process.env.CODEX_SECURITY_DEEP_SCAN_CONFIG_PATH, 'utf8') : undefined; -writeFileSync(process.env.FAKE_CODEX_MARKER, JSON.stringify({ ...(scopeInput === undefined ? {} : { scopeInput }), argv: process.argv.slice(2), stdin, cwd: process.cwd(), knowledgePath, knowledgeDocuments, deepConfig, codexHome: process.env.CODEX_HOME, temporaryEnvironment, cacheDirectory: process.env.XDG_CACHE_HOME, gitEnvironment: Object.fromEntries(['PATH', 'CODEX_SECURITY_GIT', 'GIT_SSH_COMMAND', 'GIT_CONFIG_GLOBAL'].map(name => [name, process.env[name]])), configPath: process.env.CODEX_SECURITY_CONFIG_PATH, deepConfigPath: process.env.CODEX_SECURITY_DEEP_SCAN_CONFIG_PATH, python: process.env.PYTHON, mcpNodePath: process.env.CODEX_MCP_NODE_PATH, bundledTool: toolProbe?.stdout.trim(), pythonPrefix: pythonRuntime?.[0], pythonLibraryPath: pythonRuntime?.[1], runtimeEnvironment, providerKey: process.env.SYNTHETIC_GATEWAY_KEY, providerHeader: process.env.SYNTHETIC_HEADER_VALUE, originator: process.env.CODEX_INTERNAL_ORIGINATOR_OVERRIDE, ...(profileContents === undefined ? {} : { profileContents }), ...(stdin.includes('COMPLETE_THEN_HANG') ? { pid: process.pid } : {}), ...(openaiAuthentication ? { openaiAuthentication } : {}), ...(bedrockAuthentication ? { bedrockAuthentication } : {}) })); +writeFileSync(process.env.FAKE_CODEX_MARKER, JSON.stringify({ ...(scopeInput === undefined ? {} : { scopeInput }), executable: process.execPath, codexCliPath: process.env.CODEX_CLI_PATH, scanValue: process.env.FAKE_CODEX_SCAN_VALUE, argv: process.argv.slice(2), stdin, cwd: process.cwd(), knowledgePath, knowledgeDocuments, codexHome: process.env.CODEX_HOME, temporaryEnvironment, deepConfig, cacheDirectory: process.env.XDG_CACHE_HOME, gitEnvironment: Object.fromEntries(['PATH', 'CODEX_SECURITY_GIT', 'GIT_SSH_COMMAND', 'GIT_CONFIG_GLOBAL'].map(name => [name, process.env[name]])), configPath: process.env.CODEX_SECURITY_CONFIG_PATH, deepConfigPath: process.env.CODEX_SECURITY_DEEP_SCAN_CONFIG_PATH, python: process.env.PYTHON, mcpNodePath: process.env.CODEX_MCP_NODE_PATH, bundledTool: toolProbe?.stdout.trim(), pythonPrefix: pythonRuntime?.[0], pythonLibraryPath: pythonRuntime?.[1], runtimeEnvironment, providerKey: process.env.SYNTHETIC_GATEWAY_KEY, providerHeader: process.env.SYNTHETIC_HEADER_VALUE, originator: process.env.CODEX_INTERNAL_ORIGINATOR_OVERRIDE, ...(profileContents === undefined ? {} : { profileContents }), ...(stdin.includes('COMPLETE_THEN_HANG') ? { pid: process.pid } : {}), ...(openaiAuthentication ? { openaiAuthentication } : {}), ...(bedrockAuthentication ? { bedrockAuthentication } : {}), ...(providerAuthentication ? { providerAuthentication } : {}) })); if (stdin.includes('COMPLETE_THEN_HANG')) process.on('SIGTERM', () => { if (!stdin.includes('IGNORE_TERMINATION')) setTimeout(() => process.exit(0), 100); }); if (stdin.includes('THREAD_START_CONFIG_ERROR')) { console.error('Error: thread/start: thread/start failed: agents.max_threads cannot be set when features.multi_agent_v2 is enabled (code -32600)'); process.exit(1); } diff --git a/plugins/codex-security/mcp-app/tests/support/workbench-process.ts b/plugins/codex-security/mcp-app/tests/support/workbench-process.ts index 5e398ad826..8f5373da8a 100644 --- a/plugins/codex-security/mcp-app/tests/support/workbench-process.ts +++ b/plugins/codex-security/mcp-app/tests/support/workbench-process.ts @@ -19,6 +19,9 @@ export async function loadWorkbenchProcess( __dirname: JSON.stringify(applicationRoot), "import.meta.url": JSON.stringify(pathToFileURL(file).href), }, + banner: { + js: `import { createRequire as fixtureCreateRequire } from "node:module"; const require = fixtureCreateRequire(${JSON.stringify(pathToFileURL(file).href)});`, + }, loader: { ".md": "text" }, }); } diff --git a/plugins/codex-security/mcp-app/tests/test_accepted_source_replay.mjs b/plugins/codex-security/mcp-app/tests/test_accepted_source_replay.mjs new file mode 100644 index 0000000000..2b87097731 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_accepted_source_replay.mjs @@ -0,0 +1,69 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readFile, realpath, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { + partitions, + permutations, + sourceFixes, + workers, +} from "./fixtures/accepted_source_bank.mjs"; +import { loadReducer, replay } from "./fixtures/accepted_source_replay.mjs"; + +test("accepted sources survive completion orders and eligible batch partitions", async () => { + const reducer = await loadReducer(); + const root = await realpath( + await mkdtemp(path.join(tmpdir(), "accepted-source-replay-")), + ); + let count = 0; + try { + for (const order of permutations(workers.map((worker) => worker.id))) { + for (const batches of partitions(order)) { + // Ordinary rolling runs require two successes for the first merge. + if (batches[0].length < 2) continue; + const runRoot = path.join(root, String(count++)); + const { result, steps } = await replay(reducer, runRoot, batches); + assert.equal(result.findings.length, 3); + const refs = result.findings.flatMap( + (finding) => finding.provenance.sourceFindingIds, + ); + assert.deepEqual(refs.toSorted(), Object.keys(sourceFixes).toSorted()); + for (const step of steps) { + assert.deepEqual( + step.receipt.consumedWorkerIds, + batches[steps.indexOf(step)], + ); + } + for (const finding of result.findings) { + for (const source of finding.provenance.sourceFindings) { + const [workerId, index] = source.id.split(":"); + const original = workers.find((worker) => worker.id === workerId) + .result.findings[Number(index)]; + assert.deepEqual(source.finding, original); + } + } + for (const worker of workers) { + const persisted = JSON.parse( + await readFile( + path.join( + runRoot, + "artifacts", + "deep_discovery", + "workers", + worker.id, + "output", + "result.json", + ), + "utf8", + ), + ); + assert.deepEqual(persisted, worker.result); + } + } + } + assert.equal(count, 96); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/plugins/codex-security/mcp-app/tests/test_artifact_deep_reducer.ts b/plugins/codex-security/mcp-app/tests/test_artifact_deep_reducer.ts index 7bf807ccc1..b1a2a8d9af 100644 --- a/plugins/codex-security/mcp-app/tests/test_artifact_deep_reducer.ts +++ b/plugins/codex-security/mcp-app/tests/test_artifact_deep_reducer.ts @@ -2,21 +2,45 @@ import { readJson, writeJson } from "./support/json.ts"; import { sourceReferences } from "./support/source-references.ts"; import { temporaryDirectory } from "./support/temporary-directories.ts"; import { finding, scanId, workerDraft } from "./scan-draft-fixture.ts"; +import { + legacyAcceptedFinding, + legacyRawDiscoveryFinding, +} from "./legacy-accepted-finding.ts"; import assert from "node:assert/strict"; import { mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises"; import path from "node:path"; import { importSource } from "./import-module.ts"; type Finding = ReturnType; +type SourceCoverage = + import("../src/artifact-scan-draft.js").ScanDraftInput["coverage"] & { + reviews: unknown[]; + deferred: { provenance: unknown }[]; + surfaces: { receiptRefs: unknown[] }[]; + }; const { deepReducerInputsInputSchema, deepReductionInputSchema, - getCodexSecurityDeepReducerInputs, + getCodexSecurityDeepReducerInputs: getModelInputs, + readDeepReductionSources, recordCodexSecurityDeepReduction, -} = await importSource( +} = (await importSource( path.join(import.meta.dirname, "../src/artifact-deep-reducer.ts"), -); +)) as typeof import("../src/artifact-deep-reducer.js"); + +async function getCodexSecurityDeepReducerInputs( + context: Parameters[0], +) { + return (await readDeepReductionSources(context)) as Omit< + Awaited>, + "discoveries" + > & { + discoveries: (Awaited< + ReturnType + >["discoveries"][number] & { coverage: SourceCoverage })[]; + }; +} const validReduction = reduction([]); @@ -54,6 +78,7 @@ assert.equal( const root = await temporaryDirectory("codex-security-deep-reducer-", true); try { + await testHistoricalAssignedInputs(root); const scanRoot = path.join(root, "scan"); const workersRoot = path.join( scanRoot, @@ -117,9 +142,10 @@ try { root: outputRoot, repoRoot: root, scanId, - layout: "reducer", + layout: "reducer" as const, deepReducer: { scanRoot, + persistSourceCoverage: true, claimedWorkers: [first, second], }, }; @@ -164,13 +190,39 @@ try { /evidenceRefs must refer/, "live reducer submissions reject unknown evidence references instead of silently removing them", ); - assert.deepEqual(inputs, { - discoveries: [ - { workerId: first.id, result: withSourceRefs(first) }, - { workerId: second.id, result: withSourceRefs(second) }, - ], - previous: null, + assert.deepEqual( + { + ...inputs, + discoveries: inputs.discoveries.map(({ coverage, ...source }) => source), + }, + { + discoveries: [ + { workerId: first.id, result: withSourceRefs(first) }, + { workerId: second.id, result: withSourceRefs(second) }, + ], + previous: null, + }, + ); + assert.equal(inputs.discoveries[0]!.coverage!.completeness, undefined); + assert.equal(inputs.discoveries[1]!.coverage!.completeness, undefined); + assert.deepEqual( + await getModelInputs(context), + { + discoveries: inputs.discoveries.map(({ workerId, result }) => ({ + workerId, + result, + })), + previous: null, + }, + "coverage accounting does not change reducer model inputs", + ); + assert.deepEqual(inputs.discoveries[0]!.coverage!.deferred[0].provenance, { + workerId: first.id, + candidateId: "candidate-upload", }); + assert.deepEqual(inputs.discoveries[0]!.coverage!.surfaces[0].receiptRefs, [ + "artifacts/deep_discovery/workers/discovery-0001/output/artifacts/missing-worker-receipt.md", + ]); assert.equal(JSON.stringify(inputs).includes(root), false); assert.equal(JSON.stringify(inputs).includes("result.json"), false); @@ -194,6 +246,13 @@ try { const outcome = await recordCodexSecurityDeepReduction(context, merged); const mergedWithSources = { ...merged, + sourceCoverage: { + ...inputs.discoveries[0]!.coverage!, + reviews: [ + ...inputs.discoveries[0]!.coverage!.reviews, + ...inputs.discoveries[1]!.coverage!.reviews, + ], + }, findings: [ retainedFinding(shared, [ { id: "worker-001:0", finding: shared }, @@ -217,7 +276,7 @@ try { assert.deepEqual( await readJson(outputRoot, "checkpoints", checkpointNames[0]), mergedWithSources, - "reducer checkpoints retain the accepted findings and scope without coverage", + "reducer checkpoints retain accepted findings, scope and source coverage", ); assert.deepEqual( @@ -254,8 +313,8 @@ try { await getCodexSecurityDeepReducerInputs(collisionContext); const sourceFindingIds = collisionInputs.discoveries[0].result.findings.flatMap( - (finding: { provenance: { sourceFindingIds: string[] } }) => - finding.provenance.sourceFindingIds, + (finding) => + (finding.provenance as { sourceFindingIds: string[] }).sourceFindingIds, ); assert.deepEqual(sourceFindingIds, [ "worker-collision:0", @@ -314,18 +373,33 @@ try { root: nextOutputRoot, repoRoot: root, scanId, - layout: "reducer", + layout: "reducer" as const, deepReducer: { scanRoot, + persistSourceCoverage: true, claimedWorkers: [third], previousReducerResultPath: path.join(outputRoot, "result.json"), }, }; const nextInputs = await getCodexSecurityDeepReducerInputs(nextContext); - assert.deepEqual(nextInputs, { - discoveries: [{ workerId: third.id, result: withSourceRefs(third) }], - previous: mergedWithSources, - }); + const { sourceCoverage, ...previousModelInput } = mergedWithSources; + assert.deepEqual( + (await getModelInputs(nextContext)).previous, + previousModelInput, + "host coverage metadata is excluded from the previous model input too", + ); + assert.deepEqual( + { + ...nextInputs, + discoveries: nextInputs.discoveries.map( + ({ coverage, ...source }) => source, + ), + }, + { + discoveries: [{ workerId: third.id, result: withSourceRefs(third) }], + previous: mergedWithSources, + }, + ); await assert.rejects( recordCodexSecurityDeepReduction(nextContext, reduction([shared])), (error: NodeJS.ErrnoException) => @@ -335,17 +409,29 @@ try { await recordCodexSecurityDeepReduction(nextContext, merged), { findingCount: 2, consumedWorkerIds: [third.id] }, ); - assert.deepEqual(await readJson(nextOutputRoot, "result.json"), { - ...mergedWithSources, - findings: [ - retainedFinding(shared, [ - { id: "worker-003:0", finding: shared }, - { id: "worker-001:0", finding: shared }, - { id: "worker-002:0", finding: shared }, - ]), - mergedWithSources.findings[1], - ], - }); + assert.deepEqual( + JSON.parse( + await readFile(path.join(nextOutputRoot, "result.json"), "utf8"), + ), + { + ...mergedWithSources, + sourceCoverage: { + ...mergedWithSources.sourceCoverage, + reviews: [ + ...(mergedWithSources.sourceCoverage as SourceCoverage).reviews, + ...nextInputs.discoveries[0]!.coverage!.reviews, + ], + }, + findings: [ + retainedFinding(shared, [ + { id: "worker-003:0", finding: shared }, + { id: "worker-001:0", finding: shared }, + { id: "worker-002:0", finding: shared }, + ]), + mergedWithSources.findings[1], + ], + }, + ); const enrichedPrevious = structuredClone(mergedWithSources); enrichedPrevious.findings[0].summary = @@ -450,6 +536,63 @@ try { console.log("artifact deep reducer tests passed"); +async function testHistoricalAssignedInputs(root: string) { + const scanRoot = path.join(root, "historical"); + const workersRoot = path.join( + scanRoot, + "artifacts", + "deep_discovery", + "workers", + ); + const dedupRoot = path.join(scanRoot, "artifacts", "deep_discovery", "dedup"); + const current = finding("historical-checkpoint", "fixture.py"); + const legacy = legacyAcceptedFinding(current); + const worker = await createWorker({ + workersRoot, + label: "discovery-history", + id: "worker-history", + result: workerDraft([legacyRawDiscoveryFinding(current)]), + }); + const output = path.join(dedupRoot, "dedup-history", "output"); + await mkdir(output, { recursive: true }); + const previousReducerResultPath = path.join(output, "previous.json"); + await writeJson(previousReducerResultPath, reduction([legacy])); + const context = { + root: output, + repoRoot: root, + scanId, + layout: "reducer" as const, + deepReducer: { + scanRoot, + claimedWorkers: [worker], + previousReducerResultPath, + }, + }; + const sourceBytes = await readFile(worker.resultPath, "utf8"); + const previousBytes = await readFile(previousReducerResultPath, "utf8"); + const sources = await readDeepReductionSources(context); + assert.deepEqual( + sources.discoveries[0].result.findings[0].root_cause, + legacy.root_cause, + ); + assert.deepEqual(sources.previous!.findings[0], legacy); + await assert.rejects( + recordCodexSecurityDeepReduction(context, reduction([legacy])), + ); + await recordCodexSecurityDeepReduction(context, reduction([current])); + const saved = await readJson(path.join(output, "result.json")); + assert.equal(saved.findings[0].root_cause, undefined); + assert.deepEqual( + saved.findings[0].provenance.sourceFindings[0].finding.root_cause, + legacy.root_cause, + ); + assert.equal(await readFile(worker.resultPath, "utf8"), sourceBytes); + assert.equal( + await readFile(previousReducerResultPath, "utf8"), + previousBytes, + ); +} + async function createWorker({ workersRoot, label, diff --git a/plugins/codex-security/mcp-app/tests/test_audit_acceptance_contract.ts b/plugins/codex-security/mcp-app/tests/test_audit_acceptance_contract.ts new file mode 100644 index 0000000000..179f733f6d --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_audit_acceptance_contract.ts @@ -0,0 +1,257 @@ +import type { ArtifactContext } from "../src/artifact-context.js"; +import { recordCodexSecurityScanDraft } from "./scan-draft-recovery-fixture.ts"; +import assert from "node:assert/strict"; +import { + mkdtemp, + mkdir, + readFile, + readdir, + realpath, + rm, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { build } from "esbuild"; + +const bundle = await build({ + stdin: { + contents: `export * from "./src/artifact-scan-draft.ts"; + export * from "./src/deep-scan/artifact-validation.ts"; + export * from "./src/deep-scan/artifacts.ts"; + export * from "../../../sdk/typescript/src/accepted-audit.ts";`, + resolveDir: path.resolve(import.meta.dirname, ".."), + }, + bundle: true, + format: "esm", + platform: "node", + write: false, + footer: { js: "//# sourceURL=audit-acceptance-contract.js" }, +}); +const { + createDeepScanArtifacts, + recordCodexSecurityWorkerScanDraft, + validateDiscoveryArtifacts, + readDiscoveryAuditDraft, + auditEvidence, + runAcceptedAudit, +} = await import( + `data:text/javascript;base64,${Buffer.from(bundle.outputFiles[0].contents).toString("base64")}` +); + +const scanId = "811aef98-3709-4c2d-8b7a-742977521865"; +const finding = { + ruleId: "path-traversal.archive-extraction", + title: "Unsafe archive extraction", + summary: "An archive entry reaches a filesystem write.", + severity: { level: "high" }, + confidence: { level: "high", rationale: "Source review." }, + taxonomy: { category: "path-traversal", cwe: ["CWE-22"] }, + locations: [{ path: "src/extract.py", startLine: 4 }], + remediation: "Validate the resolved output path before writing.", + provenance: { source: "local_plugin", candidateId: "archive-entry" }, +}; + +for (const completeness of ["complete", "partial", "unknown"]) { + test(`Standard and Deep retain accepted semantic evidence with ${completeness} coverage`, async () => { + const root = await realpath( + await mkdtemp(path.join(tmpdir(), "audit-contract-")), + ); + try { + const repository = path.join(root, "repository"); + const scanDir = path.join(root, "scan"); + const artifacts = createDeepScanArtifacts(scanDir); + const workerRoot = path.join( + artifacts.workersRoot, + "discovery-0001", + "output", + ); + await Promise.all([ + mkdir(repository), + mkdir(workerRoot, { recursive: true }), + ]); + const semantic = { + scanId, + complete: true, + threatModel: { + summary: "An untrusted caller supplies archive entries.", + }, + findings: [finding], + coverage: { + completeness, + surfaces: [ + { + id: "archive", + label: "Archive extraction", + disposition: "reported", + receiptRefs: [], + }, + ], + explicitExclusions: [], + deferred: + completeness === "complete" + ? [] + : [ + { + id: "deployment-controls", + reason: "Deployment controls remain unverified.", + }, + ], + }, + }; + const standard: ArtifactContext = { + root: scanDir, + repoRoot: repository, + layout: "scan", + scanId, + mode: "standard", + status: "running", + scope: ".", + targetContract: { + target: { + allowedKinds: ["directory_snapshot"], + targetId: "target_example", + displayName: "example", + requiredSnapshotDigest: `codex-security-snapshot/v1:sha256:${"a".repeat(64)}`, + }, + scope: { requiredIncludePaths: ["."], requiredExcludePaths: [] }, + diffTarget: null, + }, + }; + const worker: ArtifactContext = { + root: workerRoot, + repoRoot: repository, + layout: "worker", + scanId, + }; + const checkpoint = { ...semantic, complete: false }; + await recordCodexSecurityScanDraft(standard, checkpoint); + await recordCodexSecurityWorkerScanDraft(worker, checkpoint); + await assert.rejects( + validateDiscoveryArtifacts( + artifacts, + path.join(workerRoot, "result.json"), + scanId, + ), + /checkpoint/, + ); + assert.equal( + JSON.parse( + await readFile(path.join(scanDir, "scan-manifest.json"), "utf8"), + ).scan.complete, + false, + ); + const controller = new AbortController(); + const execute = async () => ({ + threadId: "audit-conversation", + usage: null, + }); + const accept = async () => + auditEvidence( + await readDiscoveryAuditDraft( + artifacts, + path.join(workerRoot, "result.json"), + scanId, + ), + ); + const unfinished = await runAcceptedAudit({ + signal: controller.signal, + execute, + accept, + }); + assert.equal(unfinished.status, "checkpoint"); + assert.equal(unfinished.checkpoint.complete, false); + assert.equal(unfinished.accepted, undefined); + assert.equal(unfinished.execution.usage, null); + const standardWrite = await recordCodexSecurityScanDraft( + standard, + semantic, + ); + const deepWrite = await recordCodexSecurityWorkerScanDraft( + worker, + semantic, + ); + assert.equal(standardWrite.status, "draft_written"); + assert.equal(deepWrite.status, "draft_written"); + const accepted = await validateDiscoveryArtifacts( + artifacts, + path.join(workerRoot, "result.json"), + scanId, + ); + const manifest = JSON.parse( + await readFile(path.join(scanDir, "scan-manifest.json"), "utf8"), + ); + const findings = JSON.parse( + await readFile(path.join(scanDir, "findings.json"), "utf8"), + ); + const coverage = JSON.parse( + await readFile(path.join(scanDir, "coverage.json"), "utf8"), + ); + assert.deepEqual(accepted.findings, [finding]); + for (const [key, value] of Object.entries(finding)) { + assert.deepEqual(findings.findings[0][key], value); + } + assert.ok(findings.findings[0].identity.anchor); + assert.deepEqual(accepted.threatModel, manifest.scan.threatModel); + for (const field of [ + "completeness", + "surfaces", + "explicitExclusions", + "deferred", + ]) { + assert.deepEqual(coverage[field], accepted.coverage[field]); + } + assert.equal(accepted.scanId, scanId); + const audit = await runAcceptedAudit({ + signal: controller.signal, + execute, + accept, + }); + assert.equal(audit.status, "accepted"); + assert.deepEqual(audit.accepted, accepted); + assert.deepEqual(audit.checkpoint, accepted); + const failure = new Error("Synthetic execution failure"); + await assert.rejects( + runAcceptedAudit({ + signal: controller.signal, + execute: async () => { + throw failure; + }, + accept: async () => { + assert.fail("An execution failure cannot accept old output"); + }, + }), + (error) => error === failure, + ); + await assert.rejects( + runAcceptedAudit({ + signal: controller.signal, + execute, + accept: async () => { + const evidence = await accept(); + controller.abort("user canceled"); + return evidence; + }, + }), + (error) => error === controller.signal.reason, + ); + assert.equal(manifest.scan.sealedAt, undefined); + assert.equal(manifest.scan.artifacts, undefined); + assert.equal( + (await readdir(workerRoot)).includes("scan-manifest.json"), + false, + ); + assert.equal((await readdir(scanDir)).includes("report.md"), false); + await assert.rejects( + validateDiscoveryArtifacts( + artifacts, + path.join(workerRoot, "result.json"), + "b4c84677-5aaf-410c-88d2-3e97e6f8c4d8", + ), + /scan/, + ); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +} diff --git a/plugins/codex-security/mcp-app/tests/test_codex_profile.ts b/plugins/codex-security/mcp-app/tests/test_codex_profile.ts index 7c2b4091c5..5a07e2d955 100644 --- a/plugins/codex-security/mcp-app/tests/test_codex_profile.ts +++ b/plugins/codex-security/mcp-app/tests/test_codex_profile.ts @@ -1,7 +1,15 @@ import assert from "node:assert/strict"; import childProcess, { type SpawnOptions } from "node:child_process"; -import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; -import { syncBuiltinESMExports } from "node:module"; +import { + chmod, + copyFile, + mkdir, + mkdtemp, + readFile, + rm, + writeFile, +} from "node:fs/promises"; +import { createRequire, syncBuiltinESMExports } from "node:module"; import { tmpdir } from "node:os"; import path from "node:path"; import { mock, test } from "node:test"; @@ -110,6 +118,8 @@ async function fixture(mode = "success") { }, })); const emit = (event) => process.stdout.write(JSON.stringify(event) + "\n"); + if (process.env.PROFILE_TEST_MODE === "slow_completion") + process.once("SIGTERM", () => setTimeout(() => process.exit(0), 50)); if (process.env.PROFILE_TEST_MODE === "error") { process.stderr.write("synthetic token=fixture-secret: upstream diagnostic\n"); process.exit(3); @@ -139,7 +149,10 @@ async function fixture(mode = "success") { if (process.env.PROFILE_TEST_MODE === "null_usage") completion.usage = null; else if (process.env.PROFILE_TEST_MODE !== "missing_usage") completion.usage = { input_tokens: 2, cached_input_tokens: 1, output_tokens: 3, reasoning_output_tokens: 0, ...(args.includes("resume") ? { cache_write_input_tokens: 9 } : {}) }; emit(completion); - process.exit(0); + if (process.env.PROFILE_TEST_MODE === "slow_completion") { + setInterval(() => {}, 1000); + await new Promise(() => {}); + } else process.exit(0); } `, ); @@ -177,6 +190,48 @@ async function fixture(mode = "success") { }; } +async function npmLauncherFixture() { + const f = await fixture("slow_completion"); + const codexRequire = createRequire(import.meta.resolve("@openai/codex-sdk")); + const packageRoot = path.dirname( + codexRequire.resolve("@openai/codex/package.json"), + ); + const launcherRoot = path.join(f.directory, "npm-codex"); + const targetTriple = new Map([ + ["linux-x64", "x86_64-unknown-linux-musl"], + ["linux-arm64", "aarch64-unknown-linux-musl"], + ["darwin-x64", "x86_64-apple-darwin"], + ["darwin-arm64", "aarch64-apple-darwin"], + ]).get(`${process.platform}-${process.arch}`); + assert.ok(targetTriple); + const launcher = path.join(launcherRoot, "bin", "codex.js"); + const native = path.join( + launcherRoot, + "vendor", + targetTriple, + "bin", + "codex", + ); + await mkdir(path.dirname(launcher), { recursive: true }); + await mkdir(path.dirname(native), { recursive: true }); + await copyFile( + path.join(packageRoot, "package.json"), + path.join(launcherRoot, "package.json"), + ); + await copyFile(path.join(packageRoot, "bin", "codex.js"), launcher); + await chmod(launcher, 0o755); + await writeFile( + native, + "#!/usr/bin/env node\n" + + ( + await readFile(path.join(f.directory, "native-fixture.mjs"), "utf8") + ).replace("process.argv.slice(1)", "process.argv.slice(2)"), + ); + await chmod(native, 0o755); + const { NODE_OPTIONS: _fixtureImport, ...env } = f.options.env; + return { ...f, options: { ...f.options, codexPathOverride: launcher, env } }; +} + test("native profile turns preserve settings, JSON events, schema cleanup, and resume identity", async () => { const f = await fixture(); const previousEnvironment = process.env.PROFILE_CLIENT_SHOULD_NOT_LEAK; @@ -406,18 +461,28 @@ test("native profile failures preserve upstream diagnostics and remove schema fi function captureChildClose() { let closed: Promise | undefined; + let isClosed = false; + let pid: number | undefined; const originalSpawn = childProcess.spawn; const spawnSpy = mock.method( childProcess, "spawn", (command: string, args: readonly string[], options: SpawnOptions) => { const child = originalSpawn(command, args, options); - closed = new Promise((resolve) => child.once("close", () => resolve())); + pid = child.pid; + closed = new Promise((resolve) => + child.once("close", () => { + isClosed = true; + resolve(); + }), + ); return child; }, ); syncBuiltinESMExports(); return { + isClosed: () => isClosed, + pid: () => pid, wait: () => { assert.ok(closed); return closed; @@ -429,6 +494,93 @@ function captureChildClose() { }; } +for (const resumed of [false, true]) { + test(`native profile ${resumed ? "resumed" : "fresh"} completion waits for its slow child to close`, async () => { + const f = await fixture("slow_completion"); + const childClose = captureChildClose(); + try { + const client = createCodexProfileClient(f.options); + const thread = resumed + ? client.resumeThread("existing-fixture") + : client.startThread(); + const { events } = await thread.runStreamed("synthetic prompt"); + for await (const event of events) { + if (event.type === "turn.completed") break; + } + assert.equal(childClose.isClosed(), true); + const record = await f.record(); + assert.throws(() => process.kill(record.pid, 0), { code: "ESRCH" }); + } finally { + await childClose.wait(); + childClose.restore(); + await f.cleanup(); + } + }); +} + +for (const resumed of [false, true]) { + test( + `native profile ${resumed ? "resumed" : "fresh"} completion releases npm launcher inherited output`, + { skip: process.platform === "win32" }, + async () => { + const f = await npmLauncherFixture(); + const childClose = captureChildClose(); + let completion: Promise | undefined; + let timeout: ReturnType | undefined; + try { + const client = createCodexProfileClient(f.options); + const thread = resumed + ? client.resumeThread("existing-fixture") + : client.startThread(); + const { events } = await thread.runStreamed("synthetic prompt", { + outputSchema: { type: "object" }, + }); + completion = (async () => { + for await (const event of events) { + if (event.type === "turn.completed") break; + } + })(); + await Promise.race([ + completion, + new Promise((_, reject) => { + timeout = setTimeout(() => { + reject( + new Error( + "Completed profile turn did not release npm launcher output.", + ), + ); + }, 1_000); + }), + ]); + clearTimeout(timeout); + assert.equal(childClose.isClosed(), true); + assert.ok(childClose.pid()); + assert.throws(() => process.kill(childClose.pid()!, 0), { + code: "ESRCH", + }); + const record = await f.record(); + assert.equal(record.args.includes("resume"), resumed); + assert.equal(record.environment.CODEX_HOME, f.options.env.CODEX_HOME); + assert.equal(record.environment.CODEX_API_KEY, "synthetic-api-key"); + assert.throws(() => process.kill(record.pid, 0), { code: "ESRCH" }); + await assert.rejects(readFile(record.schemaPath), { code: "ENOENT" }); + } finally { + clearTimeout(timeout); + const record = await f.record().catch(() => undefined); + if (record?.pid) { + try { + process.kill(record.pid, "SIGKILL"); + } catch {} + } + await completion?.catch(() => {}); + await childClose.wait(); + childClose.restore(); + await f.cleanup(); + } + }, + ); +} + test("native profile abort and abandoned streams close the actual child", async () => { for (const abort of [true, false]) { const f = await fixture("wait"); diff --git a/plugins/codex-security/mcp-app/tests/test_compact_artifact_server.ts b/plugins/codex-security/mcp-app/tests/test_compact_artifact_server.ts index 4f76bfd674..a2d3ada52e 100644 --- a/plugins/codex-security/mcp-app/tests/test_compact_artifact_server.ts +++ b/plugins/codex-security/mcp-app/tests/test_compact_artifact_server.ts @@ -19,7 +19,11 @@ type CompletedResult = { fingerprints: { primary: string }; })[]; }; - coverage: Record & { surfaces: { disposition: string }[] }; + coverage: Record & { + surfaces: { disposition: string }[]; + deferred: unknown[]; + explicitExclusions: unknown[]; + }; }; type ToolResponse = Awaited>; type WorkspaceResult = { @@ -28,6 +32,8 @@ type WorkspaceResult = { type ScanResult = { scan: { progress: Record; + scanDir: string; + usage: unknown; reportAvailable: boolean; continuationThreadId?: string; handoffClaimToken?: string; @@ -300,8 +306,12 @@ async function testCompactDiffScanCompletion( }), `${runtimeLabel}: record compact diff canonical semantics`, ); + const completion = await call("complete_codex_security_scan", { + scanId, + handoffClaimToken, + }); requireSuccessfulTool( - await call("complete_codex_security_scan", { scanId, handoffClaimToken }), + completion, `${runtimeLabel}: complete compact diff scan`, ); const completed = requireSuccessfulTool( @@ -319,6 +329,12 @@ async function testCompactDiffScanCompletion( ); assert.equal(completed.coverage.inventoryStrategy, "diff"); assert.equal(completed.findings.findings.length, 0); + await assertCompletionCoverageAndReplay( + call, + { scanId, handoffClaimToken }, + completion, + completed.coverage, + ); } finally { await client.close(); } @@ -760,8 +776,12 @@ async function testSemanticScanDraftCompletion( unit: null, }); + const completion = await call("complete_codex_security_scan", { + scanId, + handoffClaimToken, + }); const completed = requireSuccessfulTool( - await call("complete_codex_security_scan", { scanId, handoffClaimToken }), + completion, `${runtimeLabel}: finalize the accepted draft exactly once`, ); assert.equal(completed.scan.progress.status, "complete"); @@ -804,6 +824,12 @@ async function testSemanticScanDraftCompletion( assert.equal(results.manifest.scan.target.kind, "directory_snapshot"); assert.equal(results.coverage.inventoryStrategy, "directory"); assert.equal(results.coverage.completeness, undefined); + await assertCompletionCoverageAndReplay( + call, + { scanId, handoffClaimToken }, + completion, + results.coverage, + ); assert.deepEqual(results.coverage.includePaths, ["."]); assert.deepEqual(results.coverage.excludePaths, []); assert.equal(results.coverage.surfaces[0].disposition, "reported"); @@ -1132,6 +1158,46 @@ async function testClaimedParentArtifactOperations( } } +async function assertCompletionCoverageAndReplay( + call: ReturnType, + identity: { scanId: string; handoffClaimToken?: string }, + completion: ToolResponse, + coverage: CompletedResult["coverage"], +) { + const expected = { + surfaceCount: coverage.surfaces.length, + deferredCount: coverage.deferred.length, + explicitExclusionCount: coverage.explicitExclusions.length, + }; + type Completion = ScanResult & { coverageSummary: typeof expected }; + const scan = requireSuccessfulTool(completion).scan; + const snapshot = () => + Promise.all( + ["scan-manifest.json", "findings.json", "coverage.json", "report.md"].map( + (name) => readFile(path.join(scan.scanDir, name), "utf8"), + ), + ); + const sealed = await snapshot(); + const replay = await call("complete_codex_security_scan", identity); + for (const result of [completion, replay]) { + const structured = requireSuccessfulTool(result); + assert.equal(structured.scan.progress.status, "complete"); + const content = (result.content as TextContent[])[0]; + assert.equal(content.type, "text"); + assert.equal( + content.text, + "Validated and indexed the completed Codex Security scan.", + ); + assert.deepEqual(structured.coverageSummary, expected); + assert.deepEqual(structured.scan.usage, scan.usage); + } + assert.deepEqual( + await snapshot(), + sealed, + "completion replay preserves sealed artifacts", + ); +} + function requireSuccessfulTool>( result: ToolResponse, label?: string, @@ -1381,6 +1447,7 @@ async function testDiscoveryWorkerToolList(bundle: string) { for (const forbiddenInput of [ { ...input, scope: { includePaths: ["src"] } }, { ...input, coverage: { ...input.coverage, mode: "repository" } }, + { ...input, coverage: { ...input.coverage, scanId } }, { ...input, findings: [ diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_artifact_validation.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_artifact_validation.ts index 1a13b09cbd..85e1e63ec9 100644 --- a/plugins/codex-security/mcp-app/tests/test_deep_scan_artifact_validation.ts +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_artifact_validation.ts @@ -1,5 +1,9 @@ import { readJson, writeJsonLine as writeResult } from "./support/json.ts"; import { finding, scanId, workerDraft as draft } from "./scan-draft-fixture.ts"; +import { + legacyAcceptedFinding, + legacyRawDiscoveryFinding, +} from "./legacy-accepted-finding.ts"; import { temporaryDirectory } from "./support/temporary-directories.ts"; import assert from "node:assert/strict"; import { @@ -18,6 +22,8 @@ import type { ScanDraftInput } from "../src/artifact-scan-draft.ts"; import type { JsonObject } from "../src/types.ts"; const { + parseDeepReduction, + projectDiscoveryCoverage, validateDiscoveryArtifacts, validateReducerArtifacts, aggregateDiscoveryCoverage, @@ -68,12 +74,70 @@ try { await testCurrentAnonymousReceiptSelection(root, candidate, reverse); await testReducerValidation(root); await testEmptyDiscoveryAndReduction(root); + await testLongDeferredCandidateRecovery(root); + await testAcceptedHistoricalFindings(root); } finally { await rm(root, { recursive: true, force: true }); } console.log("deep scan artifact validation tests passed"); +async function testAcceptedHistoricalFindings(root: string) { + const current = finding("historical-checkpoint", "fixture.py"); + const legacy = legacyAcceptedFinding(current); + const { artifacts, ...worker } = await createWorker( + path.join(root, "accepted-history"), + "worker-history", + draft([legacyRawDiscoveryFinding(current)]), + ); + const workerBytes = await readFile(worker.resultPath, "utf8"); + await assert.rejects( + validateDiscoveryArtifacts(artifacts, worker.resultPath, scanId), + ); + const recovered = await validateDiscoveryArtifacts( + artifacts, + worker.resultPath, + scanId, + true, + true, + ); + assert.deepEqual(recovered.findings, [legacy]); + assert.equal(await readFile(worker.resultPath, "utf8"), workerBytes); + + const artifactDir = path.join(artifacts.dedupRoot, "dedup-history", "output"); + const resultPath = path.join(artifactDir, "result.json"); + await mkdir(artifactDir, { recursive: true }); + await writeResult(resultPath, reduction([legacy])); + const input = { + artifacts, + artifactDir, + resultPath, + reducerId: "dedup-history", + }; + const previousBytes = await readFile(resultPath, "utf8"); + await assert.rejects(validateReducerArtifacts(input, scanId)); + assert.deepEqual( + (await validateReducerArtifacts({ ...input, accepted: true }, scanId)) + .result.findings, + [legacy], + ); + assert.equal(await readFile(resultPath, "utf8"), previousBytes); + + const currentPath = path.join(artifactDir, "current.json"); + await writeResult(currentPath, reduction([current])); + const next = await validateReducerArtifacts( + { + ...input, + resultPath: currentPath, + previousReducerResultPath: resultPath, + }, + scanId, + ); + assert.equal(next.newFindings, 0); + assert.deepEqual(next.result.findings, [current]); + assert.equal(await readFile(resultPath, "utf8"), previousBytes); +} + async function testDiscoveryReceipts(root: string) { const result = draft([], { coverage: { @@ -883,9 +947,20 @@ async function testReducerValidation(root: string) { ], previous: null, }; - const validateSnapshot = (reducerId = "dedup-0001", snapshot = sources) => + const validateSnapshot = ( + reducerId = "dedup-0001", + snapshot = sources, + persistSourceCoverage = false, + ) => validateReducerArtifacts( - { artifacts, artifactDir, resultPath, reducerId, sources: snapshot }, + { + artifacts, + artifactDir, + resultPath, + reducerId, + sources: snapshot, + persistSourceCoverage, + }, scanId, ); await assert.rejects(validateSnapshot(), /unaccounted source findings/); @@ -894,10 +969,18 @@ async function testReducerValidation(root: string) { const validatedSnapshot = await validateSnapshot(); assert.equal(validatedSnapshot.newFindings, 2); const admitted = await readJson(resultPath); + const { sourceCoverage, ...legacySnapshot } = validatedSnapshot.result; + assert.equal(sourceCoverage.completeness, undefined); assert.deepEqual( - validatedSnapshot.result, + legacySnapshot, admitted, - "validation returns the same reconciled result that was accepted on disk", + "v1 preserves host coverage in memory while retaining the legacy persisted shape", + ); + const versionedSnapshot = await validateSnapshot("dedup-0001", sources, true); + assert.deepEqual( + versionedSnapshot.result, + JSON.parse(await readFile(resultPath, "utf8")), + "v2 persists the full host projection", ); assert.equal(Object.hasOwn(admitted, "coverage"), false); assert.deepEqual(admitted.findings[1].provenance.sourceFindingIds, [ @@ -1311,6 +1394,81 @@ async function testEmptyDiscoveryAndReduction(root: string) { ); } +async function testLongDeferredCandidateRecovery(root: string) { + const candidateIds = ["a".repeat(512), "b".repeat(512)]; + const { artifacts, ...worker } = await createWorker( + path.join(root, "long-deferred-candidates"), + "903ef9e0-b34e-4b59-99ac-31d047f82742", + draft([], { + coverage: { + surfaces: [], + explicitExclusions: [], + deferred: candidateIds.map((candidateId) => ({ + candidateId, + reason: "Follow up on this accepted candidate.", + })), + }, + }), + ); + const discovery = await validateDiscoveryArtifacts( + artifacts, + worker.resultPath, + scanId, + ); + const prefix = "artifacts/deep_discovery/workers/discovery-0001/output"; + const coverage = projectDiscoveryCoverage( + discovery.coverage, + { id: worker.id, attempt: 2 }, + prefix, + ); + const artifactDir = path.join(artifacts.dedupRoot, "dedup-long", "output"); + const resultPath = path.join(artifactDir, "result.json"); + await mkdir(artifactDir, { recursive: true }); + await writeResult(resultPath, reduction([])); + const accepted = await validateReducerArtifacts( + { + artifacts, + artifactDir, + resultPath, + reducerId: "dedup-long", + persistSourceCoverage: true, + sources: { + discoveries: [ + { workerId: worker.id, attempt: 2, coverage, result: discovery }, + ], + previous: null, + }, + }, + scanId, + ); + const recovered = parseDeepReduction(await readJson(resultPath), true); + assert.deepEqual(recovered.sourceCoverage, accepted.result.sourceCoverage); + assert.deepEqual( + recovered.sourceCoverage.deferred.map( + (item: { provenance: { candidateId: string } }) => + item.provenance.candidateId, + ), + candidateIds, + ); + assert.equal( + new Set( + recovered.sourceCoverage.deferred.map( + (item: { candidateId: string }) => item.candidateId, + ), + ).size, + candidateIds.length, + ); + const nextAttempt = projectDiscoveryCoverage( + discovery.coverage, + { id: worker.id, attempt: 3 }, + prefix, + ); + assert.notEqual( + nextAttempt.deferred[0].candidateId, + recovered.sourceCoverage.deferred[0].candidateId, + ); +} + async function createWorker(scanDir: string, id: string, result: unknown) { const artifacts = { scanDir, @@ -1321,7 +1479,7 @@ async function createWorker(scanDir: string, id: string, result: unknown) { await mkdir(output, { recursive: true }); const resultPath = path.join(output, "result.json"); await writeResult(resultPath, result); - return { artifacts, id, resultPath }; + return { artifacts, id, artifactDir: output, resultPath }; } async function testSharedMissingReceiptRepairHistory(root: string) { diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_attempt_replay.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_attempt_replay.mjs new file mode 100644 index 0000000000..3e184794fb --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_attempt_replay.mjs @@ -0,0 +1,367 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import { fileURLToPath } from "node:url"; +import { build } from "esbuild"; +import { finding as fixtureFinding } from "./scan-draft-fixture.ts"; + +const app = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const plugin = path.resolve(app, ".."); +const bundle = await build({ + bundle: true, + format: "esm", + platform: "node", + write: false, + loader: { ".md": "text" }, + stdin: { + resolveDir: app, + contents: [ + 'export { WorkbenchDeepScanStore } from "./src/deep-scan/store.ts";', + 'export { DeepScanWorkerRunner } from "./src/deep-scan/worker-runner.ts";', + 'export { DeepScanCoordinator } from "./src/deep-scan/coordinator.ts";', + 'export { createDeepScanArtifacts, ensureDeepScanDirectories } from "./src/deep-scan/artifacts.ts";', + ].join("\n"), + }, +}); +const { + WorkbenchDeepScanStore, + DeepScanWorkerRunner, + DeepScanCoordinator, + createDeepScanArtifacts, + ensureDeepScanDirectories, +} = await import( + `data:text/javascript;base64,${Buffer.from(bundle.outputFiles[0].contents).toString("base64")}` +); +const execute = promisify(execFile); +for (const responseLosses of [3, 1]) await testResponseLoss(responseLosses); + +async function testResponseLoss(responseLosses) { + const root = await mkdtemp(path.join(tmpdir(), "deep-attempt-receipt-")); + const target = path.join(root, "target"); + const environment = { + ...process.env, + CODEX_HOME: path.join(root, "home"), + CODEX_SECURITY_STATE_DIR: path.join(root, "state"), + }; + const counts = new Map(); + const receipts = new Map(); + const raw = async (args) => { + const { stdout } = await execute( + process.env.PYTHON || "python3", + [path.join(plugin, "scripts/workbench_db.py"), ...args], + { + env: environment, + timeout: 30_000, + maxBuffer: 4 * 1024 * 1024, + }, + ); + return JSON.parse(stdout); + }; + const store = new WorkbenchDeepScanStore(async (args) => { + const result = await raw(args); + const key = + args[0] === "commit-deep-scan-dedup" + ? "merge" + : args[0] === "upsert-deep-scan-worker" && + args[args.indexOf("--status") + 1] === "succeeded" + ? "acceptance" + : null; + const operation = + key === "acceptance" + ? `${key}:${args[args.indexOf("--worker-id") + 1]}` + : key; + if (key) { + counts.set(key, (counts.get(key) ?? 0) + 1); + const receipt = + key === "acceptance" + ? result.deepScan.workerReceipt + : result.deepScan.committedMerge; + if (!receipts.has(operation)) receipts.set(operation, receipt); + else + assert.deepEqual( + receipt, + receipts.get(operation), + "replay returns the original operation receipt", + ); + if (counts.get(key) <= responseLosses) { + const error = new Error("fixture response lost after committed write"); + error.code = "ETIMEDOUT"; + throw error; + } + } + return result; + }); + try { + await mkdir(target); + await writeFile(path.join(target, "fixture.py"), "print('fixture')\n"); + const run = await store.begin({ + targetPath: target, + scope: ".", + threadId: "fixture-owner", + scanRoot: path.join(root, "scans"), + }); + const artifacts = createDeepScanArtifacts(run.scanDir); + await ensureDeepScanDirectories(artifacts); + let executions = 0; + const runner = new DeepScanWorkerRunner({ + run, + store, + artifacts, + pluginRoot: plugin, + signal: new AbortController().signal, + random: () => 0.5, + log: () => {}, + retryDelaysMs: [], + clock: { now: () => Date.now(), sleep: async () => {} }, + executor: { + async run(request) { + executions++; + if (request.kind === "dedup") { + const workers = request.artifactContext.deepReducer.claimedWorkers; + const prompt = await readFile(request.promptPath, "utf8"); + const configuration = JSON.parse( + prompt.match(/```json\n([\s\S]*?)\n```/)[1], + ); + assert.deepEqual( + configuration.claimedWorkerIds, + workers.map((worker) => worker.id), + ); + assert.equal( + workers.every((worker) => + worker.resultPath.includes("checkpoints"), + ), + true, + ); + assert.deepEqual( + workers.map((worker) => worker.attempt), + [1, 1], + "execution uses the immutable claim attempts", + ); + } + await request.onThreadStarted?.(`fixture-session-${executions}`); + const draft = { + scanId: run.scanId, + findings: [], + threatModel: { summary: "Synthetic fixture." }, + }; + if (request.kind === "discovery") + draft.coverage = { + completeness: "complete", + surfaces: [{ label: "Fixture", disposition: "no_issue_found" }], + explicitExclusions: [], + deferred: [], + }; + await writeFile( + path.join(request.artifactContext.root, "result.json"), + JSON.stringify(draft), + ); + return { threadId: `fixture-session-${executions}` }; + }, + }, + }); + if (responseLosses === 3) { + const outcome = await runner + .runDiscoveryWorker(randomUUID(), "discovery-1") + .catch((error) => error); + assert.equal( + counts.get("acceptance"), + 3, + "the runner must not multiply the store's retry policy", + ); + assert.match(outcome.message, /response lost/); + assert.equal(executions, 1); + return; + } + const discovery = await runner.runDiscoveryWorker( + randomUUID(), + "discovery-1", + ); + assert.equal(discovery.status, "succeeded"); + assert.match(discovery.worker.resultPath, /checkpoints/); + const second = await runner.runDiscoveryWorker(randomUUID(), "discovery-2"); + // Acceptance receipts are operation-specific, so only the first discovery loses a response. + await rm(path.join(discovery.worker.artifactDir, "result.json")); + const merged = await runner.runReducer({ + id: randomUUID(), + label: "dedup-1", + consumed: [discovery.worker, second.worker].map((worker) => ({ + ...worker, + resultPath: path.join(worker.artifactDir, "result.json"), + attempt: 99, + })), + }); + assert.equal(merged.error, undefined, merged.error?.stack); + assert.match(merged.resultPath, /checkpoints/); + assert.equal(merged.newFindings, 0); + assert.equal( + merged.run.persistedDedupInputs.filter( + (input) => input.dedupWorkerId === merged.id, + ).length, + 2, + ); + assert.equal(counts.get("merge"), 2); + assert.equal(executions, 3); + assert.equal(run.workflowVersion, "deep-security-scan/v2"); + assert.deepEqual( + JSON.parse(await readFile(merged.resultPath, "utf8")), + merged.result, + ); + assert.equal(merged.result.sourceCoverage.completeness, undefined); + const snapshot = await store.get(run.scanId, "fixture-owner"); + const resumed = new DeepScanCoordinator({ + run: snapshot, + store, + pluginRoot: plugin, + executor: { + run: async () => + assert.fail("accepted recovery must not execute another model"), + }, + }); + const beforeRecovery = await readFile(merged.resultPath, "utf8"); + const recovered = await resumed.recoverAcceptedDiscoveries(); + assert.deepEqual( + recovered.map((worker) => worker.resultPath), + [discovery.worker.resultPath, second.worker.resultPath], + ); + await rm(path.join(path.dirname(merged.resultPath), "..", "result.json")); + const reducers = await resumed.recoverCompletedReducers(recovered); + assert.equal(reducers.resultPath, merged.resultPath); + assert.deepEqual(reducers.result, merged.result); + assert.equal( + await readFile(merged.resultPath, "utf8"), + beforeRecovery, + "recovery cannot rewrite accepted bytes", + ); + } finally { + await rm(root, { recursive: true, force: true }); + } +} + +for (const [workflowVersion, missing] of [ + ["deep-security-scan/v2", true], + ["deep-security-scan/v1", false], + ["deep-security-scan/v1", true], +]) + await testAcceptedWriteupRecovery(workflowVersion, missing); + +async function testAcceptedWriteupRecovery(workflowVersion, missing) { + const root = await mkdtemp(path.join(tmpdir(), "deep-accepted-writeup-")); + const target = path.join(root, "target"); + const environment = { + ...process.env, + CODEX_HOME: path.join(root, "home"), + CODEX_SECURITY_STATE_DIR: path.join(root, "state"), + }; + const raw = async (args) => { + const selected = [...args]; + if (selected[0] === "begin-deep-scan") { + selected[selected.indexOf("--workflow-version") + 1] = workflowVersion; + } + const { stdout } = await execute( + process.env.PYTHON || "python3", + [path.join(plugin, "scripts/workbench_db.py"), ...selected], + { env: environment, timeout: 30_000, maxBuffer: 4 * 1024 * 1024 }, + ); + return JSON.parse(stdout); + }; + const store = new WorkbenchDeepScanStore(raw); + try { + await mkdir(target); + await writeFile(path.join(target, "fixture.py"), "print('fixture')\n"); + const run = await store.begin({ + targetPath: target, + scope: ".", + threadId: "fixture-owner", + scanRoot: path.join(root, "scans"), + }); + assert.equal(run.workflowVersion, workflowVersion); + const artifacts = createDeepScanArtifacts(run.scanDir); + await ensureDeepScanDirectories(artifacts); + const reportPath = "findings/accepted/accepted.md"; + await mkdir(path.join(run.scanDir, "findings", "accepted"), { + recursive: true, + }); + await writeFile( + path.join(run.scanDir, reportPath), + "Synthetic accepted evidence.\n", + ); + const runner = new DeepScanWorkerRunner({ + run, + store, + artifacts, + pluginRoot: plugin, + signal: new AbortController().signal, + random: () => 0.5, + log: () => {}, + retryDelaysMs: [], + clock: { now: () => Date.now(), sleep: async () => {} }, + executor: { + async run(request) { + await request.onThreadStarted?.("fixture-worker-session"); + await writeFile( + path.join(request.artifactContext.root, "result.json"), + JSON.stringify({ + scanId: run.scanId, + findings: [ + { + ...fixtureFinding("accepted", "fixture.py"), + writeup: { reportPath }, + }, + ], + coverage: { + completeness: "complete", + surfaces: [], + explicitExclusions: [], + deferred: [], + }, + }), + ); + return { threadId: "fixture-worker-session" }; + }, + }, + }); + const accepted = await runner.runDiscoveryWorker( + randomUUID(), + "discovery-1", + ); + assert.equal(accepted.status, "succeeded", accepted.error?.message); + const before = await readFile(accepted.worker.resultPath); + if (missing) await rm(path.join(run.scanDir, reportPath)); + const snapshot = await store.get(run.scanId, "fixture-owner"); + const resumed = new DeepScanCoordinator({ + run: snapshot, + store, + pluginRoot: plugin, + executor: { + run: async () => + assert.fail("accepted recovery must not rerun a model"), + }, + }); + if (missing && workflowVersion === "deep-security-scan/v2") { + await assert.rejects( + resumed.recoverAcceptedDiscoveries(), + /writeup.reportPath/, + ); + } else { + const recovered = await resumed.recoverAcceptedDiscoveries(); + assert.equal(recovered.length, 1); + assert.equal(recovered[0].resultPath, accepted.worker.resultPath); + const result = JSON.parse( + await readFile(recovered[0].resultPath, "utf8"), + ); + assert.equal(result.findings.length, 1); + assert.equal(result.findings[0].writeup.reportPath, reportPath); + } + assert.deepEqual( + await readFile(accepted.worker.resultPath), + before, + "recovery preserves accepted bytes", + ); + } finally { + await rm(root, { recursive: true, force: true }); + } +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_cancel_response.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_cancel_response.ts index bd3402b8a7..0e017bb2e1 100644 --- a/plugins/codex-security/mcp-app/tests/test_deep_scan_cancel_response.ts +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_cancel_response.ts @@ -101,6 +101,9 @@ try { new URL("../server.ts", import.meta.url).href, ), }, + banner: { + js: `import { createRequire as fixtureCreateRequire } from "node:module"; const require = fixtureCreateRequire(${JSON.stringify(new URL("../server.ts", import.meta.url).href)});`, + }, loader: { ".md": "text" }, })) as CancelResponseModule; const server = createCodexSecurityServer(); diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_checkpoint_coverage.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_checkpoint_coverage.mjs new file mode 100644 index 0000000000..6098680e4d --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_checkpoint_coverage.mjs @@ -0,0 +1,83 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { publishCoverageFixture } from "./deep_scan_coverage_fixture.mjs"; + +for (const continueAfterResume of [false, true]) { + test(`immutable discovery receipts survive resumed publication (continued: ${continueAfterResume})`, async () => { + const root = await mkdtemp( + path.join(tmpdir(), "deep-checkpoint-coverage-"), + ); + try { + const fixture = path.join(root, "fixture"); + await mkdir(fixture, { mode: 0o700 }); + const { scanDir } = await publishCoverageFixture(fixture, "partial", { + resume: true, + continueAfterResume, + immutableInputs: true, + }); + const coverage = JSON.parse( + await readFile(path.join(scanDir, "coverage.json"), "utf8"), + ); + assert.equal(coverage.completeness, undefined); + assert.deepEqual( + coverage.reviews.map((review) => review.completeness), + [undefined, undefined, undefined], + ); + assert.equal(coverage.reviews[0].attempt, 2); + assert.equal( + new Set(coverage.deferred.map((item) => item.candidateId)).size, + 2, + ); + const report = await readFile(path.join(scanDir, "report.md"), "utf8"); + for (const item of coverage.deferred) + assert.ok(report.includes(item.reason)); + for (const surface of coverage.surfaces) { + for (const receipt of surface.receiptRefs) { + assert.equal( + await readFile(path.join(scanDir, receipt), "utf8"), + "Synthetic review evidence.\n", + ); + } + } + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +} + +for (const retry of ["missing", "collision"]) { + test(`archived discovery receipt preserves its origin after retry (${retry})`, async () => { + const root = await mkdtemp(path.join(tmpdir(), "deep-retry-coverage-")); + try { + const fixture = path.join(root, "fixture"); + await mkdir(fixture, { mode: 0o700 }); + const { scanDir } = await publishCoverageFixture(fixture, "complete", { + discoveryReceiptRetry: retry, + }); + const coverage = JSON.parse( + await readFile(path.join(scanDir, "coverage.json"), "utf8"), + ); + const archived = coverage.surfaces.find( + (surface) => surface.label === "Archived boundary", + ); + assert.ok(archived, "accepted archived coverage reaches publication"); + assert.equal(archived.receiptRefs.length, 1); + assert.equal( + await readFile(path.join(scanDir, archived.receiptRefs[0]), "utf8"), + "Archived review evidence.\n", + ); + const current = coverage.surfaces.find( + (surface) => surface.label === "Archive route", + ); + assert.equal( + await readFile(path.join(scanDir, current.receiptRefs[0]), "utf8"), + "Synthetic review evidence.\n", + ); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_compatibility.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_compatibility.mjs new file mode 100644 index 0000000000..64e5935ed7 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_compatibility.mjs @@ -0,0 +1,204 @@ +import assert from "node:assert/strict"; +import { build } from "esbuild"; + +const bundle = await build({ + bundle: true, + entryPoints: [ + new URL("../src/deep-scan/registry.ts", import.meta.url).pathname, + ], + format: "esm", + loader: { ".md": "text" }, + platform: "node", + write: false, +}); +const { startOrJoinDeepScanCoordinator, DeepScanRemoteCoordinator } = + await import( + `data:text/javascript;base64,${Buffer.from(bundle.outputFiles[0].contents).toString("base64")}` + ); + +await testUnsupportedWorkflowDoesNotAcquireOwnership(); +await testUnsupportedSelectionDoesNotAcquireOwnership(); + +async function testUnsupportedSelectionDoesNotAcquireOwnership() { + for (const selection of [ + { + workflowVersion: "deep-security-scan/v1", + finalizationInput: { version: 1 }, + }, + { + workflowVersion: "deep-security-scan/v2", + finalizationInput: { version: 99 }, + }, + ]) { + await assert.rejects( + startOrJoinDeepScanCoordinator({ + run: { scanId: "fixture", schemaVersion: 1, ...selection }, + registry: { + get: () => + assert.fail("unsupported selection inspected a live coordinator"), + start: () => + assert.fail("unsupported selection started a coordinator"), + }, + options: { + threadId: "fixture-thread", + prepareExecutor: async () => + assert.fail("unsupported selection resolved settings"), + store: { + claimCoordinator: async () => + assert.fail("unsupported selection acquired ownership"), + }, + }, + }), + /finalization input version/, + ); + } +} + +async function testUnsupportedWorkflowDoesNotAcquireOwnership() { + for (const version of [ + { schemaVersion: 99, workflowVersion: "deep-scan-mcp/v1" }, + { schemaVersion: 1, workflowVersion: "future/v99" }, + ]) { + let mutations = 0; + await assert.rejects( + startOrJoinDeepScanCoordinator({ + run: { scanId: "fixture", ...version }, + registry: { + get: () => undefined, + start: () => { + mutations += 1; + }, + }, + options: { + threadId: "fixture-thread", + store: { + claimCoordinator: async () => { + mutations += 1; + }, + }, + }, + }), + /unsupported workflow or schema version/, + ); + assert.equal(mutations, 0); + } +} + +// A joining client must not resolve or replace the live executor's settings. +for (const workflowVersion of [ + "deep-scan-mcp/v1", + "deep-security-scan/v1", + "deep-security-scan/v2", +]) { + for (const local of [true, false]) { + let preparations = 0; + const run = { scanId: "fixture", status: "running", workflowVersion }; + const options = { + threadId: "fixture-thread", + executor: { marker: "observer" }, + prepareExecutor: async () => { + preparations += 1; + return {}; + }, + store: { claimCoordinator: async () => ({ run, acquired: false }) }, + }; + await startOrJoinDeepScanCoordinator({ + run, + registry: { + get: () => (local ? {} : undefined), + start: () => assert.fail("observer started"), + }, + options, + }); + assert.equal(preparations, 0); + } +} + +// Selected publication has no worker launch and must not need current settings. +for (const selected of [false, true]) { + const run = { + scanId: "fixture", + status: "running", + workflowVersion: "deep-security-scan/v2", + ...(selected ? { finalizationInput: { version: 1 } } : {}), + }; + let preparations = 0; + const fallback = {}; + const restored = {}; + await startOrJoinDeepScanCoordinator({ + run, + registry: { + get: () => undefined, + start: (options) => { + assert.equal(options.executor, selected ? fallback : restored); + assert.equal(options.run, run); + return {}; + }, + }, + options: { + threadId: "fixture-thread", + executor: fallback, + prepareExecutor: async () => { + preparations += 1; + return restored; + }, + store: { claimCoordinator: async () => ({ run, acquired: true }) }, + }, + }); + assert.equal(preparations, selected ? 0 : 1); +} + +const originalNow = Date.now; +try { + let now = 0; + Date.now = () => now; + const run = { + scanId: "fixture", + status: "running", + updatedAt: "1970-01-01T00:00:00Z", + }; + const acquired = { + ...run, + model: "original-model", + coordinatorGeneration: 3, + }; + let preparations = 0; + const executor = { marker: "restored" }; + const registry = { + get: () => undefined, + start: (options) => { + assert.equal(options.run, acquired); + assert.equal(options.executor, executor); + return { wait: async () => ({ ...acquired, status: "succeeded" }) }; + }, + }; + const options = { + threadId: "fixture-thread", + executor: { marker: "observer" }, + prepareExecutor: async (state) => { + assert.equal(state, acquired); + preparations += 1; + return executor; + }, + store: { + get: async () => run, + claimCoordinator: async () => ({ run: acquired, acquired: true }), + }, + }; + await startOrJoinDeepScanCoordinator({ + run: { ...run, startDisposition: "created" }, + registry, + options, + }); + assert.equal(preparations, 1); + const remote = new DeepScanRemoteCoordinator({ run, registry, options }); + now = 60_000; + assert.equal((await remote.wait(undefined, 1_000)).status, "succeeded"); + assert.equal( + preparations, + 2, + "takeover resolves settings from the newly acquired run", + ); +} finally { + Date.now = originalNow; +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.ts index dbc36a059e..3775624ae0 100644 --- a/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.ts +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.ts @@ -1,3 +1,8 @@ +import { testSelectedFinalizationOwner } from "./deep_scan_selected_finalization_cases.ts"; +import { + legacyAcceptedFinding, + legacyRawDiscoveryFinding, +} from "./legacy-accepted-finding.ts"; import { readJson, writeJson } from "./support/json.ts"; import type { CoordinatorOptions } from "../src/deep-scan/coordinator.js"; import type { ScanDraftInput } from "../src/artifact-scan-draft.js"; @@ -10,7 +15,7 @@ import type { } from "../src/deep-scan/types.js"; import { mock } from "node:test"; import assert from "node:assert/strict"; -import { randomUUID } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import { mkdir, readFile, @@ -25,6 +30,7 @@ import { testSuccessfulDeepCoverageRetainsIndependentWorkerReviewStatus, } from "./deep_scan_lifecycle_cases.ts"; import { + DeepScanCoordinator, DeepScanCoordinatorRegistry, WorkbenchDeepScanStore, DeepScanNonRetryableError, @@ -71,6 +77,8 @@ async function testCappedQueueAndSerialDedup() { handoffClaimToken: "claim-fixture", onComplete: async (draft) => void completedDrafts.push(structuredClone(draft)), + onFinalized: async () => + assert.fail("Legacy scans retain explicit parent completion"), }); assert.equal(terminal?.status, "succeeded"); assert.equal(terminal?.terminalReason, "capped"); @@ -2032,7 +2040,7 @@ async function testCancellationDuringDiscoveryAcceptanceRejectsLateSuccess() { const preserved = Promise.withResolvers(); const releasePreservation = Promise.withResolvers(); const coordinator = createCoordinator(fixture, store, executor, { - log: (event) => events.push(event), + log: (event: DeepScanLogEvent) => events.push(event), threadId: "checkpoint-owner", onStopped: async () => { const worker = [...store.workers.values()].find( @@ -2149,7 +2157,7 @@ async function testStoppedPublicationFailurePreservesOriginalDiagnostic() { const events: DeepScanLogEvent[] = []; const coordinator = createCoordinator(fixture, store, executor, { threadId: "checkpoint-owner", - log: (event) => events.push(event), + log: (event: DeepScanLogEvent) => events.push(event), onStopped: async () => { throw new Error( `fixture retained result publication failure ${"y".repeat(2_350)}`, @@ -3144,11 +3152,12 @@ async function testResumedManifestPreservesCompletedReducer( includeUnstartedReducer = false, removeHistoricalPrompts = true, includePaths?: string[], + historicalFindings = false, ) { const fixture = await fixtureRun({ workers: 2, stopAfterNoNew: 2, - maxDiscoveryRuns: 3, + maxDiscoveryRuns: historicalFindings ? 2 : 3, }); fixture.run.includePaths = includePaths; const store = new FakeStore(fixture.run); @@ -3158,7 +3167,11 @@ async function testResumedManifestPreservesCompletedReducer( store, new FakeExecutor({ blockDiscoveryAfterCalls: 2, - discoveryCandidateId: includePaths ? "scope-candidate" : undefined, + discoveryCandidateId: includePaths + ? "scope-candidate" + : historicalFindings + ? "historical-checkpoint" + : undefined, }), { run: store.run }, ); @@ -3210,6 +3223,17 @@ async function testResumedManifestPreservesCompletedReducer( const acceptedWorkers = store.run.persistedWorkers!.filter( (worker) => worker.status === "succeeded", ); + if (historicalFindings) { + for (const worker of acceptedWorkers) { + const result = await readJson(worker.resultManifestPath!); + result.findings = result.findings.map( + worker.kind === "discovery" + ? legacyRawDiscoveryFinding + : legacyAcceptedFinding, + ); + await writeJson(worker.resultManifestPath!, result); + } + } const acceptedResults = await Promise.all( acceptedWorkers.map((worker) => readFile(worker.resultManifestPath!, "utf8"), @@ -3756,6 +3780,30 @@ async function testSaturationIgnoresDiscoveryCancellationWriteFailure( stopAfterNoNew: 2, maxDiscoveryRuns: 6, }); + await mkdir(fixture.run.scanDir, { mode: 0o700 }); + fixture.run.workflowVersion = "deep-security-scan/v2"; + store.run.workflowVersion = "deep-security-scan/v2"; + store.selectFinalization = async (input) => { + const checkpointRoot = path.join( + path.dirname(input.resultPath!), + "checkpoints", + ); + const [name] = await readdir(checkpointRoot); + const checkpoint = path.join(checkpointRoot, name); + const bytes = await readFile(checkpoint); + store.run.finalizationInput = { + version: 1, + resultPath: path + .relative(fixture.run.scanDir, checkpoint) + .split(path.sep) + .join("/"), + resultSha256: createHash("sha256").update(bytes).digest("hex"), + terminalReason: input.reason, + omittedWorkerIds: input.omittedWorkerIds ?? [], + selectedAt: "2026-01-01T00:00:00Z", + }; + return structuredClone(store.run); + }; const executor = new FakeExecutor({ blockDiscoveryAfterCalls: 2, }); @@ -3856,7 +3904,7 @@ async function testSaturationIgnoresDiscoveryCancellationWriteFailure( ); const { coverage, ...publishedReduction } = completed[0]; assert.deepEqual( - publishedReduction, + { ...publishedReduction, sourceCoverage: coverage }, await readJson(acceptedReducer!.resultManifestPath!), "the accepted aggregate still reaches publication when redundant cancellation writes fail", ); @@ -4226,6 +4274,7 @@ async function testQuotaFailureStopsDispatchBeforeWorkerShutdown( } try { + await testSelectedFinalizationOwner(); await testDeepScanLifecycle({ fixtureRun, FakeStore, @@ -4336,6 +4385,12 @@ try { await testResumedManifestPreservesCompletedReducer(false, false); await testResumedManifestPreservesCompletedReducer(); await testResumedManifestPreservesCompletedReducer(true); + await testResumedManifestPreservesCompletedReducer( + false, + true, + undefined, + true, + ); await testResumedManifestPreservesCompletedReducer(false, true, [ "api", "background jobs", diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_coverage_descriptions.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_coverage_descriptions.mjs new file mode 100644 index 0000000000..da8b947fa8 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_coverage_descriptions.mjs @@ -0,0 +1,81 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { publishCoverageFixture } from "./deep_scan_coverage_fixture.mjs"; + +const descriptions = { + source: "source-review", + worker: "focused-reviewer", + task: "Check the filesystem boundary.", +}; +const sourceProvenance = { + ...descriptions, + workerId: "worker-local-label", + attempt: 99, + sourceId: "worker-local-source", + candidateId: "worker-local-candidate", +}; + +for (const options of [ + {}, + { resume: true }, + { resume: true, continueAfterResume: true }, + { resume: true, selectedRecovery: true, discardMutableResults: true }, + { + resume: true, + selectedRecovery: true, + discardMutableResults: true, + splitSeededReducers: true, + }, +]) { + test(`coverage source descriptions survive publication and recovery: ${JSON.stringify(options)}`, async () => { + const root = await mkdtemp( + path.join(tmpdir(), "deep-coverage-descriptions-"), + ); + try { + const fixture = path.join(root, "fixture"); + await mkdir(fixture, { mode: 0o700 }); + const { scanDir } = await publishCoverageFixture(fixture, "partial", { + ...options, + immutableInputs: true, + materialFindings: true, + sourceProvenance, + }); + const coverage = JSON.parse( + await readFile(path.join(scanDir, "coverage.json"), "utf8"), + ); + assert.equal(coverage.completeness, undefined); + assert.equal( + new Set(coverage.deferred.map((item) => item.candidateId)).size, + 2, + ); + for (const field of [ + "surfaces", + "explicitExclusions", + "deferred", + "openQuestions", + ]) { + assert.ok(coverage[field].length > 0); + for (const item of coverage[field]) { + const review = coverage.reviews.find( + (review) => review.workerId === item.provenance.workerId, + ); + assert.ok(review, "host identity names an actual independent review"); + assert.deepEqual(item.provenance, { + ...descriptions, + workerId: review.workerId, + attempt: review.attempt, + ...(field === "surfaces" ? { sourceId: "shared-surface" } : {}), + ...(field === "deferred" + ? { sourceId: "same-id", candidateId: "candidate-1" } + : {}), + }); + } + } + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts index 1f270499c9..60f84250d1 100644 --- a/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts @@ -1,3 +1,7 @@ +import { testRestoredPrivateProfileHomes } from "./deep_scan_private_profile_cases.ts"; +import { testWorkerCancellation } from "./deep_scan_cancellation_cases.ts"; +import { testHome } from "./deep_scan_home_case.ts"; +import { testReconstructedWorkers } from "./deep_scan_reconstructed_cases.ts"; import { windowsAcl } from "./support/windows-output-fixture.ts"; import { createFakeCodexFixture, @@ -13,7 +17,10 @@ import { finding, workerDraft } from "./scan-draft-fixture.ts"; import { assertConfigOverrides, assertFlagPair, + assertReadOnlyWorkerPolicy, + assertWorkerSubagentPolicy, nativeConfigOverrides, + workerPermissionProfileOverride, } from "./assertions.ts"; import { createTemporaryDirectories } from "./support/temporary-directories.ts"; import { mock } from "node:test"; @@ -70,6 +77,10 @@ const { CodexSdkWorkerExecutor, resolveCodexPath, snapshotWorkerEnvironment, + WorkbenchDeepScanStore, + captureDeepScanExecutionSettings, + loadDeepScanExecutionSettings, + restoredDeepScanWorkerSettings, appendItemDiagnostic, classifyCodexWorkerError, codexWorkerTurnError, @@ -85,6 +96,8 @@ const { contents: `${await readFile(executorSource, "utf8")} export { snapshotWorkerEnvironment, appendItemDiagnostic }; export * from "./errors.js"; +export * from "./recovery-settings.js"; +export { WorkbenchDeepScanStore } from "./store.js"; export { parseDeepScan } from "./response.js";`, loader: "ts", resolveDir: path.dirname(fileURLToPath(executorSource)), @@ -93,7 +106,7 @@ export { parseDeepScan } from "./response.js";`, }); const temporaryDirectories = createTemporaryDirectories(true); -const fakeCodexFixture = createFakeCodexFixture.bind( +export const fakeCodexFixture = createFakeCodexFixture.bind( null, temporaryDirectories, ); @@ -105,11 +118,35 @@ try { testCodeModeFrameDiagnosticBoundaries(); await testLargeScopeTransport(); await testOpenAiCredentialsReachWorker(); - await testWorkerCancellation(); + await testWorkerCancellation({ + CodexSdkWorkerExecutor, + fakeCodexFixture, + trustedParentSandbox, + restoreEnv, + }); testUsageLimitClassification(); await testWorkerTerminalEvents(); await testWorkerRuntimeSettings(); await testUnsupportedProviderSnapshotFailsBeforeLaunch(); + await testReconstructedWorkers({ + CodexSdkWorkerExecutor, + WorkbenchDeepScanStore, + captureDeepScanExecutionSettings, + loadDeepScanExecutionSettings, + restoredDeepScanWorkerSettings, + fakeCodexFixture, + trustedParentSandboxWithDenials, + deniedWorkerPermissionProfile, + restoreEnv, + }); + await testRestoredPrivateProfileHomes({ + CodexSdkWorkerExecutor, + captureDeepScanExecutionSettings, + restoredDeepScanWorkerSettings, + fakeCodexFixture, + workerRequest, + emptyWorkerPermissionProfile, + }); await testWorkerCyberAccessSettings({ CodexSdkWorkerExecutor, fakeCodexFixture, @@ -132,6 +169,14 @@ try { await testRuntimePermissionProfileFallbackStopsAndDiscards(); await testWorkerLaunchesWithoutGlobalCodex(); await testPreflightBindsExecutableAndHomeBeforeChangingCwd(); + await testHome( + CodexSdkWorkerExecutor, + captureDeepScanExecutionSettings, + restoredDeepScanWorkerSettings, + loadDeepScanExecutionSettings, + withWorkerFixture, + trustedParentSandbox, + ); await testConcurrentWorkerExecutableAliases({ CodexSdkWorkerExecutor, fakeCodexFixture, @@ -144,6 +189,8 @@ try { await testBedrockCredentialsReachWorker("oidc"); await testExternalProviderCredentialsReachWorker(); await testArtifactServerUsesExtendedStartupTimeout(); + await testReducerCoveragePersistenceBinding(); + await testWorkerProviderSelection(); await testZeroSubagentsPreservesHostRestrictions(); await testSdkResumesExistingThread(); await testRetryNotificationDoesNotInterruptTurn(); @@ -424,7 +471,7 @@ async function testWindowsLongExecutableLaunches() { ); children.push(child); return child; - }) as typeof childProcess.spawn; + }) as unknown as typeof childProcess.spawn; syncBuiltinESMExports(); try { process.env.CODEX_HOME = codexHome; @@ -1865,7 +1912,7 @@ env_key = "SYNTHETIC_GATEWAY_KEY"`, : args, { ...options, env: environment }, ); - }) as typeof childProcess.spawn; + }) as unknown as typeof childProcess.spawn; syncBuiltinESMExports(); const nodePaths = settings.map((_, index) => path.join(fixture.root, `selected node ${index}`, "node"), @@ -1889,7 +1936,6 @@ env_key = "SYNTHETIC_GATEWAY_KEY"`, }, }), ); - // A running coordinator retains its settings if the source file changes. for (const kind of ["discovery", "dedup"] as const) { for (const resumeThreadId of [undefined, "fixture-resumed-thread"]) { launches.length = 0; @@ -2698,6 +2744,27 @@ async function testZeroSubagentsPreservesHostRestrictions() { } async function testLargeScopeTransport() { + const engineSource = new URL("../src/deep-scan/engine.ts", import.meta.url); + const { startDeepScanEngine } = await importModule({ + entryPoints: [fileURLToPath(engineSource)], + loader: { ".md": "text" }, + define: { "import.meta.url": JSON.stringify(engineSource.href) }, + banner: { + js: `import { createRequire as engineScopeRequire } from "node:module"; const require = engineScopeRequire(${JSON.stringify(engineSource.href)});`, + }, + plugins: [ + { + name: "capture-coordinator-options", + setup(build) { + build.onLoad({ filter: /[/\\]registry\.ts$/ }, () => ({ + contents: + "export function startOrJoinDeepScanCoordinator(options) { return options; }", + loader: "js", + })); + }, + }, + ], + }); const names = [ "CODEX_CLI_PATH", "CODEX_HOME", @@ -2757,36 +2824,34 @@ async function testLargeScopeTransport() { const filePaths = includePaths.filter( (_, index) => index % 2 === 0, ); - const run = parseDeepScan({ - deepScan: { - scanId, - status: "running", - targetPath: fixture.root, - scope: ".", - includePaths, - filePaths, - scanDir: workerRoot, - config: { - workers: 1, - subagents: 0, - stopAfterNoNew: 1, - maxDiscoveryRuns: 1, - }, - dispatchedCount: 0, - noNewStreak: 0, + const run = { + scanId, + scanDir: workerRoot, + targetPath: fixture.root, + scope: ".", + includePaths, + filePaths, + executionSettings: { + version: 1, + settings: await captureDeepScanExecutionSettings( + {}, + trustedParentSandbox, + ), }, - }); - const executor = new CodexSdkWorkerExecutor({ + }; + const captured = startDeepScanEngine({ + run, + store: {}, + registry: {}, + runWorkbench: async () => ({}), + pluginRoot: fixture.root, + pythonCommand: process.env.PYTHON || "python3", parentSandbox: trustedParentSandbox, - artifactContext: { - pluginRoot: fixture.root, - repoRoot: run.targetPath, - scanId: run.scanId, - scope: run.scope, - includePaths: run.includePaths, - filePaths: run.filePaths, - }, + threadId: "scope-parent", }); + const executor = resumed + ? await captured.options.prepareExecutor(run) + : captured.options.executor; await executor.run( workerRequest(prompt, output, { kind, @@ -3408,12 +3473,7 @@ async function completedWorkerSettlesWithoutWaitingForProcessExit( const promptPath = path.join(fixture.root, "prompt.md"); const workingDirectory = path.join(fixture.root, "artifacts"); await mkdir(workingDirectory); - await writeFile( - promptPath, - kind === undefined - ? "COMPLETE_THEN_HANG\n" - : "COMPLETE_THEN_HANG_IGNORE_TERMINATION\n", - ); + await writeFile(promptPath, "COMPLETE_THEN_HANG\n"); execution = new CodexSdkWorkerExecutor({ parentSandbox: trustedParentSandbox, }).run({ @@ -3452,9 +3512,7 @@ async function completedWorkerSettlesWithoutWaitingForProcessExit( await new Promise((resolve) => setTimeout(resolve, 250)); assert.deepEqual(unexpectedErrors, []); - if (kind === undefined) { - assert.throws(() => process.kill(childPid, 0), { code: "ESRCH" }); - } + assert.throws(() => process.kill(childPid, 0), { code: "ESRCH" }); } finally { clearTimeout(timeout); controller.abort("completed worker fixture cleanup"); @@ -3685,61 +3743,6 @@ async function runFixtureWorker( }); } -function assertReadOnlyWorkerPolicy(args: readonly string[]) { - assert.equal(args.includes("--sandbox"), false); - assert.equal(args.includes("--add-dir"), false); - assert.deepEqual( - args.filter((arg: string) => arg.startsWith("approval_policy=")), - ['approval_policy="never"'], - ); - assert.equal( - args.some((arg: string) => arg.includes("network_access")), - false, - ); - const override = workerPermissionProfileOverride(args); - assertConfigOverrides(args, { - "permissions.codex_security_deep_scan_worker.extends": ":read-only", - "permissions.codex_security_deep_scan_worker.filesystem.:root": "read", - "permissions.codex_security_deep_scan_worker.network.enabled": false, - }); - assert.equal(override.includes('"write"'), false); -} - -function workerPermissionProfileOverride(args: readonly string[]) { - assert.deepEqual( - args.filter((arg: string) => arg.startsWith("default_permissions=")), - ['default_permissions="codex_security_deep_scan_worker"'], - ); - const overrides = args.filter((arg: string) => - arg.startsWith("permissions.codex_security_deep_scan_worker="), - ); - assert.equal(overrides.length, 1); - return overrides[0]; -} - -function assertWorkerSubagentPolicy( - args: readonly string[], - subagents: number, -) { - assertConfigOverrides(args, { - "features.multi_agent_v2.enabled": false, - "features.multi_agent_v2.max_concurrent_threads_per_session": subagents + 1, - "features.multi_agent": undefined, - "features.code_mode.excluded_tool_namespaces": undefined, - ...(subagents === 0 - ? { - "agents.max_threads": undefined, - "features.enable_fanout": false, - "features.code_mode.enabled": undefined, - } - : { - "agents.max_threads": subagents, - "features.enable_fanout": undefined, - }), - }); - assert.equal(args.includes("features.multi_agent_v2.enabled=true"), false); -} - function restoreEnv(name: string, value: string | undefined) { if (value === undefined) delete process.env[name]; else process.env[name] = value; @@ -3766,162 +3769,73 @@ async function withWorkerFixture( } } -async function testWorkerCancellation() { +async function testWorkerProviderSelection() { const fixture = await fakeCodexFixture(); - const previousPath = process.env.CODEX_CLI_PATH; + const saved = Object.fromEntries( + [ + "CODEX_CLI_PATH", + "CODEX_SECURITY_CONFIG_PATH", + "OPENAI_API_KEY", + "CODEX_API_KEY", + ].map((name) => [name, process.env[name]]), + ); const originalSpawn = childProcess.spawn; - let worker: ChildProcess | undefined; - let workerSignal: AbortSignal | undefined; - let cancelDuringCleanup: (() => void) | undefined; - let timeout: NodeJS.Timeout | undefined; - childProcess.spawn = (( - command: string, - args: readonly string[] = [], - options: SpawnOptions = {}, - ) => { - const child = originalSpawn( - command, - command === process.execPath || - command === path.toNamespacedPath(process.execPath) - ? [fixture.executablePath, ...args] - : args, - options, - ); - if (args[0] === "exec") { - assertFlagPair(args, "--thread-source", "security_scan"); - worker = child; - workerSignal = options.signal; - const kill = child.kill; - child.kill = function (...args: Parameters) { - cancelDuringCleanup?.(); - return kill.apply(this, args); - }; - } - return child; - }) as typeof childProcess.spawn; - syncBuiltinESMExports(); - process.env.CODEX_CLI_PATH = process.execPath; try { + delete process.env.OPENAI_API_KEY; + delete process.env.CODEX_API_KEY; + const configPath = path.join(fixture.root, "scan config.toml"); const promptPath = path.join(fixture.root, "prompt.md"); - const workingDirectory = path.join(fixture.root, "artifacts"); - await mkdir(workingDirectory); - const executor = new CodexSdkWorkerExecutor({ - parentSandbox: trustedParentSandbox, - artifactContext: { - pluginRoot: fixture.root, - repoRoot: fixture.root, - scanId: randomUUID(), - includePaths: ["api"], + await writeFile(promptPath, "fixture provider selection"); + process.env.CODEX_CLI_PATH = process.execPath; + process.env.CODEX_SECURITY_CONFIG_PATH = configPath; + childProcess.spawn = (( + command: string, + args: readonly string[] = [], + options: SpawnOptions = {}, + ) => + originalSpawn( + command, + command === process.execPath || + command === path.toNamespacedPath(process.execPath) + ? [fixture.executablePath, ...args] + : args, + options, + )) as unknown as typeof childProcess.spawn; + syncBuiltinESMExports(); + for (const config of [ + { model_provider: "fixture-provider" }, + { + profile: "selected.profile", + profiles: { + "selected.profile": { model_provider: "fixture-provider" }, + }, }, - }); - const run = (controller: AbortController, onThreadStarted?: () => void) => - executor.run({ - kind: "discovery", - promptPath, - workingDirectory, - subagents: 0, - signal: controller.signal, - onThreadStarted, - artifactContext: { root: workingDirectory, layout: "worker" }, - }); - const preAborted = new AbortController(); - const beforeStartup = new Error( - "coordinator canceled before worker startup", - ); - preAborted.abort(beforeStartup); - await assert.rejects( - run(preAborted), - (error: Error) => error === beforeStartup, - ); - assert.equal(worker, undefined); - assert.equal( - (await readdir(fixture.root)).some((name) => name.startsWith("scope-")), - false, - ); - await assert.rejects(readFile(fixture.preflightMarkerPath), { - code: "ENOENT", - }); - await assert.rejects(readFile(fixture.markerPath), { code: "ENOENT" }); - - for (const prompt of [ - "BLOCK_AFTER_START", - "COMPLETE_THEN_HANG", - "FAIL_THEN_HANG", ]) { - await writeFile(promptPath, `${prompt}\n`); - const controller = new AbortController(); - const cancellation = new Error( - "coordinator canceled its remaining workers", - ); - // Exercise cancellation inside the real SDK's iterator cleanup, before kill returns. - cancelDuringCleanup = - prompt === "COMPLETE_THEN_HANG" - ? () => controller.abort(cancellation) - : undefined; - const deadline = new Promise((_, reject) => { - timeout = setTimeout( - () => reject(new Error(`${prompt} worker did not settle`)), - 10_000, - ); + await writeFile(configPath, stringifyToml(config)); + const executor = new CodexSdkWorkerExecutor({ + parentSandbox: trustedParentSandbox, }); - const execution = Promise.race([ - run(controller, () => { - if (prompt === "BLOCK_AFTER_START") controller.abort(cancellation); - }), - deadline, - ]); - if (prompt === "BLOCK_AFTER_START") { - await assert.rejects( - execution, - (error: Error) => - error?.name === "AbortError" || - /abort|SIGTERM/i.test(error?.message ?? ""), + for (const kind of ["discovery", "dedup"]) { + await executor.run({ + kind, + promptPath, + workingDirectory: fixture.root, + subagents: 0, + signal: new AbortController().signal, + }); + const invocation = JSON.parse( + await readFile(fixture.markerPath, "utf8"), + ); + assert.equal( + invocation.argv.includes('model_provider="fixture-provider"'), + true, ); - assert(workerSignal); - assert.equal(workerSignal.aborted, true); - assert.equal(workerSignal.reason, cancellation); - } else { - if (prompt === "FAIL_THEN_HANG") { - await assert.rejects(execution, /fixture worker failed/); - controller.abort(cancellation); - } else { - const result = await execution; - assert.equal(result.threadId, "fixture-thread-id"); - } - assert.equal(controller.signal.aborted, true); - assert(workerSignal); - assert.equal(workerSignal.aborted, false); - } - assert.equal( - (await readdir(fixture.root)).some((name) => name.startsWith("scope-")), - false, - ); - assert.notEqual(workerSignal, controller.signal); - const currentWorker = worker as ChildProcess | undefined; - assert(currentWorker); - assert.equal(currentWorker.killed, true); - if ( - currentWorker.exitCode === null && - currentWorker.signalCode === null - ) { - await Promise.race([ - new Promise((resolve) => - worker!.once("close", () => resolve()), - ), - deadline, - ]); } - clearTimeout(timeout); - cancelDuringCleanup = undefined; } } finally { - clearTimeout(timeout); - cancelDuringCleanup = undefined; - if (worker && worker.exitCode === null && worker.signalCode === null) - worker.kill("SIGKILL"); childProcess.spawn = originalSpawn; syncBuiltinESMExports(); - restoreEnv("CODEX_CLI_PATH", previousPath); + for (const [name, value] of Object.entries(saved)) restoreEnv(name, value); } } @@ -4027,3 +3941,94 @@ async function testWorkerTerminalEvents() { restoreEnv("CODEX_CLI_PATH", previousPath); } } + +async function testReducerCoveragePersistenceBinding() { + const fixture = await fakeCodexFixture(); + const previousPath = process.env.CODEX_CLI_PATH; + const previousMarker = process.env.FAKE_CODEX_MARKER; + process.env.CODEX_CLI_PATH = fixture.executablePath; + try { + const promptPath = path.join(fixture.root, "prompt.md"); + const workingDirectory = path.join(fixture.root, "artifacts"); + await mkdir(workingDirectory); + await writeFile(promptPath, "fixture reducer prompt\n"); + const launches: Promise[] = []; + for (const resume of [false, true]) { + for (const persistSourceCoverage of [false, true]) { + const markerPath = path.join( + fixture.root, + `coverage-${resume}-${persistSourceCoverage}.json`, + ); + process.env.FAKE_CODEX_MARKER = markerPath; + const scanRoot = path.join( + fixture.root, + `scan-${resume}-${persistSourceCoverage}`, + ); + const deepReducer = { + scanRoot, + claimedWorkers: [ + { + id: "worker-1", + attempt: 2, + resultPath: path.join( + scanRoot, + "worker", + "checkpoints", + "accepted.json", + ), + artifactDir: path.join(scanRoot, "worker"), + }, + ], + persistSourceCoverage, + }; + const launch = new CodexSdkWorkerExecutor({ + parentSandbox: trustedParentSandbox, + artifactContext: { + pluginRoot: fixture.root, + scanRoot: deepReducer.scanRoot, + repoRoot: fixture.root, + scanId: "fixture-scan-id", + }, + }).run({ + kind: "dedup", + promptPath, + workingDirectory, + subagents: 0, + signal: new AbortController().signal, + ...(resume + ? { + resumeThreadId: "fixture-existing-thread", + continuationPrompt: "continue the reducer\n", + } + : {}), + artifactContext: { + root: workingDirectory, + layout: "reducer", + deepReducer, + }, + }); + launches.push( + launch.then(async () => { + const invocation = JSON.parse(await readFile(markerPath, "utf8")); + const servers = parseToml( + invocation.argv + .filter((arg: string) => arg.startsWith("mcp_servers=")) + .join("\n"), + ).mcp_servers as Record }>; + const encoded = + servers.cs_artifacts.env.CODEX_SECURITY_REDUCER_CONTEXT_JSON; + assert.ok( + encoded, + "the launched reducer receives its host-bound artifact context", + ); + assert.deepEqual(JSON.parse(encoded), deepReducer); + }), + ); + } + } + await Promise.all(launches); + } finally { + restoreEnv("CODEX_CLI_PATH", previousPath); + restoreEnv("FAKE_CODEX_MARKER", previousMarker); + } +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_finalization.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_finalization.mjs new file mode 100644 index 0000000000..bcdbb65dde --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_finalization.mjs @@ -0,0 +1,333 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { promises as fs, renameSync, symlinkSync } from "node:fs"; +import { syncBuiltinESMExports } from "node:module"; +import { + mkdir, + mkdtemp, + readFile, + readdir, + realpath, + rm, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { build } from "esbuild"; + +const bundle = await build({ + stdin: { + contents: `export * from "./src/deep-scan/finalization.ts"; + export * from "./src/deep-scan/artifacts.ts"; + export * from "./src/artifact-scan-draft.ts"; + export { parseDeepReduction } from "./src/deep-scan/artifact-validation.ts"; + export { legacyAcceptedFinding } from "./tests/legacy-accepted-finding.ts";`, + resolveDir: path.resolve(import.meta.dirname, ".."), + }, + bundle: true, + format: "esm", + platform: "node", + write: false, + footer: { js: "//# sourceURL=deep-scan-finalization-contract.js" }, +}); +const { + publishSelectedDeepScan, + readSelectedDeepScanDraft, + createDeepScanArtifacts, + saveScanDraftCheckpoint, + parseDeepReduction, + parseScanDraft, + legacyAcceptedFinding, +} = await import( + `data:text/javascript;base64,${Buffer.from(bundle.outputFiles[0].contents).toString("base64")}` +); +const scanId = "4e7b4acb-ac80-4d68-98cd-3d5ac5581cd1"; + +for (const rootCause of ["object", "text"]) { + test(`replays accepted legacy finding details with ${rootCause} root cause`, async () => { + const scanDir = await realpath( + await mkdtemp(path.join(tmpdir(), "selected-legacy-finding-")), + ); + try { + const finding = legacyAcceptedFinding(); + if (rootCause === "text") finding.root_cause = "Synthetic root cause"; + const saved = { + scanId, + findings: [finding], + sourceCoverage: { surfaces: [], explicitExclusions: [], deferred: [] }, + }; + const contents = JSON.stringify(saved); + const selected = path.join(scanDir, "selected.json"); + await writeFile(selected, contents); + const draft = await readSelectedDeepScanDraft( + createDeepScanArtifacts(scanDir), + scanId, + { + version: 1, + resultPath: "selected.json", + resultSha256: createHash("sha256").update(contents).digest("hex"), + terminalReason: "capped", + omittedWorkerIds: [], + selectedAt: "2030-01-01T00:00:00Z", + }, + ); + assert.deepEqual(draft.findings, saved.findings); + assert.equal(await readFile(selected, "utf8"), contents); + assert.throws(() => parseScanDraft(draft)); + assert.throws(() => parseDeepReduction(saved)); + + const invalid = structuredClone(saved); + invalid.findings[0].attackPath.dataFlow.evidence_refs = ["missing"]; + assert.throws( + () => parseDeepReduction(invalid, true), + /attackPath\.dataFlow\.evidence_refs.*existing code-evidence IDs/, + ); + } finally { + await rm(scanDir, { recursive: true, force: true }); + } + }); +} + +for (const mode of ["normal", "symlink", "swapped", "changed", "canceled"]) { + test(`selected reducer read preserves accepted-file boundary: ${mode}`, async (t) => { + const root = await realpath( + await mkdtemp(path.join(tmpdir(), "selected-reader-")), + ); + try { + const scanDir = path.join(root, "scan"); + await mkdir(scanDir, { mode: 0o700 }); + const selected = path.join(scanDir, "selected.json"); + const outside = path.join(root, "outside.json"); + const contents = JSON.stringify({ + scanId, + complete: true, + findings: [], + sourceCoverage: { + completeness: "complete", + surfaces: [], + explicitExclusions: [], + deferred: [], + }, + }); + await writeFile(selected, contents); + await writeFile(outside, contents); + const selection = { + version: 1, + resultPath: "selected.json", + resultSha256: createHash("sha256").update(contents).digest("hex"), + terminalReason: "capped", + omittedWorkerIds: [], + selectedAt: "2030-01-01T00:00:00Z", + }; + const swap = () => { + renameSync(selected, `${selected}.original`); + symlinkSync(outside, selected, "file"); + }; + if (mode === "symlink") swap(); + if (mode === "changed") await fs.appendFile(selected, "\n"); + const controller = new AbortController(); + if (mode === "canceled") + controller.abort(new Error("Stopped selected publication")); + const lstat = fs.lstat; + let swapped = false; + const spy = t.mock.method(fs, "lstat", async (...args) => { + const result = await lstat(...args); + if (mode === "swapped" && args[0] === selected && !swapped) { + swap(); + swapped = true; + } + return result; + }); + syncBuiltinESMExports(); + try { + const reading = readSelectedDeepScanDraft( + createDeepScanArtifacts(scanDir), + scanId, + selection, + controller.signal, + ); + if (mode === "normal") assert.equal((await reading).scanId, scanId); + else + await assert.rejects( + reading, + mode === "canceled" + ? /Stopped selected publication/ + : mode === "changed" + ? /changed after acceptance/ + : /file|directory|symlink/, + ); + if (mode === "swapped") assert.equal(swapped, true); + } finally { + spy.mock.restore(); + syncBuiltinESMExports(); + } + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +} + +for (const terminalReason of ["capped", "saturated"]) { + test(`replays the selected ${terminalReason} input after replaceable results change`, async () => { + const scanDir = await realpath( + await mkdtemp(path.join(tmpdir(), "selected-finalization-")), + ); + try { + const artifacts = createDeepScanArtifacts(scanDir); + const root = path.join(artifacts.dedupRoot, "dedup-0001", "output"); + await mkdir(root, { recursive: true }); + const draft = { + scanId, + complete: true, + findings: [], + coverage: { + completeness: "partial", + surfaces: [], + explicitExclusions: [], + deferred: [ + { id: "review", reason: "A dependency remains unreviewed." }, + ], + }, + }; + const { coverage, ...reduction } = draft; + await saveScanDraftCheckpoint( + { root, repoRoot: scanDir, layout: "reducer" }, + { + ...reduction, + sourceCoverage: coverage, + }, + ); + delete draft.coverage.completeness; + const [name] = await readdir(path.join(root, "checkpoints")); + const checkpoint = path.join(root, "checkpoints", name); + const contents = await readFile(checkpoint); + const selection = { + version: 1, + resultPath: path + .relative(scanDir, checkpoint) + .split(path.sep) + .join("/"), + resultSha256: createHash("sha256").update(contents).digest("hex"), + terminalReason, + omittedWorkerIds: [], + selectedAt: "2026-01-01T00:00:00Z", + }; + // A replacement result is not a new finalization selection. + await writeFile( + path.join(root, "result.json"), + JSON.stringify({ ...draft, complete: false }), + ); + assert.deepEqual( + await readSelectedDeepScanDraft(artifacts, scanId, selection), + draft, + ); + assert.deepEqual( + await readSelectedDeepScanDraft(artifacts, scanId, selection), + draft, + ); + await assert.rejects( + readSelectedDeepScanDraft( + artifacts, + "e14e9229-653a-4385-bec0-8745f0b037cb", + selection, + ), + /complete result for this scan/, + ); + await writeFile(checkpoint, JSON.stringify({ ...draft, findings: [] })); + await assert.rejects( + readSelectedDeepScanDraft(artifacts, scanId, selection), + /changed after acceptance/, + ); + } finally { + await rm(scanDir, { recursive: true, force: true }); + } + }); +} + +test("retains unresolved coverage for a persisted zero-success deadline selection", async () => { + const selection = { + version: 1, + resultPath: null, + resultSha256: null, + terminalReason: "capped", + omittedWorkerIds: [], + selectedAt: "2026-01-01T00:00:00Z", + }; + const result = await readSelectedDeepScanDraft( + createDeepScanArtifacts("unused"), + scanId, + selection, + ); + assert.equal(result.scanId, scanId); + assert.deepEqual(result.findings, []); + assert.equal(result.coverage.completeness, undefined); + assert.equal(result.coverage.deferred.length, 1); + await assert.rejects( + readSelectedDeepScanDraft(createDeepScanArtifacts("unused"), scanId, { + ...selection, + terminalReason: "saturated", + }), + /recorded discovery deadline/, + ); +}); + +for (const status of ["failed", "canceled", "interrupted"]) { + test(`saved selection does not turn a ${status} scan into success`, async () => { + await assert.rejects( + publishSelectedDeepScan({ + run: { + scanId, + scanDir: "unused", + workflowVersion: "deep-security-scan/v2", + status, + finalizationInput: { + version: 1, + resultPath: null, + resultSha256: null, + terminalReason: "capped", + omittedWorkerIds: [], + selectedAt: "2026-01-01T00:00:00Z", + }, + }, + artifacts: createDeepScanArtifacts("unused"), + signal: new AbortController().signal, + publish: async () => + assert.fail("Stopped work cannot publish successful results"), + finish: async () => + assert.fail("Stopped work cannot finish successfully"), + }), + /Stopped Deep Scan/, + ); + }); +} + +test("cancellation prevents selected publication and preserves its input", async () => { + const controller = new AbortController(); + controller.abort("cost limit or user cancellation"); + const selection = { + version: 1, + resultPath: null, + resultSha256: null, + terminalReason: "capped", + omittedWorkerIds: [], + selectedAt: "2026-01-01T00:00:00Z", + }; + await assert.rejects( + publishSelectedDeepScan({ + run: { + scanId, + scanDir: "unused", + workflowVersion: "deep-security-scan/v2", + status: "running", + finalizationInput: selection, + }, + artifacts: createDeepScanArtifacts("unused"), + signal: controller.signal, + publish: async () => assert.fail("Canceled work cannot publish"), + finish: async () => assert.fail("Canceled work cannot finish"), + }), + (error) => error === controller.signal.reason, + ); + assert.equal(selection.terminalReason, "capped"); +}); diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_finite_publication.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_finite_publication.ts index b296f26d04..fae9184377 100644 --- a/plugins/codex-security/mcp-app/tests/test_deep_scan_finite_publication.ts +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_finite_publication.ts @@ -18,6 +18,9 @@ import { finding, workerDraft } from "./scan-draft-fixture.ts"; const { startDeepScanEngine } = await importSource("src/deep-scan/engine.ts", { loader: { ".md": "text" }, + banner: { + js: `import { createRequire as fixtureRequire } from "node:module"; const require = fixtureRequire(${JSON.stringify(import.meta.url)});`, + }, define: { "import.meta.url": JSON.stringify( new URL("../src/deep-scan/engine.ts", import.meta.url).href, @@ -31,6 +34,13 @@ after(() => temporaryDirectories.cleanup()); test("Deep engine preserves stopped results for an option-shaped thread ID", async () => { const fixture = await fixtureRun(); + fixture.run.executionSettings = { + version: 1, + settings: { + codexPath: process.execPath, + codexHome: join(fixture.pluginRoot, "codex-home"), + }, + }; const store = new FakeStore(fixture.run); store.run.coordinatorGeneration = 7; const handoffClaimToken = randomUUID(); @@ -76,6 +86,13 @@ test("Deep engine preserves stopped results for an option-shaped thread ID", asy for (const route of ["mcp", "file"] as const) { test(`Deep engine publishes historical metadata inherited by a ${route} reducer`, async () => { const fixture = await fixtureRun({ maxDiscoveryRuns: 1 }); + fixture.run.executionSettings = { + version: 1, + settings: { + codexPath: process.execPath, + codexHome: join(fixture.pluginRoot, "codex-home"), + }, + }; const store = new FakeStore(fixture.run); const registry = new DeepScanCoordinatorRegistry(); const run = store.run; diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_handoff_races.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_handoff_races.ts index 43af8d9616..03d38012eb 100644 --- a/plugins/codex-security/mcp-app/tests/test_deep_scan_handoff_races.ts +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_handoff_races.ts @@ -247,7 +247,9 @@ try { } else { assert.equal( reads, - scenario === "terminal state" || userCanceled ? 1 : 3, + // A replacement observer re-reads terminal state, including the + // injected transient failure, before releasing the old coordinator. + userCanceled ? 1 : 3, ); if (acceptance) assert.equal(executions, 2); if (concurrent) { diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_host_shutdown.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_host_shutdown.ts index 25ea53ebd0..be01c9ef02 100644 --- a/plugins/codex-security/mcp-app/tests/test_deep_scan_host_shutdown.ts +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_host_shutdown.ts @@ -1,7 +1,9 @@ import assert from "node:assert/strict"; import { readFile } from "node:fs/promises"; +import { join } from "node:path"; import { mock, test } from "node:test"; import { fileURLToPath } from "node:url"; +import type { DeepScanExecutionSettings } from "../src/deep-scan/recovery-settings.js"; import { Client } from "@modelcontextprotocol/sdk/client/index.js"; import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; import { importModule } from "./import-module.ts"; @@ -47,12 +49,17 @@ const { createCodexSecurityServer } = await importModule({ name: "synthetic-worker", setup(build) { build.onLoad({ filter: /[/\\]deep-scan[/\\]executor\.ts$/ }, () => ({ - contents: `export class CodexSdkWorkerExecutor { constructor() { return ${fixtureAccess}.executor; } }`, + contents: `export class CodexSdkWorkerExecutor { constructor() { return ${fixtureAccess}.executor; } } + export const resolveCodexPath = () => process.execPath; + export const workerRuntimeSettingsFromConfig = () => ({});`, loader: "ts", })); }, }, ], + banner: { + js: `import { createRequire as fixtureRequire } from "node:module"; const require = fixtureRequire(${JSON.stringify(import.meta.url)});`, + }, define: { __dirname: JSON.stringify(applicationRoot), "import.meta.url": JSON.stringify( @@ -77,6 +84,12 @@ try { const signals: AbortSignal[] = []; let claims = 0; let now = 0; + const originalEnvironment = process.env; + process.env = { + ...process.env, + CODEX_HOME: join(fixture.pluginRoot, "codex-home"), + }; + delete process.env.CODEX_SECURITY_CONFIG_PATH; const hold = async () => { entered.resolve(); await release.promise; @@ -86,7 +99,16 @@ try { registry, executor, store: Object.assign(store, { - begin: async () => { + begin: async ({ + executionSettings, + }: { + executionSettings?: DeepScanExecutionSettings | null; + }) => { + assert.ok(executionSettings); + store.run.executionSettings ??= { + version: 1, + settings: executionSettings, + }; if (stage === "begin" || stage === "queued") await hold(); return structuredClone(store.run); }, @@ -205,6 +227,7 @@ try { await Promise.allSettled([coordinator?.settled()]); await server.close(); clock.mock.restore(); + process.env = originalEnvironment; Reflect.deleteProperty(globalThis, fixtureKey); } }); diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_material_coverage.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_material_coverage.mjs new file mode 100644 index 0000000000..1766e93485 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_material_coverage.mjs @@ -0,0 +1,78 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { + materialRemediations, + materialRemediationTests, + publishCoverageFixture, +} from "./deep_scan_coverage_fixture.mjs"; + +for (const [resume, continueAfterResume] of [ + [false, false], + [true, false], + [true, true], +]) { + test(`material fixes and unresolved coverage survive canonical publication (resume: ${resume}, continued: ${continueAfterResume})`, async () => { + const root = await mkdtemp(path.join(tmpdir(), "deep-material-coverage-")); + try { + const fixture = path.join(root, "fixture"); + await mkdir(fixture, { mode: 0o700 }); + const { scanDir } = await publishCoverageFixture(fixture, "partial", { + resume, + continueAfterResume, + immutableInputs: true, + materialFindings: true, + }); + const { findings } = JSON.parse( + await readFile(path.join(scanDir, "findings.json"), "utf8"), + ); + assert.equal(findings.length, 1); + const sources = findings[0].provenance.sourceFindings; + assert.deepEqual( + sources.map((source) => source.finding.remediation), + materialRemediations, + ); + assert.equal(new Set(sources.map((source) => source.id)).size, 2); + const report = await readFile(path.join(scanDir, "report.md"), "utf8"); + for (const text of [ + ...materialRemediations, + ...materialRemediationTests, + ]) { + assert.equal( + report.split(text).length - 1, + 1, + `canonical report retains ${text}`, + ); + } + const coverage = JSON.parse( + await readFile(path.join(scanDir, "coverage.json"), "utf8"), + ); + assert.equal(coverage.completeness, undefined); + assert.deepEqual( + coverage.reviews.map((review) => review.completeness), + [undefined, undefined, undefined], + ); + assert.equal( + coverage.deferred.length, + 2, + "a finding does not discharge independent unresolved work", + ); + assert.equal( + new Set(coverage.deferred.map((item) => item.candidateId)).size, + 2, + ); + for (const item of coverage.deferred) + assert.ok(report.includes(item.reason)); + for (const surface of coverage.surfaces) { + assert.equal( + await readFile(path.join(scanDir, surface.receiptRefs[0]), "utf8"), + "Synthetic review evidence.\n", + ); + } + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_publication_integration.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_publication_integration.ts index 462e191ccd..b007c19c59 100644 --- a/plugins/codex-security/mcp-app/tests/test_deep_scan_publication_integration.ts +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_publication_integration.ts @@ -18,7 +18,7 @@ import { Client } from "@modelcontextprotocol/sdk/client/index.js"; import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js"; import { buildServer } from "./build-server.ts"; import { importModule } from "./import-module.ts"; -import type { RunArtifactWorkbench } from "../src/artifact-context.ts"; +import { legacyAcceptedFinding } from "./legacy-accepted-finding.ts"; import type { DeepScanRunState, DeepScanWorkerMutation, @@ -43,11 +43,24 @@ type HostModule = Pick< Pick< typeof import("../src/artifact-scan-draft.ts"), "recordCodexSecurityScanDraftViaWorkbench" + > & + Pick< + typeof import("../src/deep-scan/finalization.ts"), + "readSelectedDeepScanDraft" + > & + Pick< + typeof import("../src/deep-scan/artifacts.ts"), + "createDeepScanArtifacts" + > & + Pick< + typeof import("../src/deep-scan/artifact-validation.ts"), + "parseDeepReduction" | "deepReductionToScanDraft" >; type ToolResult = Awaited>; interface FixturePaths { target?: string; scanRoot?: string; + maxDiscoveryRuns?: number; } type Scenario = readonly [ label: string, @@ -92,6 +105,9 @@ const host = (await importModule({ 'export { DeepScanCoordinator } from "./src/deep-scan/coordinator.ts";', 'export { createScanArtifactContext } from "./src/artifact-context.ts";', 'export { recordCodexSecurityScanDraftViaWorkbench } from "./src/artifact-scan-draft.ts";', + 'export { readSelectedDeepScanDraft } from "./src/deep-scan/finalization.ts";', + 'export { createDeepScanArtifacts } from "./src/deep-scan/artifacts.ts";', + 'export { parseDeepReduction, deepReductionToScanDraft } from "./src/deep-scan/artifact-validation.ts";', ].join("\n"), resolveDir: mcpAppRoot, }, @@ -102,8 +118,126 @@ const { DeepScanCoordinator, createScanArtifactContext, recordCodexSecurityScanDraftViaWorkbench, + readSelectedDeepScanDraft, + createDeepScanArtifacts, + parseDeepReduction, + deepReductionToScanDraft, } = host; +for (const version of [ + "deep-security-scan/v2", + "deep-security-scan/v1", + "deep-scan-mcp/v1", +]) { + test(`historical accepted findings survive ${version} publication and public completion`, async (t) => { + const fixture = await createFixture(t, { maxDiscoveryRuns: 4 }, version); + const { run, store, call, runWorkbench } = fixture; + const legacy = legacyAcceptedFinding(); + const threatModel = { + summary: "Previously recorded synthetic threat model", + }; + assertSuccess( + await call("record_codex_security_scan_draft", { + ...partial(run, "previous-review"), + threatModel, + }), + ); + const claimed = await store.claimCoordinator({ + scanId: run.scanId, + threadId: owner, + }); + // Seed the accepted worker records and bytes produced by the earlier contract. + const results = await commitReducers( + fixture, + [1, 2], + [highId, lowId], + undefined, + { + findings: [legacy], + omitThreatModel: true, + }, + ); + const manifestPath = path.join(run.scanDir, "scan-manifest.json"); + let publicationRun = await store.get(run.scanId, owner); + const acceptedReducer = publicationRun.persistedWorkers?.find( + (worker) => worker.id === lowId, + ); + let resultPath = + acceptedReducer?.acceptedResultPath ?? + acceptedReducer?.resultManifestPath; + assert.ok(resultPath); + let draft: ScanDraftInput; + if (version === "deep-security-scan/v2") { + publicationRun = await store.selectFinalization({ + scanId: run.scanId, + coordinatorGeneration: claimed.run.coordinatorGeneration, + reason: "capped", + manifestPath, + resultPath: results[1], + omittedWorkerIds: [], + }); + assert.ok(publicationRun.finalizationInput?.resultPath); + resultPath = path.resolve( + run.scanDir, + publicationRun.finalizationInput.resultPath, + ); + draft = await readSelectedDeepScanDraft( + createDeepScanArtifacts(run.scanDir), + run.scanId, + publicationRun.finalizationInput, + ); + } else { + draft = deepReductionToScanDraft( + parseDeepReduction( + JSON.parse(await readFile(resultPath, "utf8")), + true, + ), + ); + } + const original = await readFile(resultPath, "utf8"); + assert.deepEqual(draft.findings, [legacy]); + assertToolError(await call("record_codex_security_scan_draft", draft)); + const context = await createScanArtifactContext(run.scanId, runWorkbench, { + requireRunning: true, + }); + await recordCodexSecurityScanDraftViaWorkbench( + context, + draft, + runWorkbench, + undefined, + { + coordinatorGeneration: publicationRun.coordinatorGeneration, + resultPath, + }, + ); + await store.finish({ + scanId: run.scanId, + reason: "capped", + manifestPath, + omittedWorkerIds: [], + }); + assertSuccess( + await call("complete_codex_security_scan", { scanId: run.scanId }), + ); + const saved = JSON.parse( + await readFile(path.join(run.scanDir, "findings.json"), "utf8"), + ); + for (const field of [ + "rootCause", + "root_cause", + "codeEvidence", + "code_evidence", + "validation", + "attackPath", + ] as const) { + assert.deepEqual(saved.findings[0][field], legacy[field]); + } + assert.equal(await readFile(resultPath, "utf8"), original); + const manifest = JSON.parse(await readFile(manifestPath, "utf8")); + assert.deepEqual(manifest.scan.threatModel, threatModel); + }); +} + test("public partial drafts retain checkpoints after coordinator adoption", async (t) => { const fixture = await createFixture(t); const { run, store, call, runWorkbench } = fixture; @@ -259,6 +393,13 @@ for (const [ const context = await createScanArtifactContext(run.scanId, runWorkbench, { requireRunning: true, }); + const acceptedResults = ids.map((id) => { + const result = persisted.persistedWorkers!.find( + (worker) => worker.id === id, + )?.acceptedResultPath; + assert.ok(result); + return result; + }); const publish = async (index: number, coordinatorGeneration = 2) => recordCodexSecurityScanDraftViaWorkbench( context, @@ -268,7 +409,7 @@ for (const [ }, runWorkbench, undefined, - { coordinatorGeneration, resultPath: results[index] }, + { coordinatorGeneration, resultPath: acceptedResults[index] }, ); if (!recoveryOnly) { @@ -309,8 +450,18 @@ for (const [ "succeeded", terminal?.error ?? "Coordinator did not succeed", ); + assert.ok(terminal.finalizationInput?.resultPath); + const acceptedResult = path.resolve( + run.scanDir, + terminal.finalizationInput.resultPath, + ); + assert.deepEqual( + JSON.parse(await readFile(acceptedResult, "utf8")), + JSON.parse(await readFile(results[1], "utf8")), + "selection retains the latest reducer payload as immutable evidence", + ); assert.deepEqual(publications, [ - { coordinatorGeneration: 2, resultPath: results[1] }, + { coordinatorGeneration: 2, resultPath: acceptedResult }, ]); const beforeCompletion = await snapshot(run); @@ -519,7 +670,11 @@ for (const complete of [false, true]) { for (const claimed of [false, true]) { test(`partial drafts retain a capped publication with claimed=${claimed}`, async (t) => { - const fixture = await createFixture(t); + const fixture = await createFixture( + t, + {}, + claimed ? "deep-security-scan/v2" : "deep-security-scan/v1", + ); const { run, store, call, runWorkbench, instant } = fixture; const selectedRun = claimed ? (await store.claimCoordinator({ scanId: run.scanId, threadId: owner })) @@ -678,7 +833,11 @@ async function assertInterleavedProgressRetained(fixture: PublicationFixture) { assert.deepEqual(JSON.parse(after.checkpoints[added[0]]), progress); } -async function createFixture(t: TestContext, paths: FixturePaths = {}) { +async function createFixture( + t: TestContext, + paths: FixturePaths = {}, + workflowVersion = "deep-security-scan/v2", +) { const root = await mkdtemp(path.join(tmpdir(), "deep-publication-")); let client: Client | undefined; t.after(async () => { @@ -700,19 +859,43 @@ async function createFixture(t: TestContext, paths: FixturePaths = {}) { }).filter((entry): entry is [string, string] => entry[1] !== undefined), ); delete environment.CODEX_SECURITY_DEEP_SCAN_CONFIG_PATH; + if (paths.maxDiscoveryRuns !== undefined) { + const directory = path.join(environment.CODEX_HOME, "codex-security"); + await mkdir(directory, { recursive: true }); + await writeFile( + path.join(directory, "config.toml"), + `[deep_scan]\nmax_discovery_runs = ${paths.maxDiscoveryRuns}\n`, + ); + } const instant = new Date().toISOString(); let now = instant; - const runWorkbench: RunArtifactWorkbench = async (args, input) => { - const child = execFileAsync( - python, - [path.join(testsRoot, "clock_workbench.py"), workbenchPath, ...args], - { - cwd: pluginRoot, - env: { ...environment, TEST_WORKBENCH_NOW: now }, - timeout: 30_000, - maxBuffer: 4 * 1024 * 1024, - }, - ); + const runWorkbench = async ( + args: string[], + input?: string | Buffer, + selectFinalization = false, + ) => { + if (args[0] === "begin-deep-scan") { + args = [...args]; + const version = args.indexOf("--workflow-version"); + assert.notEqual(version, -1); + args[version + 1] = workflowVersion; + } + const clock = path.join(testsRoot, "clock_workbench.py"); + const pythonArgs = selectFinalization + ? [ + "-c", + "import runpy, sys; script = sys.argv.pop(1); runpy.run_path(script)['main'](select_finalization=True)", + clock, + workbenchPath, + ...args, + ] + : [clock, workbenchPath, ...args]; + const child = execFileAsync(python, pythonArgs, { + cwd: pluginRoot, + env: { ...environment, TEST_WORKBENCH_NOW: now }, + timeout: 30_000, + maxBuffer: 4 * 1024 * 1024, + }); assert.ok(child.child.stdin); child.child.stdin.end(input); return JSON.parse((await child).stdout) as Record; @@ -758,8 +941,13 @@ async function commitReducers( offsets: readonly [number, number], ids: readonly [string, string], reducerLabels?: readonly [string, string], + historical?: { + findings: Record[]; + omitThreatModel: boolean; + }, ) { const { run, store } = fixture; + const findings = historical?.findings ?? []; const results = []; for (let batch = 0; batch < 2; batch++) { const workerIds = []; @@ -779,7 +967,7 @@ async function commitReducers( const resultManifestPath = path.join(artifact.artifactDir, "result.json"); await writeFile( resultManifestPath, - JSON.stringify({ scanId: run.scanId, findings: [], coverage }), + JSON.stringify({ scanId: run.scanId, findings, coverage }), ); await store.updateWorker({ ...update, @@ -806,8 +994,10 @@ async function commitReducers( resultManifestPath, JSON.stringify({ scanId: run.scanId, - findings: [], - threatModel: { summary: label }, + findings, + ...(historical?.omitThreatModel + ? {} + : { threatModel: { summary: label } }), }), ); fixture.setTime(offsets[batch]); @@ -815,7 +1005,7 @@ async function commitReducers( id, scanId: run.scanId, resultManifestPath, - newFindings: 0, + newFindings: batch === 0 ? findings.length : 0, }); fixture.setTime(0); results.push(resultManifestPath); diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_recorded_recovery.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_recorded_recovery.mjs new file mode 100644 index 0000000000..da8f484e3a --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_recorded_recovery.mjs @@ -0,0 +1,64 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { + materialRemediations, + materialRemediationTests, + publishCoverageFixture, +} from "./deep_scan_coverage_fixture.mjs"; + +for (const [continueAfterResume, legacyAttempts, splitSeededReducers] of [ + [false, false, false], + [true, false, false], + [true, true, false], + [false, false, true], + [false, true, true], +]) { + test(`recorded inputs preserve fixes and coverage after mutable outputs disappear (continued: ${continueAfterResume}, legacy attempts: ${legacyAttempts}, reducer chain: ${splitSeededReducers})`, async () => { + const root = await mkdtemp( + path.join(tmpdir(), "recorded-coverage-recovery-"), + ); + try { + const fixture = path.join(root, "fixture"); + await mkdir(fixture, { mode: 0o700 }); + const { scanDir } = await publishCoverageFixture(fixture, "partial", { + resume: true, + continueAfterResume, + legacyAttempts, + splitSeededReducers, + immutableInputs: true, + materialFindings: true, + discardMutableResults: true, + }); + const report = await readFile(path.join(scanDir, "report.md"), "utf8"); + for (const fix of [ + ...materialRemediations, + ...materialRemediationTests, + ]) { + assert.equal(report.split(fix).length - 1, 1); + } + const coverage = JSON.parse( + await readFile(path.join(scanDir, "coverage.json"), "utf8"), + ); + assert.equal(coverage.completeness, undefined); + assert.deepEqual( + coverage.reviews.map((review) => review.completeness), + [undefined, undefined, undefined], + ); + assert.equal( + new Set(coverage.deferred.map((item) => item.candidateId)).size, + 2, + ); + for (const surface of coverage.surfaces) { + assert.equal( + await readFile(path.join(scanDir, surface.receiptRefs[0]), "utf8"), + "Synthetic review evidence.\n", + ); + } + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_recovery_settings.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_recovery_settings.mjs new file mode 100644 index 0000000000..526a417193 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_recovery_settings.mjs @@ -0,0 +1,1060 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { syncBuiltinESMExports } from "node:module"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { PassThrough } from "node:stream"; +import { fileURLToPath } from "node:url"; +import { build } from "esbuild"; + +const bundle = await build({ + bundle: true, + nodePaths: [fileURLToPath(new URL("../node_modules", import.meta.url))], + define: { + "import.meta.url": JSON.stringify( + new URL("../src/deep-scan/recovery-settings.ts", import.meta.url).href, + ), + }, + entryPoints: [ + new URL("../src/deep-scan/recovery-settings.ts", import.meta.url).pathname, + ], + format: "esm", + platform: "node", + write: false, +}); +const { + beginDeepScanWithCapturedSettings: beginWithCapturedSettings, + captureDeepScanExecutionSettings: captureSettings, + restoredDeepScanWorkerSettings: restoreSettings, + loadDeepScanExecutionSettings: loadSettings, +} = await import( + `data:text/javascript;base64,${Buffer.from(bundle.outputFiles[0].contents).toString("base64")}` +); +const root = await mkdtemp(join(tmpdir(), "deep-settings-")); +try { + const settings = { + codexPath: "/fixture/runtime/codex", + codexHome: "/fixture/account", + model: "fixture-model", + modelProvider: "fixture-provider", + reasoningEffort: "high", + reasoningSummary: "detailed", + serviceTier: "fast", + }; + for (const source of ["missing", "invalid"]) { + const config = join(root, `${source}.toml`); + if (source === "invalid") await writeFile(config, "model = ["); + let captureFailure; + const capture = async () => { + try { + return await captureSettings( + {}, + { filesystemDenies: [] }, + { CODEX_HOME: root, CODEX_SECURITY_CONFIG_PATH: config }, + ); + } catch (error) { + captureFailure = error; + throw error; + } + }; + const startFailure = new Error( + "New runs require original execution settings.", + ); + await assert.rejects( + beginWithCapturedSettings(capture, async (captured) => { + assert.equal(captured, null); + throw startFailure; + }), + (error) => { + assert.ok(error instanceof AggregateError); + assert.deepEqual(error.errors, [startFailure, captureFailure]); + assert.ok(error.message.includes(startFailure.message)); + assert.ok(error.message.includes(captureFailure.message)); + return true; + }, + ); + const existing = { scanId: "existing-scan", status: "running" }; + assert.equal( + await beginWithCapturedSettings(capture, async (captured) => { + assert.equal(captured, null); + return existing; + }), + existing, + ); + } + const unrelatedFailure = new Error("Synthetic workbench failure."); + await assert.rejects( + beginWithCapturedSettings( + async () => settings, + async (captured) => { + assert.equal(captured, settings); + throw unrelatedFailure; + }, + ), + (error) => error === unrelatedFailure, + ); + const globSandbox = (depth) => ({ + filesystemDenies: ["/fixture/**/*.secret"], + ...(depth === undefined ? {} : { globScanMaxDepth: depth }), + }); + for (const [originalDepth, currentDepth, expectedDepth] of [ + [2, 5, 5], + [5, 2, 5], + [undefined, 2, undefined], + [2, undefined, undefined], + ]) { + const restored = restoreSettings( + { ...settings, parentSandbox: globSandbox(originalDepth) }, + globSandbox(currentDepth), + ); + assert.equal( + restored.parentSandbox.globScanMaxDepth, + expectedDepth, + `deny expansion must preserve both policies: ${originalDepth}, ${currentDepth}`, + ); + } + assert.equal( + restoreSettings(settings, globSandbox(2)).parentSandbox.globScanMaxDepth, + 2, + "unavailable historical policy does not establish uncapped glob expansion", + ); + assert.equal( + restoreSettings( + { ...settings, parentSandbox: globSandbox(2) }, + { + filesystemDenies: ["/fixture/exact-denial"], + }, + ).parentSandbox.globScanMaxDepth, + 2, + "exact denials do not change glob expansion", + ); + const writeSnapshot = async (directory, value) => { + const path = join( + directory, + "artifacts", + "deep_discovery", + "execution-settings.json", + ); + await mkdir(join(directory, "artifacts", "deep_discovery"), { + recursive: true, + }); + await writeFile( + path, + JSON.stringify({ version: 1, settings: value }, null, 2) + "\n", + ); + }; + const boundRun = (value) => ({ + workflowVersion: "deep-security-scan/v2", + executionSettings: { version: 1, settings: value }, + }); + await assert.rejects( + loadSettings(join(root, "missing")), + /no recorded original execution settings/, + ); + await writeSnapshot(join(root, "one"), settings); + await writeSnapshot(join(root, "two"), { ...settings, model: "other-model" }); + const savedPath = join( + root, + "one", + "artifacts", + "deep_discovery", + "execution-settings.json", + ); + const saved = await readFile(savedPath, "utf8"); + const [recovered, concurrent] = await Promise.all([ + loadSettings(join(root, "one"), boundRun(settings)), + loadSettings( + join(root, "two"), + boundRun({ ...settings, model: "other-model" }), + ), + ]); + assert.deepEqual(recovered, settings); + assert.equal(concurrent.model, "other-model"); + assert.equal(await readFile(savedPath, "utf8"), saved); + recovered.model = "caller-mutation"; + assert.deepEqual( + await loadSettings(join(root, "one"), boundRun(settings)), + settings, + ); + await writeSnapshot(join(root, "one"), { + ...settings, + codexPath: "/fixture/substituted/codex", + codexHome: "/fixture/substituted/home", + }); + assert.deepEqual( + await loadSettings(join(root, "one"), boundRun(settings)), + settings, + ); + await assert.rejects( + loadSettings(join(root, "one")), + /no recorded original execution settings/, + ); + + const configPath = join(root, "runtime.toml"); + await writeFile( + configPath, + `model = "inherited-model" +model_provider = "custom" +profile = "scan" +[profiles.scan] +model_reasoning_summary = "concise" +service_tier = "flex" +[model_providers.custom] +name = "Fixture" +http_headers = { Authorization = "synthetic-secret" } +`, + ); + const captured = await captureSettings( + { model: "original-model", reasoningEffort: "ultra" }, + { + filesystemDenies: ["/fixture/original-deny"], + literalFilesystemDenies: ["/fixture/account [original]"], + globScanMaxDepth: 3, + }, + { + CODEX_CLI_PATH: process.execPath, + CODEX_HOME: root, + CODEX_SECURITY_CONFIG_PATH: configPath, + }, + ); + assert.deepEqual(captured.parentSandbox.literalFilesystemDenies, [ + "/fixture/account [original]", + ]); + const resumedSandbox = restoreSettings(captured, { + filesystemDenies: ["/fixture/current-deny"], + literalFilesystemDenies: [ + "/fixture/account [original]", + "/fixture/account [current]", + ], + }).parentSandbox; + assert.deepEqual(resumedSandbox.literalFilesystemDenies, [ + "/fixture/account [original]", + "/fixture/account [current]", + ]); + assert.deepEqual(resumedSandbox.filesystemDenies, [ + "/fixture/original-deny", + "/fixture/current-deny", + ]); + assert.equal( + restoreSettings(settings, { filesystemDenies: [] }).parentSandbox + .literalFilesystemDenies, + undefined, + ); + assert.equal(captured.model, "original-model"); + assert.equal(captured.modelProvider, "custom"); + assert.equal(captured.reasoningSummary, "concise"); + assert.equal(captured.serviceTier, "flex"); + assert.equal(captured.providerConfig, undefined); + assert.equal(JSON.stringify(captured).includes("synthetic-secret"), false); + for (const modelProvider of ["openrouter", "fireworks", "amazon-bedrock"]) { + await writeFile( + configPath, + `model_provider = ${JSON.stringify(modelProvider)} +[model_providers.${modelProvider}.aws] +region = "us-west-2" +profile = "fixture-profile" +access_key_id = "synthetic-secret" +`, + ); + const selected = await captureSettings( + {}, + { filesystemDenies: [] }, + { + CODEX_CLI_PATH: process.execPath, + CODEX_HOME: root, + CODEX_SECURITY_CONFIG_PATH: configPath, + }, + ); + assert.equal(selected.modelProvider, modelProvider); + const expectedProvider = + modelProvider === "amazon-bedrock" + ? { + "amazon-bedrock": { + aws: { region: "us-west-2", profile: "fixture-profile" }, + }, + } + : undefined; + assert.deepEqual( + selected.providerConfig, + expectedProvider, + "saved selections exclude catalog definitions and retain Bedrock selectors", + ); + const providerDir = join(root, modelProvider); + await writeSnapshot(providerDir, selected); + const path = join( + providerDir, + "artifacts", + "deep_discovery", + "execution-settings.json", + ); + const bytes = await readFile(path, "utf8"); + assert.equal(bytes.includes("synthetic-secret"), false); + const restoredProvider = restoreSettings( + await loadSettings(providerDir, boundRun(selected)), + { filesystemDenies: [] }, + ).codexOptions.config.model_providers; + assert.deepEqual(restoredProvider, expectedProvider); + assert.equal(await readFile(path, "utf8"), bytes); + await writeFile( + join(root, "config.toml"), + `model_provider = ${JSON.stringify(modelProvider)} +[model_providers.${modelProvider}] +base_url = "https://native.example.invalid/v1" +env_key = "SYNTHETIC_NATIVE_KEY" +[model_providers.${modelProvider}.aws] +region = "us-west-2" +profile = "fixture-profile" +`, + ); + const native = await captureSettings( + {}, + { filesystemDenies: [] }, + { CODEX_CLI_PATH: process.execPath, CODEX_HOME: root }, + ); + const nativeConfig = restoreSettings(native, { filesystemDenies: [] }) + .codexOptions.config; + assert.equal(nativeConfig.model_provider, modelProvider); + assert.deepEqual(nativeConfig.model_providers, expectedProvider); + // Older snapshots can contain catalog definitions. Reading them keeps + // their bytes intact and still inherits the original native definition. + if (!expectedProvider) { + const legacyProvider = { + [modelProvider]: { + name: "Fixture", + base_url: "https://catalog.example.invalid/v1", + env_key: "SYNTHETIC_CATALOG_KEY", + wire_api: "responses", + }, + }; + await writeSnapshot(providerDir, { + ...selected, + providerConfig: legacyProvider, + }); + const legacyBytes = await readFile(path, "utf8"); + const legacy = await loadSettings( + providerDir, + boundRun({ ...selected, providerConfig: legacyProvider }), + ); + assert.equal(legacy.providerConfig, undefined); + assert.deepEqual( + restoreSettings(legacy, { filesystemDenies: [] }).codexOptions.config + .model_providers, + restoredProvider, + ); + assert.equal(await readFile(path, "utf8"), legacyBytes); + } + } + let credential = "synthetic-first"; + const restored = restoreSettings( + captured, + { filesystemDenies: ["/fixture/current-deny"] }, + () => ({ + CODEX_API_KEY: credential, + CODEX_HOME: "/fixture/observer-home", + CODEX_CLI_PATH: "/fixture/observer-codex", + }), + ); + assert.equal(restored.codexOptions.env.CODEX_API_KEY, "synthetic-first"); + credential = "synthetic-refreshed"; + assert.equal(restored.codexOptions.env.CODEX_API_KEY, "synthetic-refreshed"); + assert.equal(restored.codexOptions.env.CODEX_HOME, root); + assert.equal(restored.codexOptions.env.CODEX_CLI_PATH, captured.codexPath); + assert.deepEqual(restored.parentSandbox.filesystemDenies, [ + "/fixture/original-deny", + "/fixture/current-deny", + ]); + assert.equal(restored.codexOptions.config.model_reasoning_effort, "ultra"); + await writeFile(join(root, "config.toml"), 'model = "native-home-model"\n'); + const native = await captureSettings( + {}, + { filesystemDenies: [] }, + { CODEX_CLI_PATH: process.execPath, CODEX_HOME: root }, + ); + assert.equal(native.model, "native-home-model"); + const sessionDirectory = join(root, "sessions"); + await mkdir(sessionDirectory); + await writeFile( + join(sessionDirectory, "parent.jsonl"), + [ + { + type: "session_meta", + timestamp: "2026-01-01T00:00:00Z", + payload: { id: "fixture-parent", model_provider: "openai" }, + }, + { + type: "turn_context", + timestamp: "2026-01-01T00:00:01Z", + payload: { + turn_id: "original-turn", + model: "parent-model", + effort: "high", + summary: "none", + }, + }, + { + type: "turn_context", + timestamp: "2026-01-01T00:02:00Z", + payload: { + turn_id: "later-turn", + model: "later-model", + effort: "low", + summary: "detailed", + }, + }, + ] + .map(JSON.stringify) + .join("\n") + "\n", + ); + await writeFile( + join(sessionDirectory, "other.jsonl"), + JSON.stringify({ + type: "session_meta", + payload: { id: "fixture-other", model_provider: "other-provider" }, + }) + "\n", + ); + const parentSettings = await captureSettings( + {}, + { filesystemDenies: [] }, + { + CODEX_CLI_PATH: process.execPath, + CODEX_HOME: root, + }, + { threadId: "fixture-parent", startedAt: "2026-01-01T00:01:00Z" }, + ); + assert.equal( + parentSettings.model, + "native-home-model", + "explicit config retains precedence", + ); + assert.equal(parentSettings.modelProvider, "openai"); + assert.equal( + parentSettings.reasoningSummary, + "none", + "later owner turns are not original discovery settings", + ); + assert.equal(parentSettings.reasoningEffort, "high"); + await writeFile(join(root, "config.toml"), ""); + const parentEnvironment = { + CODEX_CLI_PATH: process.execPath, + CODEX_HOME: root, + }; + const [originalParent, otherParent, unavailableParent] = await Promise.all([ + captureSettings({}, { filesystemDenies: [] }, parentEnvironment, { + threadId: "fixture-parent", + startedAt: "2026-01-01T00:00:01Z", + }), + captureSettings({}, { filesystemDenies: [] }, parentEnvironment, { + threadId: "fixture-other", + startedAt: "2026-01-01T00:00:01Z", + }), + captureSettings( + { model: "stored-model", reasoningEffort: "ultra" }, + { filesystemDenies: [] }, + parentEnvironment, + { threadId: "fixture-unavailable", startedAt: "2026-01-01T00:00:01Z" }, + ), + ]); + assert.equal(originalParent.model, "parent-model"); + assert.equal( + originalParent.reasoningSummary, + "none", + "the original turn is included at its timestamp", + ); + assert.equal(otherParent.modelProvider, "other-provider"); + assert.equal( + otherParent.model, + undefined, + "concurrent scans do not borrow another parent's model", + ); + assert.equal(otherParent.reasoningSummary, undefined); + assert.equal(unavailableParent.model, "stored-model"); + assert.equal(unavailableParent.reasoningEffort, "ultra"); + assert.equal( + unavailableParent.modelProvider, + undefined, + "missing history does not establish a provider", + ); + assert.equal(unavailableParent.reasoningSummary, undefined); + assert.equal(unavailableParent.serviceTier, undefined); + assert.equal( + unavailableParent.nativeServiceTierAbsent, + undefined, + "missing history does not prove native absence", + ); + const originalOwner = { + threadId: "fixture-parent", + turnId: "original-turn", + startedAt: "2026-01-01T00:00:00Z", + }; + for (const workflowVersion of ["deep-security-scan/v1", "deep-scan-mcp/v1"]) { + const legacyDir = join(root, workflowVersion.replaceAll("/", "-")); + const legacy = await loadSettings( + legacyDir, + { + workflowVersion, + model: "recorded-model", + reasoningEffort: "ultra", + createdAt: "2026-01-01T00:01:00Z", + usageOwner: null, + }, + async () => ({ + config: { model_reasoning_summary: "concise", service_tier: "flex" }, + usageOwner: originalOwner, + }), + parentEnvironment, + ); + assert.equal(legacy.model, "recorded-model"); + assert.equal( + legacy.reasoningSummary, + "concise", + "recorded recipe retains precedence", + ); + assert.equal( + legacy.modelProvider, + "openai", + "recorded original owner supplies native selections", + ); + assert.equal(legacy.serviceTier, "flex"); + assert.equal( + legacy.codexPath, + undefined, + "legacy metadata did not record an executable", + ); + assert.equal( + legacy.codexHome, + undefined, + "a history lookup home is not recorded execution provenance", + ); + const restoredLegacy = restoreSettings( + legacy, + { filesystemDenies: ["/fixture/current-deny"] }, + () => ({ + CODEX_HOME: "/fixture/runtime-home", + CODEX_CLI_PATH: "/fixture/runtime-codex", + CODEX_API_KEY: "synthetic-live-key", + }), + ); + assert.equal( + restoredLegacy.codexOptions.env.CODEX_HOME, + "/fixture/runtime-home", + ); + assert.equal( + restoredLegacy.codexOptions.env.CODEX_CLI_PATH, + "/fixture/runtime-codex", + ); + assert.equal( + restoredLegacy.codexOptions.config.model_reasoning_summary, + "concise", + ); + await assert.rejects( + readFile( + join(legacyDir, "artifacts/deep_discovery/execution-settings.json"), + ), + { code: "ENOENT" }, + ); + } + await assert.rejects( + loadSettings( + join(root, "missing-v2"), + { workflowVersion: "deep-security-scan/v2" }, + async () => + assert.fail( + "missing promised v2 settings must not become legacy recovery", + ), + ), + /no recorded original/, + ); + const [rebound, unboundLegacy] = await Promise.all([ + captureSettings( + { usageOwner: originalOwner }, + { filesystemDenies: [] }, + parentEnvironment, + { threadId: "fixture-other", startedAt: "2026-01-01T00:03:00Z" }, + ), + captureSettings( + { model: "stored-model", usageOwner: null }, + { filesystemDenies: [] }, + parentEnvironment, + { threadId: "fixture-other", startedAt: "2026-01-01T00:03:00Z" }, + ), + ]); + assert.equal( + rebound.modelProvider, + "openai", + "takeover uses the recorded owner, not the invoking conversation", + ); + assert.equal( + rebound.model, + "parent-model", + "the bound turn takes precedence over later turns", + ); + assert.equal(rebound.reasoningSummary, "none"); + assert.equal(unboundLegacy.model, "stored-model"); + assert.equal( + unboundLegacy.modelProvider, + undefined, + "unrecorded legacy ownership cannot recover caller selections", + ); + assert.equal(unboundLegacy.reasoningSummary, undefined); + await writeFile( + join(sessionDirectory, "legacy-auto.jsonl"), + [ + { + type: "session_meta", + payload: { + id: "fixture-legacy-auto", + cli_version: "0.132.0", + model_provider: "openai", + }, + }, + { + type: "turn_context", + payload: { + turn_id: "legacy-turn", + model: "legacy-model", + effort: "high", + summary: "auto", + }, + }, + ] + .map(JSON.stringify) + .join("\n") + "\n", + ); + const legacyAuto = await captureSettings( + { usageOwner: { threadId: "fixture-legacy-auto", turnId: "legacy-turn" } }, + { filesystemDenies: [] }, + parentEnvironment, + ); + assert.equal( + legacyAuto.reasoningSummary, + "auto", + "older native turn-context selections remain readable", + ); + for (const version of ["0.133.0", "0.154.0"]) { + const threadId = `fixture-fresh-${version}`; + await writeFile( + join(sessionDirectory, `${threadId}.jsonl`), + [ + { + type: "session_meta", + timestamp: "2026-01-01T00:00:00Z", + payload: { + id: threadId, + cli_version: version, + model_provider: "openai", + }, + }, + { + type: "turn_context", + timestamp: "2026-01-01T00:00:01Z", + payload: { + turn_id: "fresh-turn", + model: "fresh-model", + effort: "high", + summary: "auto", + }, + }, + { + type: "event_msg", + timestamp: "2026-01-01T00:02:00Z", + payload: { + type: "thread_settings_applied", + thread_id: threadId, + thread_settings: { + model: "fresh-model", + model_provider_id: "openai", + reasoning_summary: "detailed", + }, + }, + }, + ] + .map(JSON.stringify) + .join("\n") + "\n", + ); + const owner = { + threadId, + turnId: "fresh-turn", + startedAt: "2026-01-01T00:01:00Z", + }; + const fresh = await captureSettings( + { usageOwner: owner }, + { filesystemDenies: [] }, + parentEnvironment, + ); + assert.equal(fresh.model, "fresh-model"); + assert.equal( + fresh.reasoningSummary, + undefined, + "fresh native compatibility auto is not an original selection", + ); + assert.equal( + restoreSettings(fresh, { filesystemDenies: [] }).codexOptions.config + .model_reasoning_summary, + undefined, + ); + const freshDir = join(root, threadId); + await writeSnapshot(freshDir, fresh); + const freshPath = join( + freshDir, + "artifacts", + "deep_discovery", + "execution-settings.json", + ); + const freshBytes = await readFile(freshPath, "utf8"); + assert.deepEqual( + await loadSettings(freshDir, { + ...boundRun(fresh), + usageOwner: owner, + createdAt: owner.startedAt, + }), + fresh, + ); + assert.equal( + await readFile(freshPath, "utf8"), + freshBytes, + "unknown summary is not replaced by a compatibility field or a later selection", + ); + await writeFile( + join(root, "config.toml"), + 'model_reasoning_summary = "auto"\n', + ); + const explicit = await captureSettings( + { usageOwner: owner }, + { filesystemDenies: [] }, + parentEnvironment, + ); + assert.equal( + explicit.reasoningSummary, + "auto", + "an explicit original config selection still takes precedence", + ); + await writeFile(join(root, "config.toml"), ""); + } + await writeFile( + join(sessionDirectory, "applied.jsonl"), + [ + { + type: "session_meta", + timestamp: "2026-01-01T00:00:00Z", + payload: { id: "fixture-applied", model_provider: "previous-provider" }, + }, + { + type: "event_msg", + timestamp: "2026-01-01T00:00:01Z", + payload: { + type: "thread_settings_applied", + thread_id: "fixture-applied", + thread_settings: { + model: "applied-model", + model_provider_id: "openai", + service_tier: "default", + reasoning_effort: "high", + reasoning_summary: "concise", + }, + }, + }, + { + type: "turn_context", + timestamp: "2026-01-01T00:00:02Z", + payload: { + turn_id: "applied-turn", + model: "previous-model", + effort: "low", + summary: "none", + }, + }, + { + type: "event_msg", + timestamp: "2026-01-01T00:00:03Z", + payload: { + type: "thread_settings_applied", + thread_id: "fixture-copied-owner", + thread_settings: { + model: "copied-model", + model_provider_id: "copied-provider", + service_tier: "flex", + reasoning_summary: "detailed", + }, + }, + }, + { + type: "event_msg", + timestamp: "2026-01-01T00:02:00Z", + payload: { + type: "thread_settings_applied", + thread_id: "fixture-applied", + thread_settings: { + model: "later-model", + model_provider_id: "later-provider", + service_tier: "fast", + reasoning_summary: "detailed", + }, + }, + }, + ] + .map(JSON.stringify) + .join("\n") + "\n", + ); + const appliedOwner = { + threadId: "fixture-applied", + turnId: "applied-turn", + startedAt: "2026-01-01T00:00:00Z", + }; + const applied = await captureSettings( + { usageOwner: appliedOwner }, + { filesystemDenies: [] }, + parentEnvironment, + { threadId: "fixture-other", startedAt: "2026-01-01T00:01:00Z" }, + ); + assert.equal( + applied.serviceTier, + "default", + "original explicit standard routing survives later and copied snapshots", + ); + assert.equal( + applied.reasoningSummary, + "concise", + "native applied summary overrides the legacy compatibility field", + ); + assert.equal( + applied.modelProvider, + "openai", + "complete native snapshot replaces the session metadata provider", + ); + assert.equal( + applied.model, + "applied-model", + "complete native snapshot replaces compatibility turn settings", + ); + assert.equal(applied.reasoningEffort, "high"); + assert.equal( + applied.nativeServiceTierAbsent, + undefined, + "explicit native standard remains an explicit selection", + ); + const tierDir = join(root, "missing-tier"); + const { serviceTier: omittedTier, ...withoutTier } = applied; + assert.equal(omittedTier, "default"); + await writeSnapshot(tierDir, withoutTier); + const repairedTier = await loadSettings(tierDir, { + ...boundRun(withoutTier), + usageOwner: appliedOwner, + createdAt: "2026-01-01T00:01:00Z", + }); + assert.equal(repairedTier.serviceTier, "default"); + await writeFile( + join(sessionDirectory, "applied.jsonl"), + (await readFile(join(sessionDirectory, "applied.jsonl"), "utf8")) + + JSON.stringify({ + type: "event_msg", + timestamp: "2026-01-01T00:00:04Z", + payload: { + type: "thread_settings_applied", + thread_id: "fixture-applied", + thread_settings: { + model: "applied-model", + model_provider_id: "openai", + reasoning_effort: "high", + service_tier: "priority", + }, + }, + }) + + "\n", + ); + const nativeTier = await captureSettings( + { usageOwner: appliedOwner }, + { filesystemDenies: [] }, + parentEnvironment, + { threadId: "fixture-other", startedAt: "2026-01-01T00:01:00Z" }, + ); + assert.equal( + nativeTier.serviceTier, + "priority", + "an effective tier selected by native remains unchanged", + ); + assert.equal(nativeTier.nativeServiceTierAbsent, undefined); + await writeFile( + join(sessionDirectory, "applied.jsonl"), + (await readFile(join(sessionDirectory, "applied.jsonl"), "utf8")) + + JSON.stringify({ + type: "event_msg", + timestamp: "2026-01-01T00:00:05Z", + payload: { + type: "thread_settings_applied", + thread_id: "fixture-applied", + thread_settings: { + model: "applied-model", + model_provider_id: "openai", + reasoning_effort: "high", + }, + }, + }) + + "\n", + ); + const nativeDefaults = await captureSettings( + { usageOwner: appliedOwner }, + { filesystemDenies: [] }, + parentEnvironment, + { threadId: "fixture-other", startedAt: "2026-01-01T00:01:00Z" }, + ); + assert.equal( + nativeDefaults.model, + "applied-model", + "absent optional selections do not erase the required model", + ); + assert.equal( + nativeDefaults.modelProvider, + "openai", + "absent optional selections do not erase the required provider", + ); + assert.equal(nativeDefaults.reasoningEffort, "high"); + assert.equal( + nativeDefaults.serviceTier, + "default", + "known native absence retains its omitted request tier", + ); + assert.equal( + nativeDefaults.nativeServiceTierAbsent, + true, + "known native absence is recorded separately from explicit standard", + ); + assert.equal( + nativeDefaults.reasoningSummary, + undefined, + "a compatibility summary is not a recorded native default", + ); + const incompleteDir = join(root, "incomplete"); + const incomplete = { + codexPath: process.execPath, + codexHome: root, + serviceTier: "flex", + }; + await writeSnapshot(incompleteDir, incomplete); + await writeFile( + join(root, "config.toml"), + 'model_provider = "observer-provider"\nmodel_reasoning_summary = "detailed"\n', + ); + const originalRun = { + ...boundRun(incomplete), + model: "stored-model", + reasoningEffort: "ultra", + usageOwner: originalOwner, + createdAt: "2026-01-01T00:01:00Z", + }; + const raceDir = join(root, "concurrent-recovery"); + await writeSnapshot(raceDir, incomplete); + const racePath = join( + raceDir, + "artifacts", + "deep_discovery", + "execution-settings.json", + ); + const originalCreateReadStream = fs.createReadStream; + const readingHistory = Promise.withResolvers(); + const releaseHistory = Promise.withResolvers(); + let held = false; + let pendingRead; + fs.createReadStream = (path, options) => { + const source = originalCreateReadStream(path, options); + if (path !== join(sessionDirectory, "parent.jsonl") || held) return source; + held = true; + const delayed = new PassThrough(); + source.once("error", (error) => delayed.destroy(error)); + delayed.once("close", () => source.destroy()); + void releaseHistory.promise.then(() => source.pipe(delayed)); + readingHistory.resolve(); + return delayed; + }; + syncBuiltinESMExports(); + try { + pendingRead = loadSettings(raceDir, originalRun); + await Promise.race([ + readingHistory.promise, + pendingRead.then(() => + assert.fail( + "historical recovery must reach the controlled history read", + ), + ), + ]); + const newer = { + ...settings, + modelProvider: "newer-provider", + reasoningSummary: "concise", + }; + await writeSnapshot(raceDir, newer); + const newerBytes = await readFile(racePath, "utf8"); + releaseHistory.resolve(); + const delayedProjection = await pendingRead; + assert.equal(delayedProjection.modelProvider, "openai"); + assert.equal(delayedProjection.reasoningSummary, "none"); + assert.equal( + await readFile(racePath, "utf8"), + newerBytes, + "a delayed historical projection must not overwrite a newer snapshot", + ); + assert.deepEqual(await loadSettings(raceDir, boundRun(newer)), newer); + } finally { + releaseHistory.resolve(); + await pendingRead?.catch(() => {}); + fs.createReadStream = originalCreateReadStream; + syncBuiltinESMExports(); + } + const incompletePath = join( + incompleteDir, + "artifacts", + "deep_discovery", + "execution-settings.json", + ); + const incompleteBytes = await readFile(incompletePath, "utf8"); + const repaired = await loadSettings(incompleteDir, originalRun); + assert.deepEqual(repaired, { + ...incomplete, + model: "stored-model", + reasoningEffort: "ultra", + modelProvider: "openai", + reasoningSummary: "none", + }); + assert.equal( + await readFile(incompletePath, "utf8"), + incompleteBytes, + "recovering historical fields is read-only", + ); + await rm(sessionDirectory, { recursive: true }); + const unavailable = await loadSettings(incompleteDir, originalRun); + assert.equal(unavailable.model, "stored-model"); + assert.equal(unavailable.reasoningEffort, "ultra"); + assert.equal( + unavailable.modelProvider, + undefined, + "unavailable history remains unknown", + ); + assert.equal(unavailable.reasoningSummary, undefined); + assert.equal(await readFile(incompletePath, "utf8"), incompleteBytes); + const unknownDir = join(root, "unknown"); + await writeSnapshot(unknownDir, incomplete); + const unknown = await loadSettings(unknownDir, { + ...originalRun, + usageOwner: null, + }); + assert.equal(unknown.model, "stored-model"); + assert.equal( + unknown.modelProvider, + undefined, + "missing original ownership is not current config", + ); + assert.equal(unknown.reasoningSummary, undefined); + assert.equal(unknown.nativeServiceTierAbsent, undefined); + const unsupported = JSON.stringify({ version: 99, settings }); + await writeFile(savedPath, unsupported); + await assert.rejects( + loadSettings(join(root, "one"), { + workflowVersion: "deep-security-scan/v2", + executionSettings: { version: 99, settings }, + }), + /unsupported/, + ); + assert.equal(await readFile(savedPath, "utf8"), unsupported); +} finally { + await rm(root, { recursive: true, force: true }); +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_selected_coverage.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_selected_coverage.mjs new file mode 100644 index 0000000000..24b306bd2b --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_selected_coverage.mjs @@ -0,0 +1,65 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { + materialRemediations, + materialRemediationTests, + publishCoverageFixture, +} from "./deep_scan_coverage_fixture.mjs"; + +for (const splitSeededReducers of [false, true]) { + test(`selected publication retains fixes and coverage after failure and a rejected checkpoint (reducer chain: ${splitSeededReducers})`, async () => { + const root = await mkdtemp( + path.join(tmpdir(), "selected-coverage-publication-"), + ); + try { + const fixture = path.join(root, "fixture"); + await mkdir(fixture, { mode: 0o700 }); + const { scanDir, terminal } = await publishCoverageFixture( + fixture, + "partial", + { + resume: true, + immutableInputs: true, + materialFindings: true, + discardMutableResults: true, + splitSeededReducers, + selectedRecovery: true, + }, + ); + assert.equal(terminal.workflowVersion, "deep-security-scan/v2"); + assert.match(terminal.finalizationInput.resultPath, /checkpoints/); + const report = await readFile(path.join(scanDir, "report.md"), "utf8"); + for (const fix of [ + ...materialRemediations, + ...materialRemediationTests, + ]) { + assert.equal(report.split(fix).length - 1, 1); + } + const coverage = JSON.parse( + await readFile(path.join(scanDir, "coverage.json"), "utf8"), + ); + assert.equal(coverage.completeness, undefined); + assert.deepEqual( + coverage.reviews.map((review) => review.completeness), + [undefined, undefined, undefined], + ); + assert.equal( + new Set(coverage.deferred.map((item) => item.candidateId)).size, + 2, + ); + for (const item of coverage.deferred) + assert.ok(report.includes(item.reason)); + for (const surface of coverage.surfaces) { + assert.equal( + await readFile(path.join(scanDir, surface.receiptRefs[0]), "utf8"), + "Synthetic review evidence.\n", + ); + } + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_selected_finalization_cancel.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_selected_finalization_cancel.ts new file mode 100644 index 0000000000..0b9ef732f8 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_selected_finalization_cancel.ts @@ -0,0 +1,307 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { spawn } from "node:child_process"; +import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { temporaryDirectory } from "./support/temporary-directories.ts"; +import { importModule } from "./import-module.ts"; +import type { ScanDraftInput } from "../src/artifact-scan-draft.ts"; +import type { DeepScanRunState } from "../src/deep-scan/types.ts"; +import type { DeepScanPublication } from "../src/artifact-scan-draft.ts"; +const appRoot = path.resolve(import.meta.dirname, ".."); +const pluginRoot = path.resolve(appRoot, ".."); +const workbenchPath = path.join(pluginRoot, "scripts/workbench_db.py"); +const { + DeepScanCoordinatorRegistry, + WorkbenchDeepScanStore, + createScanArtifactContext, + recordCodexSecurityScanDraftViaWorkbench, + readSelectedDeepScanDraft, + createDeepScanArtifacts, + captureDeepScanExecutionSettings, +} = await importModule({ + stdin: { + contents: `export { captureDeepScanExecutionSettings } from "./src/deep-scan/recovery-settings.ts"; + export { readSelectedDeepScanDraft } from "./src/deep-scan/finalization.ts"; + export { createDeepScanArtifacts } from "./src/deep-scan/artifacts.ts"; + export { DeepScanCoordinatorRegistry } from "./src/deep-scan/registry.ts"; + export { WorkbenchDeepScanStore } from "./src/deep-scan/store.ts"; + export { createScanArtifactContext } from "./src/artifact-context.ts"; + export { recordCodexSecurityScanDraftViaWorkbench } from "./src/artifact-scan-draft.ts";`, + resolveDir: appRoot, + }, + loader: { ".md": "text" }, +}); +for (const phase of [ + "draft", + "parent", + "resumed-parent", + "completed", +] as const) { + test(`real saved native scan cancellation during selected ${phase}`, async () => { + const root = await temporaryDirectory("real-detached-cancel-", true); + const home = path.join(root, "codex-home"); + const target = path.join(root, "target"); + const environment = { + ...process.env, + CODEX_HOME: home, + CODEX_SECURITY_STATE_DIR: path.join(root, "state"), + }; + await mkdir(target, { recursive: true }); + await writeFile(path.join(target, "fixture.py"), "print('fixture')\n"); + await mkdir(path.join(home, "codex-security"), { recursive: true }); + await writeFile( + path.join(home, "codex-security/config.toml"), + "[deep_scan]\nworkers = 1\nmax_discovery_runs = 3\nmax_time_hours = 1e-12\n", + ); + const runWorkbench = async ( + args: string[], + input?: string, + selection = false, + withSettings = false, + signal?: AbortSignal, + releaseCoordinator = false, + ) => { + const invocation = releaseCoordinator + ? "release_coordinator=True" + : selection + ? "select_finalization=True" + : withSettings + ? "with_execution_settings=True" + : undefined; + const pythonArgs = invocation + ? [ + "-c", + `import runpy, sys; script = sys.argv.pop(1); runpy.run_path(script)['main'](${invocation})`, + workbenchPath, + ...args, + ] + : [workbenchPath, ...args]; + return await new Promise((resolve, reject) => { + const child = spawn(process.env.PYTHON || "python3", pythonArgs, { + cwd: pluginRoot, + env: environment, + signal, + }); + const stdout: Buffer[] = [], + stderr: Buffer[] = []; + child.stdout.on("data", (data: Buffer) => stdout.push(data)); + child.stderr.on("data", (data: Buffer) => stderr.push(data)); + child.on("error", reject); + child.on("close", (code) => + code === 0 + ? resolve(JSON.parse(Buffer.concat(stdout).toString())) + : reject(new Error(Buffer.concat(stderr).toString())), + ); + child.stdin.end(input); + }); + }; + try { + let store = new WorkbenchDeepScanStore(runWorkbench); + const threadId = "real-selected-owner"; + const executionSettings = await captureDeepScanExecutionSettings( + {}, + { filesystemDenies: [] }, + environment, + ); + const begun = await store.begin({ + targetPath: target, + scope: ".", + threadId, + scanRoot: path.join(root, "scans"), + executionSettings, + }); + const claimed = await store.claimCoordinator({ + scanId: begun.scanId, + threadId, + }); + assert.equal(claimed.acquired, true); + let selected = await store.selectFinalization({ + scanId: begun.scanId, + coordinatorGeneration: claimed.run.coordinatorGeneration, + reason: "capped", + manifestPath: path.join(begun.scanDir, "scan-manifest.json"), + omittedWorkerIds: [], + }); + assert.equal(selected.status, "running"); + assert.equal(selected.finalizationInput.resultPath, null); + if (phase === "resumed-parent") { + const context = await createScanArtifactContext( + begun.scanId, + runWorkbench, + { requireRunning: true, pluginRoot }, + ); + const draft = await readSelectedDeepScanDraft( + createDeepScanArtifacts(begun.scanDir), + begun.scanId, + selected.finalizationInput, + ); + await recordCodexSecurityScanDraftViaWorkbench( + context, + draft, + runWorkbench, + undefined, + { + coordinatorGeneration: selected.coordinatorGeneration, + resultPath: null, + }, + ); + await store.finish({ + scanId: begun.scanId, + reason: "capped", + manifestPath: path.join(begun.scanDir, "scan-manifest.json"), + omittedWorkerIds: [], + }); + await store.releaseCoordinator(begun.scanId); + store = new WorkbenchDeepScanStore(runWorkbench); + const resumed = await store.claimCoordinator({ + scanId: begun.scanId, + threadId, + }); + assert.equal(resumed.acquired, true); + selected = resumed.run; + assert.equal( + selected.status, + "succeeded", + "the resumed Deep result is committed while its parent is still running", + ); + } + const blocked = Promise.withResolvers(), + release = Promise.withResolvers(); + const observer = new AbortController(); + let writes = 0, + seals = 0; + let publicationSignal: AbortSignal | undefined; + const pause = async (signal: AbortSignal) => { + publicationSignal = signal; + blocked.resolve(); + await release.promise; + signal.throwIfAborted(); + }; + const registry = new DeepScanCoordinatorRegistry(); + const coordinator = registry.start({ + run: selected, + store, + pluginRoot, + threadId, + executor: { + run: async () => + assert.fail("Accepted selection cannot launch new model work"), + }, + onComplete: async ( + draft: ScanDraftInput, + signal: AbortSignal, + publication: DeepScanPublication, + ) => { + if (phase === "draft") await pause(signal); + const context = await createScanArtifactContext( + begun.scanId, + runWorkbench, + { requireRunning: true, requireClaim: true, pluginRoot }, + ); + await recordCodexSecurityScanDraftViaWorkbench( + context, + draft, + runWorkbench, + signal, + publication, + ); + }, + onFinalized: async (run: DeepScanRunState, signal: AbortSignal) => { + if (!observer.signal.aborted) return; + if (phase === "parent" || phase === "resumed-parent") + await pause(signal); + await runWorkbench( + ["complete-scan", "--scan-id", run.scanId, "--thread-id", threadId], + undefined, + false, + false, + signal, + ); + seals++; + }, + }); + const settled = coordinator.settled(); + settled.catch(() => {}); + const detached = coordinator.wait(observer.signal); + observer.abort(); + await assert.rejects(detached, { name: "AbortError" }); + const parentRead = Promise.withResolvers(); + const readParentState = async () => { + const current = ( + await runWorkbench(["get-scan", "--scan-id", begun.scanId]) + ).scan; + parentRead.resolve(); + return { + status: current.progress.status, + failureMessage: current.failureMessage, + }; + }; + const persist = async () => { + writes++; + await runWorkbench([ + "cancel-scan", + "--scan-id", + begun.scanId, + "--thread-id", + threadId, + ]); + }; + if (phase === "completed") { + await settled; + const before = await readFile( + path.join(begun.scanDir, "scan-manifest.json"), + ); + assert.equal( + await registry.cancelAndWait( + begun.scanId, + "user_canceled_scan", + persist, + readParentState, + ), + false, + ); + await assert.rejects(persist(), /Only a running scan can be canceled/); + const saved = await runWorkbench([ + "get-scan", + "--scan-id", + begun.scanId, + ]); + assert.equal(saved.scan.progress.status, "complete"); + assert.deepEqual( + await readFile(path.join(begun.scanDir, "scan-manifest.json")), + before, + ); + assert.equal(seals, 1); + return; + } + await blocked.promise; + const cancellation = registry.cancelAndWait( + begun.scanId, + "user_canceled_scan", + persist, + readParentState, + ); + await parentRead.promise; + await new Promise(setImmediate); + const aborted = publicationSignal?.aborted; + release.resolve(); + assert.equal(await cancellation, true); + await settled; + // Match cancelSecurityScan: closed parents are observed; running ones cancel directly. + const saved = await runWorkbench(["get-scan", "--scan-id", begun.scanId]); + if (saved.scan.progress.status === "running") await persist(); + const final = await runWorkbench(["get-scan", "--scan-id", begun.scanId]); + assert.equal( + aborted, + true, + "cancel must reach the active parent publication signal", + ); + assert.equal(final.scan.progress.status, "canceled"); + assert.equal(writes, 1); + assert.equal(seals, 0); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_selected_replay.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_selected_replay.mjs new file mode 100644 index 0000000000..fe0bfacfb0 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_selected_replay.mjs @@ -0,0 +1,169 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, realpath, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { build } from "esbuild"; + +const bundle = await build({ + entryPoints: [ + path.resolve(import.meta.dirname, "../src/deep-scan/coordinator.ts"), + ], + loader: { ".md": "text" }, + bundle: true, + format: "esm", + platform: "node", + write: false, + footer: { js: "//# sourceURL=deep-scan-selected-replay.js" }, +}); +const { DeepScanCoordinator } = await import( + `data:text/javascript;base64,${Buffer.from(bundle.outputFiles[0].contents).toString("base64")}` +); + +for (const terminalReason of ["saturated", "capped"]) { + test(`restart publishes the selected ${terminalReason} result after an output failure`, async () => { + const scanDir = await realpath( + await mkdtemp(path.join(tmpdir(), "selected-replay-")), + ); + try { + const scanId = "4e7b4acb-ac80-4d68-98cd-3d5ac5581cd1"; + const draft = { + scanId, + complete: true, + findings: [], + coverage: { + completeness: "partial", + surfaces: [], + explicitExclusions: [], + deferred: [ + { id: "review", reason: "A dependency remains unreviewed." }, + ], + }, + }; + const { coverage, ...reduction } = draft; + const bytes = JSON.stringify({ ...reduction, sourceCoverage: coverage }); + delete draft.coverage.completeness; + const digest = createHash("sha256").update(bytes).digest("hex"); + const resultPath = `artifacts/deep_discovery/dedup/dedup-0001/output/checkpoints/${digest}.json`; + await mkdir(path.dirname(path.join(scanDir, resultPath)), { + recursive: true, + }); + await writeFile(path.join(scanDir, resultPath), bytes); + const selection = { + version: 1, + resultPath, + resultSha256: digest, + terminalReason, + omittedWorkerIds: [], + selectedAt: "2026-01-01T00:00:00Z", + }; + let run = { + scanId, + scanDir, + targetPath: scanDir, + scope: ".", + workflowVersion: "deep-security-scan/v2", + status: "running", + phase: "terminal", + coordinatorGeneration: 3, + finalizationInput: selection, + terminalReason, + createdAt: "2026-01-01T00:00:00Z", + config: { + workers: 2, + subagents: 0, + stopAfterNoNew: 2, + stopAfterConsecutiveErrors: 2, + maxDiscoveryRuns: 4, + maxTimeHours: 1, + }, + dispatchedCount: 4, + noNewStreak: 2, + consecutiveErrors: 0, + }; + const mutations = []; + const store = new Proxy( + { + get: async () => structuredClone(run), + finish: async (input) => { + mutations.push("finish"); + assert.equal(input.reason, terminalReason); + assert.deepEqual( + input.omittedWorkerIds, + selection.omittedWorkerIds, + ); + run = { + ...run, + status: "succeeded", + manifestPath: input.manifestPath, + }; + return structuredClone(run); + }, + fail: async () => { + mutations.push("fail"); + run = { ...run, status: "failed" }; + return run; + }, + }, + { + get: (target, key) => + key in target + ? target[key] + : async () => { + mutations.push(key); + throw new Error( + `Unexpected scheduler operation: ${String(key)}`, + ); + }, + }, + ); + let executions = 0; + let publications = 0; + const options = { + store, + executor: { + run: async () => { + executions++; + throw new Error("Unexpected model work"); + }, + }, + pluginRoot: scanDir, + threadId: "original-result-conversation", + retryDelaysMs: [], + // Already expired: replay must not start a discovery deadline timer. + discoveryTimeoutMs: 1, + onComplete: async (actual, _signal, publication) => { + publications++; + assert.deepEqual(actual, draft); + assert.equal(publication.coordinatorGeneration, 3); + assert.equal(publication.resultPath, path.join(scanDir, resultPath)); + if (publications === 1) + throw new Error("Synthetic publication write failure"); + }, + }; + const first = new DeepScanCoordinator({ + ...options, + run: structuredClone(run), + }); + first.start(); + await assert.rejects(first.wait(), /Synthetic publication write failure/); + assert.equal(run.status, "running"); + assert.equal(run.terminalReason, terminalReason); + assert.deepEqual(run.finalizationInput, selection); + const restarted = new DeepScanCoordinator({ + ...options, + run: structuredClone(run), + }); + restarted.start(); + const completed = await restarted.wait(); + assert.equal(completed.status, "succeeded"); + assert.equal(completed.terminalReason, terminalReason); + assert.equal(executions, 0); + assert.deepEqual(mutations, ["finish"]); + assert.equal(publications, 2); + } finally { + await rm(scanDir, { recursive: true, force: true }); + } + }); +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_selection_store.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_selection_store.mjs new file mode 100644 index 0000000000..7939a49310 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_selection_store.mjs @@ -0,0 +1,88 @@ +import assert from "node:assert/strict"; +import path from "node:path"; +import { test } from "node:test"; +import { build } from "esbuild"; + +const bundle = await build({ + entryPoints: [path.resolve(import.meta.dirname, "../src/deep-scan/store.ts")], + bundle: true, + format: "esm", + platform: "node", + write: false, + footer: { js: "//# sourceURL=deep-scan-selection-store.js" }, +}); +const { WorkbenchDeepScanStore } = await import( + `data:text/javascript;base64,${Buffer.from(bundle.outputFiles[0].contents).toString("base64")}` +); + +test("selection uses the dedicated function bridge and the store's existing replay policy", async () => { + const scanId = "ed8ff2da-01d9-4338-aeba-8bcfd4b530a9"; + const selection = { + version: 1, + resultPath: "artifacts/merge/checkpoints/aggregate.json", + resultSha256: "a".repeat(64), + terminalReason: "saturated", + omittedWorkerIds: [], + selectedAt: "2026-01-01T00:00:00Z", + }; + const response = { + deepScan: { + scanId, + targetPath: "/target", + scope: ".", + scanDir: "/scan", + status: "running", + schemaVersion: 1, + workflowVersion: "deep-security-scan/v2", + coordinatorGeneration: 2, + config: { + workers: 2, + subagents: 0, + stopAfterNoNew: 2, + stopAfterConsecutiveErrors: 2, + maxDiscoveryRuns: 4, + }, + dispatchedCount: 2, + noNewStreak: 2, + consecutiveErrors: 0, + finalizationInput: selection, + }, + }; + const calls = []; + const store = new WorkbenchDeepScanStore(async (...args) => { + calls.push(structuredClone(args)); + if (calls.length === 1) + throw Object.assign(new Error("Synthetic lost selection response"), { + code: "ETIMEDOUT", + }); + return response; + }); + const result = await store.selectFinalization({ + scanId, + coordinatorGeneration: 2, + reason: "saturated", + manifestPath: "/scan/scan-manifest.json", + resultPath: "/scan/artifacts/merge/checkpoints/aggregate.json", + omittedWorkerIds: [], + }); + assert.deepEqual(result.finalizationInput, selection); + assert.equal(calls.length, 2); + assert.deepEqual(calls[1], calls[0]); + assert.deepEqual(calls[0], [ + [ + "finish-deep-scan", + "--scan-id", + scanId, + "--coordinator-generation", + "2", + "--terminal-reason", + "saturated", + "--manifest-path", + "/scan/scan-manifest.json", + ], + JSON.stringify({ + resultPath: "/scan/artifacts/merge/checkpoints/aggregate.json", + }), + true, + ]); +}); diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_stdio_lifecycle.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_stdio_lifecycle.ts index bb38455594..69532b2a4a 100644 --- a/plugins/codex-security/mcp-app/tests/test_deep_scan_stdio_lifecycle.ts +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_stdio_lifecycle.ts @@ -7,8 +7,9 @@ import { temporaryDirectory } from "./support/temporary-directories.ts"; import assert from "node:assert/strict"; import { parse as parseToml } from "smol-toml"; import { execFile } from "node:child_process"; -import { randomUUID } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import { + appendFile, chmod, cp, mkdir, @@ -28,7 +29,10 @@ import { startRpcServer } from "./support/rpc-server.ts"; const execFileAsync = promisify(execFile); -const pluginRoot = path.resolve(mcpAppRoot, ".."); +const installedPluginRoot = process.env.CODEX_SECURITY_TEST_PLUGIN_ROOT; +const pluginRoot = installedPluginRoot + ? path.resolve(installedPluginRoot) + : path.resolve(mcpAppRoot, ".."); const workbenchPath = path.join(pluginRoot, "scripts", "workbench_db.py"); let parentSandboxState: unknown = readOnlyParentSandboxState(pluginRoot); @@ -37,16 +41,761 @@ if (process.platform === "win32") { "deep scan stdio lifecycle test skipped on Windows (POSIX fake Codex executable)", ); } else { + for (const mode of [ + "cancel-after-seal", + "cancel-finalizer", + "cancel-finalizer-failure", + "late-rejoin", + "late-rejoin-failure", + "detached", + "joined", + "remote", + "failure", + "active-failure", + "remote-replay", + "snapshot-observer", + "native-owner-usage", + "native-detached-usage", + "lost-response", + "lost-response-corrupt", + "lost-response-remove", + ]) { + await testDeepScanDetachedCompletion(mode); + } await testDeepScanStdioLifecycle(); } +async function testDeepScanDetachedCompletion(mode: string) { + const fixtureRoot = await temporaryDirectory("codex-security-deep-detached-"); + const targetPath = path.join(fixtureRoot, "target"); + const stateDir = path.join(fixtureRoot, "state"); + const scanRoot = path.join(fixtureRoot, "scans"); + const codexHome = path.join(fixtureRoot, "codex-home"); + const startLogPath = path.join(fixtureRoot, "started.jsonl"); + const exitLogPath = path.join(fixtureRoot, "exited.jsonl"); + const controlPath = path.join(fixtureRoot, "completion-control"); + const finalizerControlPath = path.join(fixtureRoot, "finalizer-control"); + const finalizerLogPath = path.join(fixtureRoot, "finalizer.jsonl"); + const committedControlPath = path.join(fixtureRoot, "committed-control"); + const committedLogPath = path.join(fixtureRoot, "committed.jsonl"); + const pythonWrapperPath = path.join(fixtureRoot, "python-wrapper.mjs"); + const fakeCodexPath = path.join(fixtureRoot, "fake-codex.mjs"); + const serverBundlePath = path.join( + pluginRoot, + "mcp", + installedPluginRoot + ? "server.mjs" + : `.deep-scan-detached-${randomUUID()}.cjs`, + ); + const threadId = "deep-scan-detached-result-conversation"; + const usageMode = + mode === "native-owner-usage" || mode === "native-detached-usage"; + const explicitCompletion = + mode === "active-failure" || + mode === "cancel-after-seal" || + mode.startsWith("cancel-finalizer") || + mode.startsWith("late-rejoin"); + const ownerRolloutPath = path.join(codexHome, "owner-rollout.jsonl"); + for (const directory of [ + targetPath, + stateDir, + scanRoot, + path.join(codexHome, "codex-security"), + ]) { + await mkdir(directory, { recursive: true }); + } + await writeFile(path.join(targetPath, "fixture.py"), "print('fixture')\n"); + await writeFile( + path.join(codexHome, "codex-security", "config.toml"), + "[deep_scan]\nworkers = 1\nsubagents = 0\nstop_after_no_new = 1\nmax_discovery_runs = 2\n", + ); + await writeFile(controlPath, "wait-for-completion"); + await writePythonWrapper(pythonWrapperPath); + if (mode === "cancel-after-seal") + await writeFile(committedControlPath, "wait"); + if (mode !== "detached" && !usageMode) + await writeFile( + finalizerControlPath, + mode === "joined" || + mode === "remote" || + mode.startsWith("cancel-finalizer") || + mode.startsWith("late-rejoin") + ? "wait" + : mode === "active-failure" || + mode === "remote-replay" || + mode === "snapshot-observer" + ? "failure" + : mode.startsWith("lost-response") + ? "lost-response" + : mode, + ); + await writeFakeCodex(fakeCodexPath); + if (!installedPluginRoot) + await buildServer(serverBundlePath, { target: "node20" }); + const environment: NodeJS.ProcessEnv = { + ...process.env, + OPENAI_API_KEY: "synthetic-stdio-key", + CODEX_API_KEY: "", + CODEX_CLI_PATH: fakeCodexPath, + CODEX_HOME: codexHome, + CODEX_SECURITY_SCAN_ROOT: scanRoot, + CODEX_SECURITY_STATE_DIR: stateDir, + FAKE_CODEX_START_LOG: startLogPath, + FAKE_CODEX_EXIT_LOG: exitLogPath, + FAKE_CODEX_RESTART_CONTROL: controlPath, + PYTHON: pythonWrapperPath, + REAL_PYTHON: process.env.PYTHON?.trim() || "python3", + FAKE_WORKBENCH_LAUNCH_LOG: path.join( + fixtureRoot, + "workbench-launches.jsonl", + ), + FAKE_WORKBENCH_FINALIZER_CONTROL: finalizerControlPath, + FAKE_WORKBENCH_FINALIZER_LOG: finalizerLogPath, + FAKE_WORKBENCH_CANCEL_LOG: path.join(fixtureRoot, "cancel.jsonl"), + FAKE_WORKBENCH_COMMITTED_CONTROL: committedControlPath, + FAKE_WORKBENCH_COMMITTED_LOG: committedLogPath, + FAKE_CODEX_SIGNAL_CHECKPOINT_CONTROL: path.join( + fixtureRoot, + "unused-signal-control", + ), + }; + if (usageMode) { + environment.CODEX_SQLITE_HOME = codexHome; + environment.CODEX_STATE_DB = ""; + await writeFile( + ownerRolloutPath, + [ + { type: "session_meta", payload: { id: threadId, source: "cli" } }, + { + timestamp: new Date().toISOString(), + type: "turn_context", + payload: { turn_id: "native-owner-turn", model: "gpt-5.6-sol" }, + }, + ] + .map((record) => JSON.stringify(record) + "\n") + .join(""), + ); + await execFileAsync(environment.REAL_PYTHON!, [ + "-c", + ` +import sqlite3, sys +with sqlite3.connect(sys.argv[1]) as connection: + connection.execute("CREATE TABLE threads (id TEXT PRIMARY KEY, rollout_path TEXT NOT NULL, cwd TEXT, created_at INTEGER)") + connection.execute("CREATE TABLE thread_spawn_edges (parent_thread_id TEXT NOT NULL, child_thread_id TEXT NOT NULL)") + connection.execute("INSERT INTO threads (id, rollout_path) VALUES (?, ?)", (sys.argv[2], sys.argv[3])) +`, + path.join(codexHome, "state_5.sqlite"), + threadId, + ownerRolloutPath, + ]); + } + const server = startServer(serverBundlePath, environment); + let remote: ReturnType | undefined; + try { + assertNoError( + await server.request(1, "initialize", { + protocolVersion: "2025-11-25", + capabilities: {}, + clientInfo: { name: "deep-scan-detached-completion", version: "0.1.0" }, + }), + ); + server.sendRequest( + 2, + "tools/call", + toolCall( + "start_codex_security_deep_scan", + { targetPath, scope: ".", userContext: "Original discovery input" }, + threadId, + ), + ); + const scanId = await waitForScanId({ server, requestId: 2 }); + await waitForDeepScanWorker({ environment, scanId, threadId }); + const [worker] = await waitForJsonLines(startLogPath, 1); + if (usageMode) await appendOwnerUsage(10); + if (mode === "joined") { + server.sendRequest( + 3, + "tools/call", + toolCall("start_codex_security_deep_scan", { scanId }, threadId), + ); + await waitFor( + () => + server + .stderrEvents() + .some((event) => event.event === "coordinator_joined"), + "second observer to join", + ); + } + if (mode === "remote") { + remote = startServer(serverBundlePath, environment); + assertNoError( + await remote.request(1, "initialize", { + protocolVersion: "2025-11-25", + capabilities: {}, + clientInfo: { name: "remote-observer", version: "0.1.0" }, + }), + ); + remote.sendRequest( + 2, + "tools/call", + toolCall("start_codex_security_deep_scan", { scanId }, threadId), + ); + await waitFor( + () => + remote! + .stderrEvents() + .some((event) => event.event === "coordinator_joined"), + "remote request to observe the existing owner", + ); + } + if ( + mode !== "active-failure" && + mode !== "native-owner-usage" && + mode !== "cancel-after-seal" && + !mode.startsWith("cancel-finalizer") && + !mode.startsWith("late-rejoin") + ) + server.notify("notifications/cancelled", { + requestId: 2, + reason: "detach original observer", + }); + const detached = await getDeepScan({ environment, scanId, threadId }); + const originalPublicScan = await runWorkbench(environment, [ + "get-scan", + "--scan-id", + scanId, + ]); + assert.equal(detached.status, "running"); + assert.equal(detached.cancelRequested, false); + assertProcessAlive(worker.pid); + assertProcessAlive(server.pid!); + + // The already-owned workers proceed after observation ends. + await writeFile(controlPath, "after-restart"); + let finished: Awaited>; + await waitFor(async () => { + finished = await getDeepScan({ environment, scanId, threadId }); + return finished.status === "succeeded"; + }, "detached discovery and reducer to finish"); + assert.equal(finished.workflowVersion, "deep-security-scan/v2"); + assert.equal(finished.finalizationInput.version, 1); + assert.equal( + finished.coordinatorGeneration, + detached.coordinatorGeneration, + ); + assert.equal(finished.userContext, detached.userContext); + assert.deepEqual(finished.usageOwner, detached.usageOwner); + assert.equal( + finished.workers.filter( + (row: PersistedDeepScanWorker) => + row.kind === "discovery" && row.status === "succeeded", + ).length, + 2, + ); + assert.equal( + finished.workers.filter( + (row: PersistedDeepScanWorker) => + row.kind === "dedup" && row.status === "succeeded", + ).length, + 1, + ); + const selectedBytes = await readFile( + path.join(finished.scanDir, finished.finalizationInput.resultPath), + ); + assert.equal( + createHash("sha256").update(selectedBytes).digest("hex"), + finished.finalizationInput.resultSha256, + ); + if (usageMode) { + if (mode === "native-owner-usage") { + assertNoError(await server.waitForResponse(2)); + } else { + await waitFor( + async () => + (await runWorkbench(environment, ["get-scan", "--scan-id", scanId])) + .scan.progress.status === "complete", + "detached native publication", + ); + await appendFile( + ownerRolloutPath, + JSON.stringify({ + timestamp: new Date().toISOString(), + type: "turn_context", + payload: { turn_id: "unrelated-later-turn", model: "gpt-5.6-sol" }, + }) + "\n", + ); + } + await appendOwnerUsage(1010); + const completed = await server.request( + 60, + "tools/call", + toolCall("complete_codex_security_scan", { scanId }, threadId), + ); + assertNoError(completed); + const afterOwner = await runWorkbench(environment, [ + "get-scan", + "--scan-id", + scanId, + ]); + assert.equal(afterOwner.scan.progress.status, "complete"); + assert.equal( + afterOwner.scan.usage.inputTokens, + mode === "native-owner-usage" ? 1010 : 10, + "completion accounts for the owning continuation, excluding later conversation work", + ); + console.log("native owning continuation usage passed", mode, scanId); + return; + } + if (explicitCompletion) { + assertNoError(await server.waitForResponse(2)); + server.sendRequest( + 60, + "tools/call", + toolCall("complete_codex_security_scan", { scanId }, threadId), + ); + } + if (mode === "cancel-after-seal") { + await waitForJsonLines(committedLogPath, 1); + const beforeCancel = await runWorkbench(environment, [ + "get-scan", + "--scan-id", + scanId, + ]); + assert.equal(beforeCancel.scan.progress.status, "complete"); + const sealedBytes = await readFile( + path.join(finished.scanDir, "scan-manifest.json"), + ); + server.sendRequest( + 6, + "tools/call", + toolCall("cancel_codex_security_scan", { scanId }, threadId), + ); + const cancellation = await server.waitForResponse(6); + await rm(committedControlPath, { force: true }); + assertNoError(await server.waitForResponse(60)); + assert.equal(cancellation.result?.isError, true); + const afterCancel = await runWorkbench(environment, [ + "get-scan", + "--scan-id", + scanId, + ]); + assert.equal(afterCancel.scan.progress.status, "complete"); + assert.equal( + afterCancel.scan.executionAttribution.completedAt, + beforeCancel.scan.executionAttribution.completedAt, + ); + assert.deepEqual( + await readFile(path.join(finished.scanDir, "scan-manifest.json")), + sealedBytes, + ); + return; + } + if (mode.startsWith("cancel-finalizer")) { + await waitForJsonLines(finalizerLogPath, 1); + server.sendRequest( + 6, + "tools/call", + toolCall("cancel_codex_security_scan", { scanId }, threadId), + ); + const canceled = await server.waitForResponse(6); + if (mode === "cancel-finalizer-failure") + await writeFile(finalizerControlPath, "failure"); + else await rm(finalizerControlPath, { force: true }); + const ownerResponse = await server.waitForResponse(60); + const publicScan = await runWorkbench(environment, [ + "get-scan", + "--scan-id", + scanId, + ]); + const stopped = await getDeepScan({ environment, scanId, threadId }); + assert.equal( + publicScan.scan.progress.status, + "canceled", + "cancellation observed before parent completion must remain canceled", + ); + assertNoError(canceled); + assert.equal(ownerResponse.result?.isError, true); + assert.match( + ownerResponse.result.content + .map((item: { text: string }) => item.text) + .join(" "), + /Only a running scan can be completed|injected complete-scan failure/, + ); + assert.equal(stopped.status, "canceled"); + assert.deepEqual( + await readFile( + path.join(finished.scanDir, finished.finalizationInput.resultPath), + ), + selectedBytes, + ); + assert.deepEqual(stopped.finalizationInput, finished.finalizationInput); + assert.equal( + publicScan.scan.continuationThreadId, + originalPublicScan.scan.continuationThreadId, + ); + return; + } + if (mode.startsWith("late-rejoin")) { + await waitForJsonLines(finalizerLogPath, 1); + remote = startServer(serverBundlePath, { + ...environment, + FAKE_WORKBENCH_FINALIZER_CONTROL: path.join( + fixtureRoot, + "remote-finalizer-control", + ), + }); + assertNoError( + await remote.request(1, "initialize", { + protocolVersion: "2025-11-25", + capabilities: {}, + clientInfo: { name: "late-remote-observer", version: "0.1.0" }, + }), + ); + remote.sendRequest( + 2, + "tools/call", + toolCall("start_codex_security_deep_scan", { scanId }, threadId), + ); + const observed = await remote!.waitForResponse(2); + const during = await runWorkbench(environment, [ + "get-scan", + "--scan-id", + scanId, + ]); + if (mode === "late-rejoin-failure") + await writeFile(finalizerControlPath, "failure"); + else await rm(finalizerControlPath); + const ownerResponse = await server.waitForResponse(60); + const invocations = await readJsonLines(finalizerLogPath); + assertNoError(observed); + assert.equal( + observed.result.structuredContent.manifestPath, + path.join(finished.scanDir, "scan-manifest.json"), + ); + assert.deepEqual( + await readFile( + path.join(finished.scanDir, finished.finalizationInput.resultPath), + ), + selectedBytes, + ); + assert.equal( + invocations.length, + 1, + "late remote observation must not invoke a competing parent finalizer", + ); + assert.equal( + during.scan.progress.status, + "running", + "only the existing owner may finish pending parent completion", + ); + if (mode === "late-rejoin-failure") { + assert.equal(ownerResponse.result?.isError, true); + assert.equal( + (await runWorkbench(environment, ["get-scan", "--scan-id", scanId])) + .scan.progress.status, + "running", + ); + return; + } + assertNoError(ownerResponse); + } + if (mode === "joined") { + await waitForJsonLines(finalizerLogPath, 1); + // Rejoin while finish-deep-scan is committed but public sealing is blocked. + server.sendRequest( + 4, + "tools/call", + toolCall("start_codex_security_deep_scan", { scanId }, threadId), + ); + await waitFor( + () => + server + .stderrEvents() + .filter((event) => event.event === "coordinator_joined").length === + 2, + "observer to join pending public completion", + ); + await delay(5_100); // Exercise an actual coordinator heartbeat during finalization. + assert.equal(server.response(3), undefined); + assert.equal(server.response(4), undefined); + assert.equal( + (await runWorkbench(environment, ["get-scan", "--scan-id", scanId])) + .scan.progress.status, + "running", + ); + assert.equal((await readJsonLines(finalizerLogPath)).length, 1); + await rm(finalizerControlPath); + for (const id of [3, 4]) assertNoError(await server.waitForResponse(id)); + } + if (mode === "remote") { + // A remote observer retains the aggregate-ready response. It cannot run + // the owning process's public finalizer, even after Deep itself succeeds. + const observed = await remote!.waitForResponse(2); + await waitForJsonLines(finalizerLogPath, 1); + assertNoError(observed); + assert.equal( + observed.result.structuredContent.manifestPath, + path.join(finished.scanDir, "scan-manifest.json"), + ); + assert.equal((await readJsonLines(finalizerLogPath)).length, 1); + await rm(finalizerControlPath); + } + if ( + mode === "failure" || + mode === "active-failure" || + mode === "remote-replay" || + mode === "snapshot-observer" || + mode.startsWith("lost-response") + ) { + if (mode !== "active-failure") + await waitFor( + () => + server + .stderrEvents() + .some( + (event) => event.event === "coordinator_publication_pending", + ), + "public finalization failure to remain pending", + ); + if (mode === "active-failure") { + const failed = await server.waitForResponse(60); + assert.equal(failed.result?.isError, true); + const message = failed.result.content + .map((item: { text: string }) => item.text) + .join(" "); + assert.match(message, /injected complete-scan failure/); + assert.match(message, /Do not retry completion/); + assert.match(message, /no final|Do not.*final/i); + } + const pending = await getDeepScan({ environment, scanId, threadId }); + assert.equal(pending.status, "succeeded"); + assert.equal(pending.terminalReason, finished.terminalReason); + assert.deepEqual(pending.finalizationInput, finished.finalizationInput); + assert.equal( + (await readJsonLines(finalizerLogPath)).length, + 1, + "the owner does not add a retry layer", + ); + const beforeReplay = await runWorkbench(environment, [ + "get-scan", + "--scan-id", + scanId, + ]); + assert.equal( + beforeReplay.scan.progress.status, + !mode.startsWith("lost-response") ? "running" : "complete", + ); + const manifestBeforeReplay = await readFile( + path.join(finished.scanDir, "scan-manifest.json"), + ); + if (mode === "snapshot-observer") { + const observed = await server.request( + 50, + "tools/call", + toolCall( + "start_codex_security_deep_scan", + { targetPath, scope: ".", userContext: "Original discovery input" }, + "independent-snapshot-observer", + ), + ); + assertNoError(observed); + assert.equal(observed.result.structuredContent.scanId, scanId); + assert.equal( + observed.result.structuredContent.manifestPath, + path.join(finished.scanDir, "scan-manifest.json"), + ); + assert.equal( + (await readJsonLines(finalizerLogPath)).length, + 1, + "a snapshot observer must not acquire publication ownership", + ); + } + let replayServer = server; + if (mode === "remote-replay") { + await server.stop(); + remote = startServer(serverBundlePath, environment); + replayServer = remote; + assertNoError( + await remote.request(1, "initialize", { + protocolVersion: "2025-11-25", + capabilities: {}, + clientInfo: { name: "reconstructed-finalizer", version: "0.1.0" }, + }), + ); + } + const damagedArtifact = path.join(finished.scanDir, "findings.json"); + if (mode === "lost-response-corrupt") { + await writeFile( + damagedArtifact, + Buffer.concat([await readFile(damagedArtifact), Buffer.from(" ")]), + ); + } else if (mode === "lost-response-remove") { + await rm(damagedArtifact); + } + const rejoined = await replayServer.request( + 5, + "tools/call", + toolCall("start_codex_security_deep_scan", { scanId }, threadId), + ); + if (mode === "lost-response-corrupt" || mode === "lost-response-remove") { + assert.equal( + rejoined.result?.isError, + true, + "completion replay must reject altered sealed artifacts", + ); + assert.match( + rejoined.result.content + .map((item: { text: string }) => item.text) + .join(" "), + /sealed artifact changed or is missing|findings\.json: expected a file/, + ); + assert.deepEqual( + await readFile(path.join(finished.scanDir, "scan-manifest.json")), + manifestBeforeReplay, + ); + assert.deepEqual( + await readFile( + path.join(finished.scanDir, finished.finalizationInput.resultPath), + ), + selectedBytes, + ); + assert.equal( + (await readJsonLines(startLogPath)).length, + 3, + "rejected replay launches no workers", + ); + const afterReplay = await runWorkbench(environment, [ + "get-scan", + "--scan-id", + scanId, + ]); + assert.equal(afterReplay.scan.progress.status, "complete"); + assert.equal( + afterReplay.scan.executionAttribution.completedAt, + beforeReplay.scan.executionAttribution.completedAt, + ); + console.log( + "native selected completion integrity passed", + mode, + scanId, + ); + return; + } + assertNoError(rejoined); + if (!mode.startsWith("lost-response")) + assertNoError( + await replayServer.request( + 61, + "tools/call", + toolCall("complete_codex_security_scan", { scanId }, threadId), + ), + ); + assert.equal( + rejoined.result.structuredContent.manifestPath, + path.join(finished.scanDir, "scan-manifest.json"), + ); + if (mode.startsWith("lost-response")) { + assert.deepEqual( + await readFile(path.join(finished.scanDir, "scan-manifest.json")), + manifestBeforeReplay, + ); + const afterReplay = await runWorkbench(environment, [ + "get-scan", + "--scan-id", + scanId, + ]); + assert.equal(afterReplay.scan.progress.status, "complete"); + assert.equal( + afterReplay.scan.executionAttribution.completedAt, + beforeReplay.scan.executionAttribution.completedAt, + ); + } + assert.deepEqual( + (await getDeepScan({ environment, scanId, threadId })) + .finalizationInput, + finished.finalizationInput, + ); + } + let publicScan: Awaited>; + await waitFor(async () => { + publicScan = await runWorkbench(environment, [ + "get-scan", + "--scan-id", + scanId, + ]); + return publicScan.scan.progress.status === "complete"; + }, "public completion without another observer"); + assert.equal(publicScan.scan.progress.status, "complete"); + const manifest = JSON.parse( + await readFile(path.join(finished.scanDir, "scan-manifest.json"), "utf8"), + ); + assert.equal(typeof manifest.scan.sealedAt, "string"); + assert.equal(typeof manifest.scan.completedAt, "string"); + assert.equal( + publicScan.scan.continuationThreadId, + originalPublicScan.scan.continuationThreadId, + ); + assert.deepEqual( + publicScan.scan.executionAttribution.owner, + originalPublicScan.scan.executionAttribution.owner, + ); + assert.equal(publicScan.scan.executionAttribution.owner.threadId, threadId); + assert.equal( + (await readJsonLines(startLogPath)).length, + 3, + "completion and replay launch no extra model workers", + ); + assert.equal( + (await readJsonLines(finalizerLogPath)).length, + mode === "failure" || + mode === "active-failure" || + mode === "remote-replay" || + mode === "snapshot-observer" + ? 2 + : 1, + ); + assertProcessAlive(mode === "remote-replay" ? remote!.pid! : server.pid!); + console.log("native selected completion passed", mode, scanId); + } catch (error) { + (error as Error).message += `\nMCP stderr:\n${server.stderrText()}`; + throw error; + } finally { + await remote?.stop(); + await server.stop(); + if (!installedPluginRoot) await rm(serverBundlePath, { force: true }); + await rm(fixtureRoot, { recursive: true, force: true }); + } + async function appendOwnerUsage(inputTokens: number) { + await appendFile( + ownerRolloutPath, + JSON.stringify({ + timestamp: new Date().toISOString(), + type: "event_msg", + payload: { + type: "token_count", + info: { + total_token_usage: { + input_tokens: inputTokens, + cached_input_tokens: 0, + cache_write_input_tokens: 0, + output_tokens: 1, + reasoning_output_tokens: 0, + total_tokens: inputTokens + 1, + }, + }, + }, + }) + "\n", + ); + } +} + async function testDeepScanStdioLifecycle() { const fixtureRoot = await temporaryDirectory("codex-security-deep-stdio-"); const targetPath = path.join(fixtureRoot, "target"); const failedTargetPath = path.join(fixtureRoot, "failed-target"); const stateDir = path.join(fixtureRoot, "state"); const codexHome = path.join(fixtureRoot, "codex [home]"); - const installedPluginRoot = path.join(codexHome, "plugins", "installed"); + const fixturePluginRoot = path.join(codexHome, "plugins", "installed"); const runtimeConfigPath = path.join(fixtureRoot, "active-config.toml"); const startLogPath = path.join(fixtureRoot, "fake-codex-started.jsonl"); const exitLogPath = path.join(fixtureRoot, "fake-codex-exited.jsonl"); @@ -70,9 +819,11 @@ async function testDeepScanStdioLifecycle() { "workbench-launches.jsonl", ); const serverBundlePath = path.join( - installedPluginRoot, + fixturePluginRoot, "mcp", - `.deep-scan-stdio-test-${randomUUID()}.cjs`, + installedPluginRoot + ? "server.mjs" + : `.deep-scan-stdio-test-${randomUUID()}.cjs`, ); const threadId = "deep-scan-stdio-lifecycle-thread"; @@ -84,10 +835,11 @@ async function testDeepScanStdioLifecycle() { "references", "schemas", ".codex-plugin", + ...(installedPluginRoot ? ["mcp"] : []), ]) { await cp( path.join(pluginRoot, directory), - path.join(installedPluginRoot, directory), + path.join(fixturePluginRoot, directory), { recursive: true }, ); } @@ -152,7 +904,8 @@ model_reasoning_summary = "none" ); await writePythonWrapper(`${pythonWrapperPath}.mjs`); await symlink(`${pythonWrapperPath}.mjs`, pythonWrapperPath); - await buildServer(serverBundlePath, { target: "node20" }); + if (!installedPluginRoot) + await buildServer(serverBundlePath, { target: "node20" }); const environment = { ...process.env, @@ -273,7 +1026,7 @@ model_reasoning_summary = "none" )?.artifactDir; assert.equal(typeof startedArtifactRoot, "string"); await writeFile( - path.join(installedPluginRoot, "scripts", "workbench_db.py"), + path.join(fixturePluginRoot, "scripts", "workbench_db.py"), 'raise RuntimeError("Installed plugin helpers replaced during another scan")\n', ); assert.equal(workerContext.pluginRoot, pluginRoot); @@ -323,13 +1076,13 @@ model_reasoning_summary = "none" // Another client can replace this shared install while the scan is active. await writeFile( - path.join(installedPluginRoot, "scripts", "workbench_db.py"), + path.join(fixturePluginRoot, "scripts", "workbench_db.py"), "raise RuntimeError('synthetic replacement plugin helper')\n", ); - await rm(path.join(installedPluginRoot, "references"), { + await rm(path.join(fixturePluginRoot, "references"), { recursive: true, }); - await rm(path.join(installedPluginRoot, "schemas"), { recursive: true }); + await rm(path.join(fixturePluginRoot, "schemas"), { recursive: true }); // Discovery progress is admitted once the first complete Standard worker is active. const discoveryProgress = await server.request( @@ -694,7 +1447,7 @@ model_reasoning_summary = "none" "the MCP server must remain responsive after canceling one scan", ); - const resumedThreadId = "deep-scan-stdio-resumed-thread"; + let resumedThreadId = "deep-scan-stdio-resumed-thread"; const opened = await server.request( 24, "tools/call", @@ -712,7 +1465,13 @@ model_reasoning_summary = "none" "tools/call", toolCall( "submit_codex_security_setup", - { sessionId, targetPath, scope: ".", mode: "deep" }, + { + sessionId, + targetPath, + scope: ".", + mode: "deep", + userContext: "Original discovery focus", + }, resumedThreadId, ), ), @@ -725,7 +1484,7 @@ model_reasoning_summary = "none" assertNoError(started); const resumedScan = started.result.structuredContent.workspace.results; const resumedScanId = resumedScan.scanId; - const handoffClaimToken = randomUUID(); + let handoffClaimToken = randomUUID(); for (const [id, name, arguments_] of [ [ 27, @@ -798,6 +1557,35 @@ model_reasoning_summary = "none" const completedDraft = await readJson(completedWorker.resultManifestPath); assert.equal(completedDraft.scanId, resumedScanId); assert.deepEqual(completedDraft.findings, []); + assert.equal( + partial!.workflowVersion, + "deep-security-scan/v2", + "new scans use selected finalization by default", + ); + assert.equal(partial!.userContext, "Original discovery focus"); + assert.equal(partial!.usageOwner.threadId, resumedThreadId); + const settingsPath = path.join( + resumedScan.scanDir, + "artifacts", + "deep_discovery", + "execution-settings.json", + ); + const originalSettings = await readFile(settingsPath, "utf8"); + assertNoError( + await server.request( + 30, + "tools/call", + toolCall( + "update_codex_security_scan_context", + { + scanId: resumedScanId, + handoffClaimToken, + userContext: "Later result discussion", + }, + resumedThreadId, + ), + ), + ); await server.stop(); assert.throws( () => process.kill(server.pid!, 0), @@ -843,7 +1631,36 @@ model_reasoning_summary = "none" path.join(stateDir, "workbench.sqlite3"), resumedScanId, ]); + await runWorkbench(environment, [ + "release-handoff-delivery", + "--scan-id", + resumedScanId, + "--claim-token", + handoffClaimToken, + ]); + handoffClaimToken = randomUUID(); + resumedThreadId = "deep-scan-stdio-replacement-thread"; + await runWorkbench(environment, [ + "claim-handoff-delivery", + "--scan-id", + resumedScanId, + "--claim-token", + handoffClaimToken, + ]); + await runWorkbench(environment, [ + "attach-scan-continuation-thread", + "--scan-id", + resumedScanId, + "--claim-token", + handoffClaimToken, + "--thread-id", + resumedThreadId, + ]); await writeFile(restartControlPath, "after-restart"); + await writeFile( + runtimeConfigPath, + 'model_reasoning_summary = "detailed"\n', + ); const resumedStartIndex = (await readJsonLines(startLogPath)).length; const restartedServer = startServer(serverBundlePath, environment); @@ -878,7 +1695,11 @@ model_reasoning_summary = "none" resumedScanId, ], ); - assert.deepEqual(JSON.parse(modelRows.stdout), ["gpt-6.1-sol", "max"]); + const recordedSettings = JSON.parse(originalSettings).settings; + assert.deepEqual(JSON.parse(modelRows.stdout), [ + recordedSettings.model, + recordedSettings.reasoningEffort, + ]); const instructions = resumed.result.structuredContent.instructions; assert.match( instructions, @@ -919,6 +1740,25 @@ model_reasoning_summary = "none" partial!.coordinatorGeneration + 1, ); assert.equal(finished.dispatchedCount, 2); + assert.equal(finished.workflowVersion, partial!.workflowVersion); + assert.equal( + finished.finalizationInput.version, + 1, + "recovery selects a persisted finalization input", + ); + assert.equal(finished.userContext, partial!.userContext); + assert.equal( + finished.createdAt, + partial!.createdAt, + "recovery retains the original deadline origin", + ); + assert.equal(finished.config.maxTimeHours, partial!.config.maxTimeHours); + assert.deepEqual( + finished.usageOwner, + partial!.usageOwner, + "a replacement continuation does not rebind original usage", + ); + assert.equal(await readFile(settingsPath, "utf8"), originalSettings); const successfulDiscoveries = finished.workers.filter( (worker: PersistedDeepScanWorker) => worker.kind === "discovery" && worker.status === "succeeded", @@ -986,7 +1826,6 @@ model_reasoning_summary = "none" restartStartIndex, ); for (const [index, execution] of executions.entries()) { - const afterRestart = index + restartStartIndex >= resumedStartIndex; assertReadOnlyWorkerInvocation(execution.argv, codexHome); assert.equal(execution.readCoreScanReference, true); const context = discoveryPromptContext(execution.stdin); @@ -999,14 +1838,10 @@ model_reasoning_summary = "none" pluginRoot, pythonWrapperPath, ); - assertFlagPair( - execution.argv, - "--model", - afterRestart ? "gpt-6.1-sol" : "gpt-5.6-sol", - ); + assertFlagPair(execution.argv, "--model", recordedSettings.model); assert.ok( execution.argv.includes( - `model_reasoning_effort=${JSON.stringify(afterRestart ? "max" : "high")}`, + `model_reasoning_effort=${JSON.stringify(recordedSettings.reasoningEffort)}`, ), ); @@ -1014,6 +1849,8 @@ model_reasoning_summary = "none" execution.argv.includes('model_reasoning_summary="none"'), true, ); + if (context.workerLabel) + assert.equal(context.userContext, "Original discovery focus"); } assert.equal( executions.filter( @@ -1038,7 +1875,7 @@ model_reasoning_summary = "none" throw error; } finally { await server.stop(); - await rm(serverBundlePath, { force: true }); + if (!installedPluginRoot) await rm(serverBundlePath, { force: true }); await rm(fixtureRoot, { recursive: true, force: true }); parentSandboxState = readOnlyParentSandboxState(pluginRoot); } @@ -1464,7 +2301,10 @@ async function writeFakeCodex(executablePath: string) { `#!/usr/bin/env node import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import path from "node:path"; +import { parse as parseToml } from ${JSON.stringify(import.meta.resolve("smol-toml"))}; if (process.argv.includes('app-server')) { + const permissionOverride = process.argv.find((arg) => arg.startsWith("permissions.") && parseToml(arg).permissions?.codex_security_deep_scan_worker); + const effectiveProfile = parseToml(permissionOverride).permissions.codex_security_deep_scan_worker; ${jsonLineInputSource(` if (message.method === 'initialized') continue; let result; if (message.method === 'initialize') { @@ -1472,7 +2312,7 @@ ${jsonLineInputSource(` if (message.method === 'initialized') continue; } else if (message.method === 'thread/read') { result = { thread: { id: message.params.threadId, path: path.join(process.env.CODEX_HOME, message.params.threadId + '.jsonl') } }; } else if (message.method === 'config/read') { - result = { config: { default_permissions: 'codex_security_deep_scan_worker', permissions: { codex_security_deep_scan_worker: { extends: ':read-only', filesystem: { ':root': 'read', [process.env.FAKE_CODEX_DENIED_HOME]: { '.': 'deny' } }, network: { enabled: false } } } }, origins: {}, layers: null }; + result = { config: { default_permissions: 'codex_security_deep_scan_worker', permissions: { codex_security_deep_scan_worker: effectiveProfile } }, origins: {}, layers: null }; } else if (message.method === 'permissionProfile/list') { result = { data: [{ id: 'codex_security_deep_scan_worker', description: null, allowed: true }], nextCursor: null }; } else if (message.method === 'account/read') { @@ -1490,6 +2330,9 @@ const root = process.argv[process.argv.indexOf('--cd') + 1]; const readCoreScanReference = !context.workerLabel || readFileSync(path.join(context.pluginRoot, 'references', 'core-scan.md'), 'utf8').length > 0; appendFileSync(process.env.FAKE_CODEX_START_LOG, JSON.stringify({ pid: process.pid, argv: process.argv.slice(2), stdin, readCoreScanReference, hasExpectedApiKey: process.env.CODEX_API_KEY === 'synthetic-stdio-key', knowledgeBase: process.env.CODEX_SECURITY_KNOWLEDGE_BASE }) + '\\n'); console.log(JSON.stringify({ type: 'thread.started', thread_id: \`stdio-fixture-\${process.pid}\` })); +while (existsSync(process.env.FAKE_CODEX_RESTART_CONTROL) && readFileSync(process.env.FAKE_CODEX_RESTART_CONTROL, 'utf8') === 'wait-for-completion') { + await new Promise((resolve) => setTimeout(resolve, 25)); +} if (existsSync(process.env.FAKE_CODEX_RESTART_CONTROL)) { const phase = readFileSync(process.env.FAKE_CODEX_RESTART_CONTROL, 'utf8'); if (phase === 'after-restart' || context.workerLabel === 'discovery-0001') { @@ -1528,26 +2371,52 @@ process.once('SIGTERM', () => stop('SIGTERM')); async function writePythonWrapper(executablePath: string) { await writeFile( executablePath, - `#!/usr/bin/env node -import { appendFileSync, existsSync, unlinkSync } from "node:fs"; -import { spawnSync } from "node:child_process"; -const args = process.argv.slice(2); -const input = Buffer.concat(await process.stdin.toArray()); -const workbenchArgs = JSON.parse(input.subarray(0, input.indexOf(10)).toString('utf8')); -appendFileSync(process.env.FAKE_WORKBENCH_LAUNCH_LOG, JSON.stringify({ args, workbenchArgs, cwd: process.cwd() }) + '\\n'); -const control = process.env.FAKE_WORKBENCH_CANCEL_FAILURE_CONTROL; -if (workbenchArgs[0] === 'cancel-scan') { - appendFileSync(process.env.FAKE_WORKBENCH_CANCEL_LOG, JSON.stringify(workbenchArgs) + '\\n'); -} -if (workbenchArgs[0] === 'cancel-scan' && control && existsSync(control)) { - unlinkSync(control); - console.error('injected cancel-scan failure'); - process.exit(1); -} -const result = spawnSync(process.env.REAL_PYTHON || 'python3', args, { input, stdio: ['pipe', 'inherit', 'inherit'] }); -if (result.error) throw result.error; -process.exit(result.status ?? 1); -`, + [ + "#!/usr/bin/env node", + 'import { appendFileSync, existsSync, readFileSync, unlinkSync } from "node:fs";', + 'import { spawnSync } from "node:child_process";', + "const args = process.argv.slice(2);", + "const input = Buffer.concat(await process.stdin.toArray());", + "const codeIndex = args.indexOf('-c');", + "const framed = codeIndex !== -1 && args[codeIndex + 1].includes('json.loads(sys.stdin.buffer.readline())');", + "const workbenchArgs = framed ? JSON.parse(input.subarray(0, input.indexOf(10)).toString('utf8')) : args.slice(1);", + "appendFileSync(process.env.FAKE_WORKBENCH_LAUNCH_LOG, JSON.stringify({ args, workbenchArgs, cwd: process.cwd() }) + '\\n');", + "const control = process.env.FAKE_WORKBENCH_CANCEL_FAILURE_CONTROL;", + "if (workbenchArgs[0] === 'cancel-scan') {", + " appendFileSync(process.env.FAKE_WORKBENCH_CANCEL_LOG, JSON.stringify(args) + '\\n');", + "}", + "if (workbenchArgs[0] === 'cancel-scan' && control && existsSync(control)) {", + " unlinkSync(control);", + " console.error('injected cancel-scan failure');", + " process.exit(1);", + "}", + "let finalizerMode;", + "if (workbenchArgs[0] === 'complete-scan' && process.env.FAKE_WORKBENCH_FINALIZER_LOG) {", + " appendFileSync(process.env.FAKE_WORKBENCH_FINALIZER_LOG, JSON.stringify(args) + '\\n');", + " const finalizerControl = process.env.FAKE_WORKBENCH_FINALIZER_CONTROL;", + " const readFinalizerControl = () => {", + " try { return readFileSync(finalizerControl, 'utf8'); }", + " catch (error) { if (error.code !== 'ENOENT') throw error; }", + " };", + " while ((finalizerMode = readFinalizerControl()) === 'wait') {", + " await new Promise((resolve) => setTimeout(resolve, 25));", + " }", + " if (finalizerMode !== undefined) {", + " unlinkSync(finalizerControl);", + " }", + " if (finalizerMode === 'failure') { console.error('injected complete-scan failure'); process.exit(1); }", + "}", + "const result = spawnSync(process.env.REAL_PYTHON || 'python3', args, { input, stdio: ['pipe', 'inherit', 'inherit'] });", + "const committedControl = process.env.FAKE_WORKBENCH_COMMITTED_CONTROL;", + "if (workbenchArgs[0] === 'complete-scan' && result.status === 0 && committedControl && existsSync(committedControl)) {", + " appendFileSync(process.env.FAKE_WORKBENCH_COMMITTED_LOG, JSON.stringify({ pid: process.pid, committed: true }) + '\\n');", + " while (existsSync(committedControl)) await new Promise((resolve) => setTimeout(resolve, 25));", + "}", + "if (result.error) throw result.error;", + "if (result.status === 0 && finalizerMode === 'lost-response') { console.error('injected lost completion response'); process.exit(1); }", + "process.exit(result.status ?? 1);", + "", + ].join("\n"), ); await chmod(executablePath, 0o755); } diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_store.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_store.ts index 478aea00bf..d8f1d6ed97 100644 --- a/plugins/codex-security/mcp-app/tests/test_deep_scan_store.ts +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_store.ts @@ -13,15 +13,19 @@ const { WorkbenchDeepScanStore, parseDeepScan } = await importSource( ); await testBeginProtocolAndParsing(); -await testScopeResponseParsing(); +await testBeginCarriesOriginalSettingsWithUserContext(); +await testClaimChecksOriginalSettingsInsideTheTransaction(); await testCanonicalCommitProtocol(); await testTerminalProtocol(); testRunErrorParsing(); +testWorkflowVersionParsing(); +await testScopeResponseParsing(); testLegacyResponseNormalization(); testConfiguredMaximumDurationParsing(); await testWriteSerializationAndRecovery(); await testBeginUsesTheWriteQueue(); await testHeartbeatBypassesBlockedWriteQueue(); +await testReleaseRetiresTheOwnedLeaseThroughWorkbench(); await testOwnershipReadClearsStaleLease(); await testWorkerResponseParsing(); await testReplaceableDiscoveryFailureProtocol(); @@ -32,6 +36,94 @@ await testPersistenceRetryExhaustionPreservesDiagnostics(); await testDeterministicPersistenceFailuresAreNotRetried(); await testNonIdempotentMutationsAreNotRetried(); testInvalidPersistedConfig(); +testOriginalUsageOwnerParsing(); + +function testOriginalUsageOwnerParsing() { + const value = stateResult(randomUUID()); + const usageOwner = { + threadId: "original-thread", + turnId: "original-turn", + startedAt: "2026-01-01T00:00:00Z", + }; + assert.deepEqual( + parseDeepScan({ deepScan: { ...value.deepScan, usageOwner } }).usageOwner, + usageOwner, + ); + assert.equal( + parseDeepScan({ deepScan: { ...value.deepScan, usageOwner: null } }) + .usageOwner, + null, + ); + assert.equal( + parseDeepScan(value).usageOwner, + null, + "old readers do not establish an original owner", + ); +} + +async function testBeginCarriesOriginalSettingsWithUserContext() { + const executionSettings = { + codexPath: "/fixture/codex", + codexHome: "/fixture/home", + model: "original-model", + reasoningSummary: "concise", + }; + const userContext = "Review the parser.\nKeep this second line."; + const runner = async ( + args: string[], + input?: string, + selectFinalization?: boolean, + withExecutionSettings?: boolean, + ) => { + assert.equal(selectFinalization, false); + assert.equal(withExecutionSettings, true); + assert.deepEqual(JSON.parse(input!), { executionSettings, userContext }); + assert.equal( + args.includes("--user-context-stdin"), + false, + "the private structured input carries context without a second stdin consumer", + ); + assert.equal( + args.some((arg) => arg.includes("execution-settings")), + false, + "no public argument is added for the internal settings handoff", + ); + return stateResult(randomUUID(), { startDisposition: "created" }); + }; + await new WorkbenchDeepScanStore(runner).begin({ + targetPath: "/fixture/repository", + threadId: "fixture-thread", + scanRoot: "/fixture/scans", + userContext, + executionSettings, + }); +} + +async function testClaimChecksOriginalSettingsInsideTheTransaction() { + const scanId = randomUUID(); + const store = new WorkbenchDeepScanStore( + async ( + args: string[], + input?: string, + selectFinalization?: boolean, + withExecutionSettings?: boolean, + ) => { + assert.equal(args[0], "claim-deep-scan-coordinator"); + assert.equal(input, undefined); + assert.equal(selectFinalization, false); + assert.equal(withExecutionSettings, true); + return { + ...stateResult(scanId, { deepScan: { coordinatorGeneration: 2 } }), + coordinatorDisposition: "claimed", + }; + }, + ); + assert.equal( + (await store.claimCoordinator({ scanId, threadId: "fixture-thread" })) + .acquired, + true, + ); +} async function testBeginProtocolAndParsing() { const scanId = randomUUID(); @@ -79,7 +171,7 @@ async function testBeginProtocolAndParsing() { ); assert.equal( flagValue(calls[0].args, "--workflow-version"), - "deep-scan-mcp/v1", + "deep-security-scan/v2", ); const claimToken = randomUUID(); @@ -584,7 +676,7 @@ async function testPersistenceRetriesRemainInsideTheWriteQueue() { if (args[0] === "claim-deep-scan-dedup" && calls.length === 1) { throw new Error("sqlite3.OperationalError: database is locked"); } - return {}; + return stateResult(scanId); }); const claim = store.claimDedup({ @@ -851,7 +943,7 @@ function idempotentPersistenceScenarios() { })), { operation: "claim-deep-scan-dedup", - result: {}, + result: stateResult(scanId), invoke: (store: Store) => store.claimDedup({ id: reducerId, @@ -1084,3 +1176,119 @@ function repeatedFlagValues(args: string[], flag: string) { value === flag ? [args[index + 1]] : [], ); } + +function testWorkflowVersionParsing() { + const value = stateResult(randomUUID()).deepScan; + const run = parseDeepScan({ + deepScan: { + ...value, + schemaVersion: 1, + workflowVersion: "deep-scan-mcp/v1", + model: "original-model", + reasoningEffort: "high", + }, + }); + assert.equal(run.model, "original-model"); + assert.equal(run.reasoningEffort, "high"); + assert.equal(run.schemaVersion, 1); + assert.equal(run.workflowVersion, "deep-scan-mcp/v1"); + const future = parseDeepScan({ + deepScan: { ...value, schemaVersion: 99, workflowVersion: "future/v99" }, + }); + assert.equal(future.schemaVersion, 99); + assert.equal( + future.workflowVersion, + "future/v99", + "inspection preserves unsupported versions", + ); +} + +for (const version of [1, 99]) { + const finalizationInput = { + version, + resultPath: null, + resultSha256: null, + terminalReason: "capped", + omittedWorkerIds: ["fixture-worker"], + selectedAt: "2026-01-01T00:00:00Z", + }; + assert.deepEqual( + parseDeepScan( + stateResult(randomUUID(), { deepScan: { finalizationInput } }), + ).finalizationInput, + finalizationInput, + "inspection preserves finalization input and version before execution compatibility checks", + ); +} + +async function testReleaseRetiresTheOwnedLeaseThroughWorkbench() { + const scanId = randomUUID(); + const handoffClaimToken = randomUUID(); + const scanDir = await mkdtemp(join(tmpdir(), "deep-scan-release-")); + const calls: { args: string[]; release: boolean }[] = []; + const store = new WorkbenchDeepScanStore( + async ( + args: string[], + _input?: string, + _selectFinalization?: boolean, + _withExecutionSettings?: boolean, + _signal?: AbortSignal, + releaseCoordinator = false, + ) => { + calls.push({ args, release: releaseCoordinator }); + return { + ...stateResult(scanId, { + deepScan: { coordinatorGeneration: 2, scanDir }, + }), + coordinatorDisposition: "claimed", + }; + }, + ); + try { + const input = { scanId, threadId: "thread-fixture", handoffClaimToken }; + await store.claimCoordinator(input); + const heartbeat = store.heartbeatCoordinator(input); + await store.releaseCoordinator(scanId); + const renewed = await heartbeat; + assert.equal(calls.length, 2); + assert.equal(calls[0]!.release, false); + assert.equal( + calls[1]!.release, + true, + "release must reach the authoritative host mutation", + ); + assert.deepEqual(calls[1]!.args, [ + "claim-deep-scan-coordinator", + "--scan-id", + scanId, + "--thread-id", + input.threadId, + "--coordinator-generation", + "2", + "--claim-token", + handoffClaimToken, + ]); + assert.deepEqual( + await readJson( + scanDir, + "artifacts", + "deep_discovery", + "coordinator-heartbeat-2.json", + ), + { + coordinatorGeneration: 2, + updatedAt: renewed.updatedAt, + }, + "artifact heartbeat data must not retire a persisted lease", + ); + assert.deepEqual(store.coordinatorLeaseArgs(scanId), []); + await store.releaseCoordinator(scanId); + assert.equal( + calls.length, + 2, + "a retired local lease must not release a later owner", + ); + } finally { + await rm(scanDir, { recursive: true, force: true }); + } +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_store_integration.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_store_integration.ts index 839bf22110..87f4ffac05 100644 --- a/plugins/codex-security/mcp-app/tests/test_deep_scan_store_integration.ts +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_store_integration.ts @@ -6,7 +6,7 @@ import { mock } from "node:test"; import { execFile, spawn } from "node:child_process"; import { randomUUID } from "node:crypto"; import { once } from "node:events"; -import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises"; import path from "node:path"; import { promisify } from "node:util"; import { importModule } from "./import-module.ts"; @@ -16,6 +16,7 @@ import type { } from "../src/artifact-scan-draft.js"; import { DeepScanCoordinator, + DeepScanCoordinatorRegistry, FakeExecutor, } from "./deep_scan_coordinator_fixture.ts"; @@ -26,17 +27,34 @@ const workbenchPath = path.join(pluginRoot, "scripts", "workbench_db.py"); const { WorkbenchDeepScanStore, createScanArtifactContext, + captureDeepScanExecutionSettings, recordCodexSecurityScanDraftViaWorkbench, + startDeepScanEngine, saveScanDraftCheckpoint, } = await importModule({ stdin: { contents: `export { WorkbenchDeepScanStore } from "./src/deep-scan/store.ts"; export { createScanArtifactContext } from "./src/artifact-context.ts"; -export { recordCodexSecurityScanDraftViaWorkbench, saveScanDraftCheckpoint } from "./src/artifact-scan-draft.ts";`, +export { captureDeepScanExecutionSettings } from "./src/deep-scan/recovery-settings.ts"; +export { recordCodexSecurityScanDraftViaWorkbench, saveScanDraftCheckpoint } from "./src/artifact-scan-draft.ts"; +export { startDeepScanEngine } from "./src/deep-scan/engine.ts";`, resolveDir: mcpAppRoot, }, + loader: { ".md": "text" }, + define: { + "import.meta.url": JSON.stringify( + new URL("../src/deep-scan/engine.ts", import.meta.url).href, + ), + }, + banner: { + js: `import { createRequire as fixtureRequire } from "node:module"; const require = fixtureRequire(${JSON.stringify(import.meta.url)});`, + }, }); +await testSelectionResponseAndReadFailure(true); +await testSelectionResponseAndReadFailure(false); +await testNativeSelectedFinalizationWithoutCheckout("deleted"); +await testNativeSelectedFinalizationWithoutCheckout("moved"); await testCanonicalParentCheckpointsSurviveSuccessiveWrites(); await testFreeformFailureMessagesAgainstRealWorkbench(); await testWorkerRetryThreadsRemainInScanUsage(); @@ -59,6 +77,286 @@ await testCoordinatorCommitResponseRecovery(); await testCoordinatorCommitResponseRecovery("commit-deep-scan-dedup"); await testCoordinatorCommitResponseRecovery("finish-deep-scan"); +async function testSelectionResponseAndReadFailure(validationFailure: boolean) { + const { fixtureRoot, targetPath, environment } = await createWorkbenchFixture( + "deep-selected-response-", + ); + const threadId = "selection-response-owner"; + const runWorkbench = createWorkbenchRunner(environment); + const readStore = new WorkbenchDeepScanStore(runWorkbench); + const lostReply = Object.assign(new Error("Synthetic selection reply lost"), { + code: "ETIMEDOUT", + }); + let selectionWrites = 0; + let workerWrites = 0; + let failureWrites = 0; + let failedStatusReads = 0; + let committedSelection: unknown; + let prematurePublication = 0; + const store = new WorkbenchDeepScanStore( + async ( + args: string[], + input?: string, + selectFinalization = false, + withExecutionSettings = false, + signal?: AbortSignal, + releaseCoordinator = false, + ) => { + if (args[0] === "fail-deep-scan") failureWrites += 1; + if (args[0] === "upsert-deep-scan-worker") workerWrites += 1; + if ( + !validationFailure && + args[0] === "get-deep-scan" && + selectionWrites > 0 && + failedStatusReads === 0 + ) { + failedStatusReads += 1; + throw Object.assign( + new Error("Synthetic selected-state read unavailable"), + { code: "ETIMEDOUT" }, + ); + } + if (selectFinalization) { + selectionWrites += 1; + if (validationFailure) { + const invalid = [...args]; + invalid[invalid.indexOf("--terminal-reason") + 1] = + "invalid-synthetic-reason"; + return await runWorkbench( + invalid, + input, + true, + withExecutionSettings, + signal, + releaseCoordinator, + ); + } + } + const result = await runWorkbench( + args, + input, + selectFinalization, + withExecutionSettings, + signal, + releaseCoordinator, + ); + if (selectFinalization) { + const saved = await readStore.get( + args[args.indexOf("--scan-id") + 1], + threadId, + ); + committedSelection ??= saved.finalizationInput; + assert.deepEqual(saved.finalizationInput, committedSelection); + throw lostReply; + } + return result; + }, + ); + try { + await mkdir(path.join(environment.CODEX_HOME, "codex-security"), { + recursive: true, + }); + await writeFile( + path.join(environment.CODEX_HOME, "codex-security", "config.toml"), + "[deep_scan]\nworkers = 1\nstop_after_no_new = 1\nmax_discovery_runs = 1\n", + ); + const run = await store.begin({ + executionSettings: await captureDeepScanExecutionSettings( + {}, + { filesystemDenies: [] }, + environment, + ), + targetPath, + threadId, + scanRoot: path.join(fixtureRoot, "scans"), + }); + const claim = await store.claimCoordinator({ + scanId: run.scanId, + threadId, + }); + const executor = new FakeExecutor(); + const coordinator = new DeepScanCoordinator({ + run: claim.run, + store, + pluginRoot, + threadId, + executor, + heartbeatIntervalMs: 60_000, + onComplete: async () => { + prematurePublication += 1; + throw new Error( + "Unexpected publication before selection acknowledgement", + ); + }, + }); + coordinator.start(); + if (validationFailure) { + const terminal = await coordinator.wait(undefined, 30_000); + assert.equal(terminal?.status, "failed"); + assert.match(terminal?.error ?? "", /invalid-synthetic-reason/); + assert.equal(failureWrites, 1); + assert.equal(prematurePublication, 0); + assert.ok(!(await readStore.get(run.scanId, threadId)).finalizationInput); + return; + } + await assert.rejects( + coordinator.wait(undefined, 30_000), + (error: Error) => { + assert.match(error.message, /Synthetic selection reply lost/); + assert.equal(error.cause, lostReply); + return true; + }, + ); + assert.equal(failedStatusReads, 1); + assert.equal(failureWrites, 0); + assert.equal(prematurePublication, 0); + const selected = await store.get(run.scanId, threadId); + assert.equal(selected.status, "running"); + assert.ok(selected.finalizationInput); + assert.deepEqual(selected.finalizationInput, committedSelection); + const parent = await runWorkbench(["get-scan", "--scan-id", run.scanId]); + assert.equal(parent.scan.progress.status, "running"); + const workerIds = selected.persistedWorkers.map( + (worker: { id: string }) => worker.id, + ); + const writesBeforeResume = { selectionWrites, workerWrites }; + await store.releaseCoordinator(run.scanId); + let finalized = 0; + const resumed = await startDeepScanEngine({ + run: selected, + store, + registry: new DeepScanCoordinatorRegistry(), + runWorkbench, + pluginRoot, + pythonCommand: process.env.PYTHON || "python3", + parentSandbox: { filesystemDenies: [] }, + threadId, + onFinalized: async () => { + finalized += 1; + }, + }); + const terminal = await resumed.coordinator.wait(undefined, 30_000); + assert.equal(terminal?.status, "succeeded"); + assert.equal(finalized, 1); + const persisted = await store.get(run.scanId, threadId); + assert.deepEqual(persisted.finalizationInput, committedSelection); + assert.deepEqual( + persisted.persistedWorkers.map((worker: { id: string }) => worker.id), + workerIds, + ); + assert.deepEqual({ selectionWrites, workerWrites }, writesBeforeResume); + assert.equal(executor.discoveryCalls, 1); + assert.equal(executor.dedupCalls, 1); + assert.equal(failureWrites, 0); + assert.ok( + Array.isArray((await readJson(run.scanDir, "findings.json")).findings), + ); + } finally { + await rm(fixtureRoot, { recursive: true, force: true }); + } +} + +async function testNativeSelectedFinalizationWithoutCheckout( + targetChange: "deleted" | "moved", +) { + const { fixtureRoot, targetPath, environment } = await createWorkbenchFixture( + "deep-native-selected-recovery-", + ); + const runWorkbench = createWorkbenchRunner(environment); + const store = new WorkbenchDeepScanStore(runWorkbench); + const threadId = "selected-recovery-owner"; + try { + await mkdir(path.join(environment.CODEX_HOME, "codex-security"), { + recursive: true, + }); + await writeFile( + path.join(environment.CODEX_HOME, "codex-security", "config.toml"), + "[deep_scan]\nworkers = 1\nstop_after_no_new = 1\nmax_discovery_runs = 1\n", + ); + const run = await store.begin({ + executionSettings: await captureDeepScanExecutionSettings( + {}, + { filesystemDenies: [] }, + environment, + ), + targetPath, + threadId, + scanRoot: path.join(fixtureRoot, "scans"), + }); + const claim = await store.claimCoordinator({ + scanId: run.scanId, + threadId, + }); + const executor = new FakeExecutor(); + const coordinator = new DeepScanCoordinator({ + run: claim.run, + store, + pluginRoot, + threadId, + executor, + onComplete: async () => { + throw new Error("Synthetic interruption before selected publication"); + }, + }); + coordinator.start(); + await assert.rejects( + coordinator.wait(undefined, 30_000), + /Synthetic interruption before selected publication/, + ); + const selected = await store.get(run.scanId, threadId); + assert.equal(selected.status, "running"); + assert.ok(selected.finalizationInput); + const savedTarget = ( + await runWorkbench(["get-scan", "--scan-id", run.scanId]) + ).scan.contract; + const workerIds = selected.persistedWorkers.map( + (worker: { id: string }) => worker.id, + ); + await store.releaseCoordinator(run.scanId); + if (targetChange === "moved") + await rename(targetPath, `${targetPath}-moved`); + else await rm(targetPath, { recursive: true }); + + let finalized = 0; + const resumed = await startDeepScanEngine({ + run: selected, + store, + registry: new DeepScanCoordinatorRegistry(), + runWorkbench, + pluginRoot, + pythonCommand: process.env.PYTHON || "python3", + parentSandbox: { filesystemDenies: [] }, + threadId, + onFinalized: async () => { + finalized += 1; + }, + }); + const terminal = await resumed.coordinator.wait(undefined, 30_000); + assert.equal(terminal?.status, "succeeded"); + assert.equal(finalized, 1); + const persisted = await store.get(run.scanId, threadId); + assert.deepEqual(persisted.finalizationInput, selected.finalizationInput); + assert.deepEqual( + persisted.persistedWorkers.map((worker: { id: string }) => worker.id), + workerIds, + ); + assert.equal(executor.discoveryCalls, 1); + assert.equal(executor.dedupCalls, 1); + const findings = await readJson(run.scanDir, "findings.json"); + assert.ok(Array.isArray(findings.findings)); + const manifest = await readJson(run.scanDir, "scan-manifest.json"); + assert.equal(persisted.targetPath, targetPath); + assert.deepEqual(manifest.scan.target, { + kind: "directory_snapshot", + targetId: savedTarget.target.targetId, + displayName: savedTarget.target.displayName, + snapshotDigest: savedTarget.target.requiredSnapshotDigest, + }); + } finally { + await rm(fixtureRoot, { recursive: true, force: true }); + } +} + async function testCoordinatorCommitResponseRecovery( exhaustOperation?: "commit-deep-scan-dedup" | "finish-deep-scan", ) { @@ -70,44 +368,63 @@ async function testCoordinatorCommitResponseRecovery( const attempts = new Map(); let failureWrites = 0; let committedManifest: string | undefined; - const store = new WorkbenchDeepScanStore(async (args: string[]) => { - const result = await runWorkbench(args); - if (args[0] === "fail-deep-scan") failureWrites += 1; - if (args[0] === "finish-deep-scan") { - committedManifest ??= await readFile( - args[args.indexOf("--manifest-path") + 1], - "utf8", + const store = new WorkbenchDeepScanStore( + async ( + args: string[], + input?: string, + selectFinalization = false, + withExecutionSettings = false, + signal?: AbortSignal, + releaseCoordinator = false, + ) => { + const result = await runWorkbench( + args, + input, + selectFinalization, + withExecutionSettings, + signal, + releaseCoordinator, ); - } - if ( - args[0] !== "commit-deep-scan-dedup" && - args[0] !== "finish-deep-scan" && - !( - args[0] === "upsert-deep-scan-worker" && - args[args.indexOf("--kind") + 1] === "discovery" && - args[args.indexOf("--status") + 1] === "succeeded" + // Selection commits before a parent manifest exists. This case injects lost + // replies after reducer acceptance and final publication, not selection. + if (selectFinalization) return result; + if (args[0] === "fail-deep-scan") failureWrites += 1; + if (args[0] === "finish-deep-scan") { + committedManifest ??= await readFile( + args[args.indexOf("--manifest-path") + 1], + "utf8", + ); + } + if ( + args[0] !== "commit-deep-scan-dedup" && + args[0] !== "finish-deep-scan" && + !( + args[0] === "upsert-deep-scan-worker" && + args[args.indexOf("--kind") + 1] === "discovery" && + args[args.indexOf("--status") + 1] === "succeeded" + ) ) - ) + return result; + const calls = attempts.get(args[0]) ?? []; + calls.push([...args]); + attempts.set(args[0], calls); + if ( + args[0] === "commit-deep-scan-dedup" && + (calls.length === 1 || exhaustCommitRetries) + ) { + throw Object.assign( + new Error("workbench response timed out after commit"), + { + code: "ETIMEDOUT", + }, + ); + } + // Reproduce truncated stdout after the actual SQLite mutation committed. + if (calls.length === 1 || args[0] === exhaustOperation) + return JSON.parse('{"deepScan":'); return result; - const calls = attempts.get(args[0]) ?? []; - calls.push([...args]); - attempts.set(args[0], calls); - if ( - args[0] === "commit-deep-scan-dedup" && - (calls.length === 1 || exhaustCommitRetries) - ) { - throw Object.assign( - new Error("workbench response timed out after commit"), - { - code: "ETIMEDOUT", - }, - ); - } - // Reproduce truncated stdout after the actual SQLite mutation committed. - if (calls.length === 1 || args[0] === exhaustOperation) - return JSON.parse('{"deepScan":'); - return result; - }); + }, + ); try { await mkdir(path.join(environment.CODEX_HOME, "codex-security"), { recursive: true, @@ -118,6 +435,11 @@ async function testCoordinatorCommitResponseRecovery( ); const threadId = "commit-response-owner"; const run = await store.begin({ + executionSettings: await captureDeepScanExecutionSettings( + {}, + { filesystemDenies: [] }, + environment, + ), targetPath, threadId, scanRoot: path.join(fixtureRoot, "scans"), @@ -224,12 +546,37 @@ async function createWorkbenchFixture(prefix: string) { function createWorkbenchRunner(environment: NodeJS.ProcessEnv, bounded = true) { const python = process.env.PYTHON?.trim() || "python3"; - return async (args: string[]) => { - const { stdout } = await execFileAsync(python, [workbenchPath, ...args], { + return async ( + args: string[], + input?: string, + selectFinalization = false, + withExecutionSettings = false, + signal?: AbortSignal, + releaseCoordinator = false, + ) => { + const invocation = releaseCoordinator + ? "release_coordinator=True" + : selectFinalization + ? "select_finalization=True" + : withExecutionSettings + ? "with_execution_settings=True" + : undefined; + const pythonArgs = invocation + ? [ + "-c", + `import runpy, sys; script = sys.argv.pop(1); runpy.run_path(script)['main'](${invocation})`, + workbenchPath, + ...args, + ] + : [workbenchPath, ...args]; + const execution = execFileAsync(python, pythonArgs, { cwd: pluginRoot, env: environment, + signal, ...(bounded ? { timeout: 30_000, maxBuffer: 4 * 1024 * 1024 } : {}), }); + execution.child.stdin?.end(input); + const { stdout } = await execution; return JSON.parse(stdout); }; } @@ -243,6 +590,11 @@ async function testWorkerRetryThreadsRemainInScanUsage() { try { const threadId = "retry-owner"; const run = await store.begin({ + executionSettings: await captureDeepScanExecutionSettings( + {}, + { filesystemDenies: [] }, + environment, + ), targetPath, threadId, scanRoot: path.join(fixtureRoot, "scans"), @@ -293,6 +645,11 @@ async function testFreeformFailureMessagesAgainstRealWorkbench() { const store = new WorkbenchDeepScanStore(createWorkbenchRunner(environment)); try { const run = await store.begin({ + executionSettings: await captureDeepScanExecutionSettings( + {}, + { filesystemDenies: [] }, + environment, + ), targetPath, threadId: "error-owner", scanRoot: path.join(fixtureRoot, "scans"), @@ -327,6 +684,11 @@ async function testRecoveredPublicationRejectsLateFailure() { try { const store = new WorkbenchDeepScanStore(runWorkbench); const run = await store.begin({ + executionSettings: await captureDeepScanExecutionSettings( + {}, + { filesystemDenies: [] }, + environment, + ), targetPath, scope: ".", threadId: "publication-failure-owner", @@ -396,6 +758,11 @@ async function testNoopStoppedRefreshRetainsPublicationFailure() { try { const store = new WorkbenchDeepScanStore(runWorkbench); const run = await store.begin({ + executionSettings: await captureDeepScanExecutionSettings( + {}, + { filesystemDenies: [] }, + environment, + ), targetPath, scope: ".", threadId: "noop-publication-owner", @@ -547,6 +914,11 @@ async function testConcurrentParentDraftsPreserveBothCheckpoints() { }; try { const run = await new WorkbenchDeepScanStore(rawRunWorkbench).begin({ + executionSettings: await captureDeepScanExecutionSettings( + {}, + { filesystemDenies: [] }, + environment, + ), targetPath, scope: ".", threadId: "concurrent-draft-owner", @@ -604,6 +976,11 @@ async function testLateParentDraftPreservesCheckpointWithoutOverwritingTerminalS const runWorkbench = createWorkbenchRunner(environment); try { const run = await new WorkbenchDeepScanStore(runWorkbench).begin({ + executionSettings: await captureDeepScanExecutionSettings( + {}, + { filesystemDenies: [] }, + environment, + ), targetPath, scope: ".", threadId: "checkpoint-owner", @@ -675,7 +1052,25 @@ async function testReducerCommitAndFinishAgainstRealWorkbench( const codexHome = path.join(fixtureRoot, "codex-home"); const threadId = "deep-scan-store-integration-thread"; const python = process.env.PYTHON?.trim() || "python3"; - const runWorkbench = mock.fn(createWorkbenchRunner(environment)); + const finishCalls: string[][] = []; + const executeWorkbench = createWorkbenchRunner(environment); + const runWorkbench = mock.fn( + async ( + args: string[], + ...rest: [string?, boolean?, boolean?, AbortSignal?, boolean?] + ) => { + const result = await executeWorkbench(args, ...rest); + if (args[0] === "finish-deep-scan") { + finishCalls.push([...args]); + if (finishCalls.length === 1) + throw Object.assign( + new Error("Synthetic lost committed finish response"), + { code: "ETIMEDOUT" }, + ); + } + return result; + }, + ); const store = new WorkbenchDeepScanStore(runWorkbench); try { @@ -692,6 +1087,11 @@ max_time_hours = 2.5 ); const run = await store.begin({ + executionSettings: await captureDeepScanExecutionSettings( + {}, + { filesystemDenies: [] }, + environment, + ), targetPath, scope: ".", threadId, @@ -706,6 +1106,11 @@ max_time_hours = 2.5 assert.equal(owned.run.coordinatorGeneration, 2); const observer = new WorkbenchDeepScanStore(runWorkbench); const joined = await observer.begin({ + executionSettings: await captureDeepScanExecutionSettings( + {}, + { filesystemDenies: [] }, + environment, + ), scanId: run.scanId, threadId, scanRoot, @@ -899,10 +1304,17 @@ connection.rollback()`, omittedWorkerIds: [late.id], }); assert.equal(finished.status, "succeeded"); + assert.equal(finishCalls.length, 2); + assert.deepEqual(finishCalls[1], finishCalls[0]); assert.equal(finished.terminalReason, "saturated"); assert.equal(finished.manifestPath, manifestPath); const continued = await store.begin({ + executionSettings: await captureDeepScanExecutionSettings( + {}, + { filesystemDenies: [] }, + environment, + ), targetPath, scope: ".", threadId: "deep-scan-store-continuation-thread", @@ -960,6 +1372,11 @@ async function testExpiredDeadlineWithoutCompletedDiscoveryAgainstRealWorkbench( ); const run = await store.begin({ + executionSettings: await captureDeepScanExecutionSettings( + {}, + { filesystemDenies: [] }, + environment, + ), targetPath, scope: ".", threadId, @@ -1041,6 +1458,11 @@ async function testPartialWorkerCoverageSurvivesParentCompletion() { "[deep_scan]\nworkers = 2\nmax_discovery_runs = 2\nstop_after_no_new = 2\n", ); const run = await store.begin({ + executionSettings: await captureDeepScanExecutionSettings( + {}, + { filesystemDenies: [] }, + environment, + ), targetPath, threadId, scanRoot: path.join(fixtureRoot, "scans"), @@ -1171,6 +1593,11 @@ async function testDeadlinePreservesValidWorkerCheckpoints() { "[deep_scan]\nworkers = 2\nmax_discovery_runs = 6\nmax_time_hours = 1e-12\n", ); const run = await store.begin({ + executionSettings: await captureDeepScanExecutionSettings( + {}, + { filesystemDenies: [] }, + environment, + ), targetPath, scope: ".", threadId, @@ -1441,6 +1868,11 @@ async function testReceiptRepairSurvivesStoppedRecovery( "[deep_scan]\nworkers = 1\nmax_discovery_runs = 1\n", ); const run = await store.begin({ + executionSettings: await captureDeepScanExecutionSettings( + {}, + { filesystemDenies: [] }, + environment, + ), targetPath, threadId, scanRoot: path.join(fixtureRoot, "scans"), @@ -1675,12 +2107,19 @@ async function testReceiptRepairSurvivesStoppedRecovery( id: string; disposition: string; receiptRefs: string[]; + provenance?: unknown; }[]; assert.equal(surfaces.length, crossSurface ? 2 : 1); assert.deepEqual( surfaces.map((row) => row.id).sort(), before.surfaces.map((row: { id: string }) => row.id).sort(), ); + for (const surface of surfaces) { + const original = before.surfaces.find( + (row: { id: string }) => row.id === surface.id, + ); + assert.deepEqual(surface.provenance, original.provenance); + } assert.ok(surfaces.every((row) => row.disposition === "no_issue_found")); assert.equal( recovered.deferred.some((row: { id: string }) => @@ -1758,6 +2197,11 @@ async function testReplacementFailureDuringCancellation() { const threadId = "cancel-replacement-owner"; const store = new WorkbenchDeepScanStore(runWorkbench); const run = await store.begin({ + executionSettings: await captureDeepScanExecutionSettings( + {}, + { filesystemDenies: [] }, + environment, + ), targetPath, scope: ".", threadId, diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_terminal_race.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_terminal_race.ts index 2db12a7660..8ffa1eb152 100644 --- a/plugins/codex-security/mcp-app/tests/test_deep_scan_terminal_race.ts +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_terminal_race.ts @@ -97,22 +97,42 @@ for (const ordering of [ : "max_time_hours = 1e-12\n"), { mode: 0o600 }, ); - const runWorkbench = async (args: string[]) => { - const { stdout } = await execFileAsync( + const runWorkbench = async ( + args: string[], + input?: string, + selectFinalization = false, + ) => { + const script = join(pluginRoot, "scripts", "workbench_db.py"); + const pythonArgs = selectFinalization + ? [ + "-c", + "import runpy, sys; script = sys.argv.pop(1); runpy.run_path(script)['main'](select_finalization=True)", + script, + ...args, + ] + : [script, ...args]; + const execution = execFileAsync( process.env.PYTHON || "python3", - [join(pluginRoot, "scripts", "workbench_db.py"), ...args], + pythonArgs, { cwd: pluginRoot, env: environment }, ); + if (input !== undefined) execution.child.stdin?.end(input); + const { stdout } = await execution; return JSON.parse(stdout); }; - const workbench = async (args: string[]) => { + const workbench = async ( + args: string[], + input?: string, + selectFinalization = false, + ) => { if (args[0] === "cancel-scan" && !canceledBeforeFailure) { cancelEntered.resolve(); await cancelRelease.promise; } - const result = await runWorkbench(args); + const result = await runWorkbench(args, input, selectFinalization); if ( args[0] === "finish-deep-scan" && + !selectFinalization && canceledBeforeAdmission && !activeBeforeAdmission ) { @@ -156,6 +176,9 @@ for (const ordering of [ ), }, loader: { ".md": "text" }, + banner: { + js: `import { createRequire as fixtureCreateRequire } from "node:module"; const require = fixtureCreateRequire(${JSON.stringify(new URL("../server.ts", import.meta.url).href)});`, + }, }); const application = createCodexSecurityServer(); server = application; diff --git a/plugins/codex-security/mcp-app/tests/test_mcp_app_smoke.ts b/plugins/codex-security/mcp-app/tests/test_mcp_app_smoke.ts index 19b7be05d2..6b088dff56 100644 --- a/plugins/codex-security/mcp-app/tests/test_mcp_app_smoke.ts +++ b/plugins/codex-security/mcp-app/tests/test_mcp_app_smoke.ts @@ -190,12 +190,22 @@ function startTestServer({ cwd: string; env?: NodeJS.ProcessEnv; }) { + const childEnvironment: NodeJS.ProcessEnv = { + ...process.env, + CODEX_HOME: path.join(scanRoot, "codex-home"), + ...env, + }; + for (const [name, value] of Object.entries(env)) { + if (value === undefined) { + delete childEnvironment[name]; + } + } return startRpcServer( { command, args, cwd, - env: { ...process.env, ...env }, + env: childEnvironment, stderr: "inherit", }, { terminateOnStop: true }, @@ -757,6 +767,8 @@ async function assertDeepScanPersistsRetryableWorkerStartupError() { await writeFile(path.join(fixtureTarget, "app", "routes.py"), "route = 1\n"); const fixtureEnvironment = { + // Keep the missing-executable fixture independent of ambient session history. + CODEX_HOME: path.join(fixtureRoot, "codex-home"), CODEX_CLI_PATH: path.join(fixtureRoot, "missing-deep-scan-codex"), CODEX_SECURITY_SCAN_ROOT: fixtureScanRoot, CODEX_SECURITY_STATE_DIR: fixtureState, diff --git a/plugins/codex-security/mcp-app/tests/test_workbench_database.ts b/plugins/codex-security/mcp-app/tests/test_workbench_database.ts index 976974ed07..1401dcbf16 100644 --- a/plugins/codex-security/mcp-app/tests/test_workbench_database.ts +++ b/plugins/codex-security/mcp-app/tests/test_workbench_database.ts @@ -986,3 +986,75 @@ test("retries an upgrade when another process holds the write lock beyond the bu await writer.terminate(); } }); + +test("original Deep history preserves saved inputs and applies local embedding migrations", (t) => { + const originalVersions = new Map([ + ["preserve original deep scan discovery context", 44], + ["retain deep scan attempts and exact merge inputs", 45], + ["persist selected deep scan finalization input", 46], + ["freeze stopped scan checkpoint selections", 47], + ["bind original deep scan parent usage turn", 48], + ["bind original deep scan execution settings", 51], + ]); + const database = memory(t); + database.exec(`CREATE TABLE schema_migrations ( + version INTEGER PRIMARY KEY, name TEXT NOT NULL, applied_at TEXT NOT NULL)`); + for (const item of migrations.filter( + (entry) => entry.version <= 43 || originalVersions.has(entry.name), + )) { + for (const statement of item.statements) database.exec(statement); + database + .prepare("INSERT INTO schema_migrations VALUES (?, ?, 'original')") + .run(originalVersions.get(item.name) ?? item.version, item.name); + } + insertScan(database); + database.exec(`INSERT INTO deep_scan_runs (scan_id, schema_version, workflow_version, + status, phase, workers, subagents, stop_after_no_new, max_discovery_runs, + discovery_user_context, finalization_input_json, usage_owner_json, + execution_settings_json, created_at, updated_at) + VALUES ('scan', 1, 'deep-scan-mcp/v1', 'running', 'discovery', 2, 0, 2, 10, + 'original context', '{"result":"original"}', '{"turn":"original"}', + '{"model":"synthetic"}', 'created', 'updated')`); + const original = database.prepare("SELECT * FROM deep_scan_runs").get(); + + applyMigrations(database); + + assert.deepEqual( + { ...database.prepare("SELECT * FROM deep_scan_runs").get() }, + { ...original, discovery_user_context_json: null }, + ); + assert.deepEqual( + database.prepare("SELECT * FROM local_finding_embeddings").all(), + [], + ); + assert.ok( + database + .prepare("PRAGMA table_info(finding_embeddings)") + .all() + .some((row) => row.name === "cache_key"), + ); + assert.ok( + database + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'trigger' AND name = 'invalidate_local_finding_embedding'", + ) + .get(), + ); + const recorded = database + .prepare("SELECT * FROM schema_migrations ORDER BY version") + .all(); + assert.deepEqual( + recorded.map((row) => [row.version, row.name]), + migrations.map((item) => [item.version, item.name]), + ); + assert.ok( + recorded + .filter((row) => originalVersions.has(String(row.name))) + .every((row) => row.applied_at === "original"), + ); + applyMigrations(database); + assert.deepEqual( + database.prepare("SELECT * FROM schema_migrations ORDER BY version").all(), + recorded, + ); +}); diff --git a/plugins/codex-security/mcp-app/tsconfig.json b/plugins/codex-security/mcp-app/tsconfig.json index 1a51147297..453480fefe 100644 --- a/plugins/codex-security/mcp-app/tsconfig.json +++ b/plugins/codex-security/mcp-app/tsconfig.json @@ -3,6 +3,12 @@ "module": "ESNext", "moduleResolution": "Bundler", "noEmit": true, + "paths": { + "@openai/codex-sdk": ["./node_modules/@openai/codex-sdk"], + "smol-toml": ["./node_modules/smol-toml/dist/index"], + "yauzl": ["./node_modules/@types/yauzl", "./node_modules/yauzl"], + "zod": ["./node_modules/zod"] + }, "resolveJsonModule": true, "skipLibCheck": true, "strict": true, diff --git a/plugins/codex-security/plugin-files.json b/plugins/codex-security/plugin-files.json index 9cbe2c5bca..c44855229a 100644 --- a/plugins/codex-security/plugin-files.json +++ b/plugins/codex-security/plugin-files.json @@ -1,8 +1,5 @@ { - "externalOwnedExact": [ - ".codex-plugin/plugin.json", - "OWNERS" - ], + "externalOwnedExact": [".codex-plugin/plugin.json", "OWNERS"], "shippedExact": [ ".app.json", ".mcp.json", @@ -27,10 +24,12 @@ "mcp/native/win32-x64/windows.node", "mcp/helpers.mjs", "mcp/helpers.mjs.br.part-000", + "mcp/helpers.mjs.br.part-001", "mcp/permission-profile-preflight.mjs", "mcp/server.mjs", "mcp/server.mjs.br.part-000", "mcp/server.mjs.br.part-001", + "mcp/server.mjs.br.part-002", "references/artifact-storage.md", "references/core-scan.md", "references/final-report.md", @@ -81,6 +80,7 @@ "scripts/workbench/handoff.py", "scripts/workbench/runtime_toml.py", "scripts/workbench/saved_coverage.py", + "scripts/workbench/saved_sources.py", "scripts/workbench/storage.py", "scripts/workbench_cli.py", "scripts/workbench_constants.py", @@ -98,6 +98,7 @@ "scripts/workbench_scan_start.py", "scripts/workbench_scan_usage.py", "scripts/workbench_schema.py", + "scripts/workbench_source_coverage.py", "scripts/workbench_source_excerpt.py", "scripts/workbench_source_scopes.py", "scripts/workbench_target.py", diff --git a/plugins/codex-security/scripts/codex_profile.mjs b/plugins/codex-security/scripts/codex_profile.mjs index 8ad053f1c8..41bf24795c 100644 --- a/plugins/codex-security/scripts/codex_profile.mjs +++ b/plugins/codex-security/scripts/codex_profile.mjs @@ -127,8 +127,9 @@ class ProfileThread { ); } if (event.type === "thread.started") this.id = event.thread_id; - if (event.type === "turn.completed" && event.usage != null) - event.usage.cache_write_input_tokens ??= 0; + if (event.type === "turn.completed") { + if (event.usage != null) event.usage.cache_write_input_tokens ??= 0; + } yield event; } } finally { @@ -235,5 +236,8 @@ async function* execute(options, args, input, signal) { } finally { lines.close(); if (child.exitCode === null && child.signalCode === null) child.kill(); + child.stdout.destroy(); + child.stderr.destroy(); + await closed; } } diff --git a/plugins/codex-security/scripts/deep_scan_workbench.py b/plugins/codex-security/scripts/deep_scan_workbench.py index 7d12f65761..59662f964c 100644 --- a/plugins/codex-security/scripts/deep_scan_workbench.py +++ b/plugins/codex-security/scripts/deep_scan_workbench.py @@ -18,11 +18,31 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) import workbench_saved_results as saved_results from deep_scan_config import resolve_deep_scan_config +from finalize_scan_contract import ( + PreparedScanFinalization, + _read_scan_local_json, + open_scan_local_file_descriptor, + write_scan_local_bytes, +) from workbench.handoff import require_current_continuation from workbench.storage import create_private_directory +from workbench_saved_results import ( + _restore_published_outputs, + _selected_publication_digest, + _snapshot_published_outputs, + _worker_checkpoint_head, +) +from workbench_scan_usage import capture_scan_usage_owner from workbench_source_scopes import capture_source_scopes -from workbench_target import directory_snapshot_regular_file_count -from workbench_validation import optional_text, require_uuid, user_context_argument +from workbench_target import ( + directory_snapshot_regular_file_count, +) +from workbench_validation import ( + optional_text, + reject_non_finite_json, + require_uuid, + user_context_argument, +) DEEP_SCAN_WORKER_KINDS = ("setup", "discovery", "dedup") DEEP_SCAN_WORKER_STATUSES = ("queued", "running", "succeeded", "failed", "canceled") @@ -32,7 +52,12 @@ "invalid_discovery_artifacts", ) DEEP_SCAN_TERMINAL_REASONS = ("saturated", "capped") -DEEP_SCAN_WORKFLOW_VERSION = "deep-security-scan/v1" +DEEP_SCAN_WORKFLOW_VERSION = "deep-security-scan/v2" +SUPPORTED_DEEP_SCAN_WORKFLOWS = { + DEEP_SCAN_WORKFLOW_VERSION, + "deep-scan-mcp/v1", + "deep-security-scan/v1", +} DEEP_SCAN_COORDINATOR_LEASE_SECONDS = 30 DEEP_SCAN_LEGACY_COORDINATOR_GRACE_SECONDS = 120 DEEP_SCAN_MAX_ERROR_LENGTH = 2400 @@ -189,26 +214,58 @@ def require_deep_scan_run(connection: sqlite3.Connection, scan_id: str) -> sqlit return row +def require_supported_deep_scan(run: sqlite3.Row) -> None: + if run["schema_version"] != 1 or run["workflow_version"] not in SUPPORTED_DEEP_SCAN_WORKFLOWS: + raise SystemExit( + "This Deep Scan uses an unsupported workflow or schema version. " + "Resume it with a compatible Codex Security release." + ) + + finalization = deep_scan_finalization_input(run) + if finalization is not None and ( + run["workflow_version"] != "deep-security-scan/v2" + or not isinstance(finalization, dict) + or finalization.get("version") != 1 + ): + raise SystemExit("This Deep Scan uses an unsupported finalization input version.") + + +def deep_scan_finalization_input(run: sqlite3.Row) -> dict[str, Any] | None: + if "finalization_input_json" not in run.keys() or run["finalization_input_json"] is None: + return None + return json.loads(run["finalization_input_json"]) + + def deep_scan_deadline_reached(run: sqlite3.Row) -> bool: elapsed = _parse_timestamp(dependencies().now()) - _parse_timestamp(str(run["created_at"])) return elapsed.total_seconds() / 3600 >= run["max_time_hours"] +def find_supported_deep_scan_run( + connection: sqlite3.Connection, scan_id: str +) -> sqlite3.Row | None: + run = connection.execute( + "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan_id,) + ).fetchone() + if run is not None: + require_supported_deep_scan(run) + return run + + def require_deep_scan_ready_for_parent_completion( connection: sqlite3.Connection, scan: sqlite3.Row ) -> None: if scan["mode"] != "deep": return - run = connection.execute( - "SELECT status, manifest_path, terminal_reason FROM deep_scan_runs WHERE scan_id = ?", - (scan["id"],), - ).fetchone() + run = find_supported_deep_scan_run(connection, scan["id"]) if run is None or run["status"] != "succeeded" or run["manifest_path"] is None: raise SystemExit( "Deep Scan discovery orchestration must finish and persist its manifest before " "the parent scan can be completed." ) - if run["terminal_reason"] == "capped": + # Cost-limit publication freezes its unmerged accepted reviews as evidence. + budget_selection = run["cancel_requested"] and deep_scan_finalization_input(run) is not None + if run["terminal_reason"] == "capped" and not budget_selection: require_deep_scan_workers_finished(connection, scan["id"], "capped") @@ -408,20 +465,35 @@ def canonical_discovery_artifacts(scan: sqlite3.Row) -> dict[str, str]: def deep_scan_state(connection: sqlite3.Connection, scan_id: str) -> dict[str, Any]: + if connection.in_transaction: + return _deep_scan_state(connection, scan_id) + connection.execute("BEGIN") + try: + state = _deep_scan_state(connection, scan_id) + connection.commit() + return state + except BaseException: + connection.rollback() + raise + + +def _deep_scan_state(connection: sqlite3.Connection, scan_id: str) -> dict[str, Any]: run = require_deep_scan_run(connection, scan_id) scan = dependencies().require_scan(connection, run["scan_id"]) worker_rows = connection.execute( """ - SELECT * - FROM deep_scan_workers - WHERE scan_id = ? - ORDER BY created_at, id + SELECT workers.*, attempts.accepted_result_path + FROM deep_scan_workers AS workers + LEFT JOIN deep_scan_attempts AS attempts + ON attempts.worker_id = workers.id AND attempts.attempt = workers.attempt + WHERE workers.scan_id = ? + ORDER BY workers.created_at, workers.id """, (run["scan_id"],), ) input_rows = connection.execute( """ - SELECT dedup_worker_id, discovery_worker_id, input_order + SELECT * FROM deep_scan_dedup_inputs WHERE scan_id = ? ORDER BY dedup_worker_id, input_order @@ -449,18 +521,30 @@ def deep_scan_state(connection: sqlite3.Connection, scan_id: str) -> dict[str, A ) return { "scanId": run["scan_id"], + "ownerThreadId": scan["deep_scan_owner_thread_id"] + or dependencies().require_workspace(connection, scan["workspace_id"])["thread_id"], "targetPath": scan["target_path"], "scope": scan["scope"], + "model": scan["model"], + "reasoningEffort": scan["reasoning_effort"], "includePaths": dependencies().requested_scan_paths(scan), "filePaths": dependencies().requested_scan_file_paths(scan), "userContext": ( json.loads(run["discovery_user_context_json"]) if run["discovery_user_context_json"] is not None + else run["discovery_user_context"] + if "discovery_user_context" in run.keys() else scan["user_context"] ), "scanDir": scan["scan_dir"], "schemaVersion": run["schema_version"], "workflowVersion": run["workflow_version"], + "finalizationInput": deep_scan_finalization_input(run), + "usageOwner": ( + json.loads(run["usage_owner_json"]) + if "usage_owner_json" in run.keys() and run["usage_owner_json"] + else None + ), "coordinatorGeneration": run["coordinator_generation"], "status": run["status"], "phase": run["phase"], @@ -485,11 +569,55 @@ def deep_scan_state(connection: sqlite3.Connection, scan_id: str) -> dict[str, A "updatedAt": run["updated_at"], "completedAt": run["completed_at"], "workers": [deep_scan_worker_state(row) for row in worker_rows], + "attempts": [ + { + "workerId": row["worker_id"], + "attempt": row["attempt"], + "status": row["status"], + "startedAt": row["started_at"], + "completedAt": row["completed_at"], + "endReason": row["end_reason"], + "error": row["error_message"], + "acceptedResultPath": row["accepted_result_path"], + "acceptedResultSha256": row["accepted_result_sha256"], + } + for row in connection.execute( + "SELECT * FROM deep_scan_attempts WHERE scan_id = ? ORDER BY worker_id, attempt", + (scan_id,), + ) + ], + "attemptSessions": [ + { + "workerId": row["worker_id"], + "attempt": row["attempt"], + "sdkThreadId": row["sdk_thread_id"], + "observedAt": row["observed_at"], + } + for row in connection.execute( + "SELECT * FROM deep_scan_attempt_sessions WHERE scan_id = ? " + "ORDER BY observed_at, worker_id, attempt, sdk_thread_id", + (scan_id,), + ) + ], + "mergeClaims": [ + { + "workerId": row["worker_id"], + "previousWorkerId": row["previous_worker_id"], + "previousResultPath": row["previous_result_path"], + "previousResultSha256": row["previous_result_sha256"], + } + for row in connection.execute( + "SELECT * FROM deep_scan_merge_claims WHERE scan_id = ? ORDER BY rowid", (scan_id,) + ) + ], "dedupInputs": [ { "dedupWorkerId": row["dedup_worker_id"], "discoveryWorkerId": row["discovery_worker_id"], "inputOrder": row["input_order"], + "resultManifestPath": row["result_manifest_path"], + "resultManifestSha256": row["result_manifest_sha256"], + "attempt": row["attempt"], } for row in input_rows ], @@ -537,6 +665,9 @@ def deep_scan_worker_state(row: sqlite3.Row) -> dict[str, Any]: "promptPath": row["prompt_path"], "artifactDir": row["artifact_dir"], "resultManifestPath": row["result_manifest_path"], + "acceptedResultPath": row["accepted_result_path"] + if "accepted_result_path" in row.keys() + else None, "attempt": row["attempt"], "sdkThreadId": row["sdk_thread_id"], "completionSequence": row["completion_sequence"], @@ -565,30 +696,73 @@ def effective_deep_scan_config(args: argparse.Namespace) -> dict[str, int | floa return resolve_deep_scan_config(available_parallelism) +def recorded_deep_scan_execution_settings(run: sqlite3.Row) -> dict[str, Any] | None: + saved = run["execution_settings_json"] if "execution_settings_json" in run.keys() else None + return json.loads(saved) if saved else None + + +def include_execution_settings(connection: sqlite3.Connection, result: dict[str, Any]) -> None: + if "deepScan" in result: + run = require_deep_scan_run(connection, result["deepScan"]["scanId"]) + result["deepScan"]["executionSettings"] = recorded_deep_scan_execution_settings(run) + + +def validate_deep_scan_execution_settings(saved: dict[str, Any]) -> dict[str, Any]: + if saved.get("version") != 1: + raise SystemExit("This Deep Scan uses an unsupported execution settings version.") + settings = saved.get("settings") + if not isinstance(settings, dict) or not all( + isinstance(settings.get(key), str) for key in ("codexPath", "codexHome") + ): + raise SystemExit( + "Deep Scan execution settings are missing the recorded executable or Codex home." + ) + return settings + + def ensure_deep_scan_run( connection: sqlite3.Connection, scan: sqlite3.Row, config: dict[str, int | float], workflow_version: str, timestamp: str, + args: argparse.Namespace | None = None, ) -> sqlite3.Row: existing = connection.execute( "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan["id"],) ).fetchone() if existing is not None: + require_supported_deep_scan(existing) return existing if scan["mode"] != "deep": raise SystemExit("Deep Scan orchestration requires a scan in deep mode.") if scan["status"] != "running": raise SystemExit("Only a running Deep Scan can start orchestration.") + saved_settings = None + if args is not None and hasattr(args, "execution_settings"): + if args.execution_settings is None: + raise SystemExit("The original Deep Scan execution settings could not be captured.") + settings = dict(args.execution_settings) + for key, column in (("model", "model"), ("reasoningEffort", "reasoning_effort")): + if scan[column] is not None: + settings[key] = scan[column] + saved_settings = {"version": 1, "settings": settings} + validate_deep_scan_execution_settings(saved_settings) + # The workbench transaction binds original settings. This artifact is a + # projection and cannot choose a fresh or resumed executable or home. + write_scan_local_bytes( + Path(scan["scan_dir"]), + "artifacts/deep_discovery/execution-settings.json", + (json.dumps(saved_settings, indent=2) + "\n").encode(), + ) connection.execute( """ INSERT INTO deep_scan_runs ( scan_id, schema_version, workflow_version, status, phase, workers, subagents, stop_after_no_new, stop_after_consecutive_errors, - max_discovery_runs, max_time_hours, discovery_user_context_json, - created_at, updated_at - ) VALUES (?, 1, ?, 'running', 'setup', ?, ?, ?, ?, ?, ?, ?, ?, ?) + max_discovery_runs, max_time_hours, discovery_user_context, discovery_user_context_json, + created_at, updated_at, execution_settings_json + ) VALUES (?, 1, ?, 'running', 'setup', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( scan["id"], @@ -599,12 +773,21 @@ def ensure_deep_scan_run( config["stopAfterConsecutiveErrors"], config["maxDiscoveryRuns"], config["maxTimeHours"], + scan["user_context"], json.dumps(scan["user_context"]), timestamp, timestamp, + json.dumps(saved_settings) if saved_settings is not None else None, ), ) - return require_deep_scan_run(connection, scan["id"]) + run = require_deep_scan_run(connection, scan["id"]) + if "usage_owner_json" in run.keys(): + connection.execute( + "UPDATE deep_scan_runs SET usage_owner_json = ? WHERE scan_id = ?", + (json.dumps(capture_scan_usage_owner(connection, scan)), scan["id"]), + ) + run = require_deep_scan_run(connection, scan["id"]) + return run def existing_deep_scan_for_target( @@ -696,6 +879,11 @@ def begin_deep_scan_for_scan( ) -> dict[str, Any]: scan_id = require_uuid(scan_id, "scan-id") candidate = dependencies().require_scan(connection, scan_id) + existing = connection.execute( + "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan_id,) + ).fetchone() + if existing is not None: + require_supported_deep_scan(existing) workspace = dependencies().require_workspace(connection, candidate["workspace_id"]) if ( candidate["mode"] == "deep" @@ -733,23 +921,10 @@ def begin_deep_scan_for_scan( ) if scan["mode"] != "deep": raise SystemExit("Deep Scan orchestration requires a scan in deep mode.") - model = optional_text(args.model, maximum=200) - reasoning_effort = optional_text(args.reasoning_effort, maximum=32) - if model is not None or reasoning_effort is not None: - connection.execute( - """ - UPDATE scans - SET model = COALESCE(?, model), reasoning_effort = COALESCE(?, reasoning_effort) - WHERE id = ? - """, - (model, reasoning_effort, scan_id), - ) - connection.commit() - existing = connection.execute( - "SELECT scan_id FROM deep_scan_runs WHERE scan_id = ?", (scan_id,) - ).fetchone() if existing is not None: return deep_scan_result(connection, scan_id, start_disposition="joined") + model = optional_text(args.model, maximum=200) + reasoning_effort = optional_text(args.reasoning_effort, maximum=32) config = effective_deep_scan_config(args) workflow_version = optional_text(args.workflow_version, maximum=256) if workflow_version is None: @@ -762,7 +937,25 @@ def begin_deep_scan_for_scan( args.claim_token, error_message="Deep Scan orchestration is owned by another continuation.", ) - ensure_deep_scan_run(connection, scan, config, workflow_version, dependencies().now()) + existing = connection.execute( + "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan_id,) + ).fetchone() + if existing is not None: + require_supported_deep_scan(existing) + connection.commit() + return deep_scan_result(connection, scan_id, start_disposition="joined") + if model is not None or reasoning_effort is not None: + connection.execute( + """ + UPDATE scans + SET model = COALESCE(?, model), reasoning_effort = COALESCE(?, reasoning_effort) + WHERE id = ? + """, + (model, reasoning_effort, scan_id), + ) + scan = dependencies().require_scan(connection, scan_id) + ensure_deep_scan_run(connection, scan, config, workflow_version, dependencies().now(), args) + connection.commit() return deep_scan_result(connection, scan_id, start_disposition="created") @@ -788,15 +981,17 @@ def begin_deep_scan_for_target( existing = existing_deep_scan_for_target(connection, thread_id, target_path, scope) if existing is not None: existing_run = connection.execute( - "SELECT 1 FROM deep_scan_runs WHERE scan_id = ?", (existing["id"],) + "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (existing["id"],) ).fetchone() + if existing_run is not None: + require_supported_deep_scan(existing_run) if existing_run is None: config = effective_deep_scan_config(args) workflow_version = optional_text(args.workflow_version, maximum=256) if workflow_version is None: raise SystemExit("workflow-version is required.") ensure_deep_scan_run( - connection, existing, config, workflow_version, dependencies().now() + connection, existing, config, workflow_version, dependencies().now(), args ) connection.commit() return deep_scan_result( @@ -891,11 +1086,15 @@ def begin_deep_scan_for_target( reasoning_effort=reasoning_effort, ) scan = dependencies().require_scan(connection, scan_id) - ensure_deep_scan_run(connection, scan, config, workflow_version, timestamp) + ensure_deep_scan_run(connection, scan, config, workflow_version, timestamp, args) + connection.commit() return deep_scan_result(connection, scan_id, start_disposition="created") def begin_deep_scan(connection: sqlite3.Connection, args: argparse.Namespace) -> dict[str, Any]: + workflow_version = optional_text(args.workflow_version, maximum=256) + if workflow_version not in SUPPORTED_DEEP_SCAN_WORKFLOWS: + raise SystemExit("This Deep Scan uses an unsupported workflow version.") thread_id = optional_text(args.thread_id, maximum=512) if thread_id is None: raise SystemExit("thread-id is required.") @@ -952,6 +1151,7 @@ def coordinator_lease_is_live( def require_current_coordinator(run: sqlite3.Row, args: argparse.Namespace) -> None: + require_supported_deep_scan(run) generation = getattr(args, "coordinator_generation", None) if run["coordinator_generation"] == 1: if generation is not None: @@ -982,7 +1182,25 @@ def claim_deep_scan_coordinator_locked( args.claim_token, error_message="Deep Scan orchestration is owned by another continuation.", ) - run, _ = require_running_deep_scan(connection, scan_id) + run = require_deep_scan_run(connection, scan_id) + require_supported_deep_scan(run) + selected_parent = ( + run["status"] == "succeeded" and deep_scan_finalization_input(run) is not None + ) + if selected_parent and scan["status"] == "complete": + connection.commit() + return { + **deep_scan_result(connection, scan_id), + "coordinatorDisposition": "observing", + } + if not selected_parent: + run, _ = require_running_deep_scan(connection, scan_id) + elif ( + run["cancel_requested"] + or scan["status"] != "running" + or scan["canceled_at"] is not None + ): + raise SystemExit("Only a running scan can complete its selected Deep Scan result.") timestamp = dependencies().now() if args.coordinator_generation is not None: require_current_coordinator(run, args) @@ -995,15 +1213,27 @@ def claim_deep_scan_coordinator_locked( } else: adopted = run["coordinator_generation"] > 1 or run["phase"] != "setup" - if adopted: - recover_expired_coordinator(connection, run, timestamp) disposition = "adopted" if adopted else "claimed" + if ( + getattr(args, "require_execution_settings", False) + and deep_scan_finalization_input(run) is None + ): + saved = recorded_deep_scan_execution_settings(run) + if saved is not None: + validate_deep_scan_execution_settings(saved) + elif run["workflow_version"] == DEEP_SCAN_WORKFLOW_VERSION: + raise SystemExit( + "This Deep Scan has no recorded original execution settings; " + "its executable and Codex home cannot be recovered." + ) + if disposition == "adopted": + recover_expired_coordinator(connection, run, timestamp) connection.execute( """ UPDATE deep_scan_runs SET coordinator_generation = coordinator_generation + ?, updated_at = ? - WHERE scan_id = ? AND status = 'running' + WHERE scan_id = ? AND status IN ('running', 'succeeded') """, (int(args.coordinator_generation != run["coordinator_generation"]), timestamp, scan_id), ) @@ -1013,9 +1243,52 @@ def claim_deep_scan_coordinator_locked( } +def release_deep_scan_coordinator( + connection: sqlite3.Connection, args: argparse.Namespace +) -> dict[str, Any]: + scan_id = require_uuid(args.scan_id, "scan-id") + with dependencies().scan_completion_lock(scan_id): + connection.execute("BEGIN IMMEDIATE") + with connection: + scan, _ = require_owned_scan(connection, scan_id, args.thread_id) + require_current_continuation( + scan, + args.claim_token, + error_message="Deep Scan orchestration is owned by another continuation.", + ) + run = require_deep_scan_run(connection, scan_id) + require_current_coordinator(run, args) + if scan["status"] != "running" or run["status"] not in {"running", "succeeded"}: + return deep_scan_result(connection, scan_id) + expired_at = ( + _parse_timestamp(dependencies().now()) + - timedelta(seconds=DEEP_SCAN_COORDINATOR_LEASE_SECONDS) + ).isoformat() + # The private host invocation retires its lease in the database; + # model-writable heartbeat fields cannot release a live owner. + write_scan_local_bytes( + Path(scan["scan_dir"]), + f"artifacts/deep_discovery/coordinator-heartbeat-{run['coordinator_generation']}.json", + json.dumps( + { + "coordinatorGeneration": run["coordinator_generation"], + "updatedAt": expired_at, + } + ).encode(), + ) + connection.execute( + "UPDATE deep_scan_runs SET updated_at = ? WHERE scan_id = ?", + (expired_at, scan_id), + ) + return deep_scan_result(connection, scan_id) + + def recover_expired_coordinator( connection: sqlite3.Connection, run: sqlite3.Row, timestamp: str ) -> None: + require_supported_deep_scan(run) + if deep_scan_finalization_input(run) is not None: + return scan_id = run["scan_id"] recover_candidate_ledger_publication(connection, scan_id) legacy_generation = int(run["coordinator_generation"] == 1) @@ -1142,6 +1415,7 @@ def require_running_deep_scan( connection: sqlite3.Connection, scan_id: str ) -> tuple[sqlite3.Row, sqlite3.Row]: run = require_deep_scan_run(connection, scan_id) + require_supported_deep_scan(run) scan = dependencies().require_scan(connection, run["scan_id"]) if run["status"] != "running" or run["cancel_requested"]: raise SystemExit("Only a running Deep Scan can update orchestration state.") @@ -1162,6 +1436,144 @@ def require_worker_transition(current: str, requested: str) -> None: raise SystemExit(f"Deep Scan worker cannot transition from {current} to {requested}.") +def snapshot_accepted_result(scan: sqlite3.Row, worker: sqlite3.Row) -> tuple[str, str]: + source = deep_scan_path( + scan, worker["result_manifest_path"], "Accepted worker result", kind="file" + ) + scan_dir = Path(scan["scan_dir"]) + with os.fdopen( + open_scan_local_file_descriptor( + scan_dir, Path(source).relative_to(scan_dir).as_posix(), "Accepted worker result" + ), + "rb", + ) as handle: + contents = handle.read() + semantic = json.loads(contents) + if isinstance(semantic, dict): + semantic.pop("handoffClaimToken", None) + directory = Path(worker["artifact_dir"]) / "checkpoints" + head = ( + _worker_checkpoint_head( + scan_dir, Path(worker["artifact_dir"]).relative_to(scan_dir).as_posix(), scan["id"] + ) + if worker["kind"] == "discovery" + else None + ) + candidates = [scan_dir / head] if head else sorted(directory.glob("*.json")) + for checkpoint in candidates: + safe = deep_scan_path(scan, str(checkpoint), "Accepted worker checkpoint", kind="file") + with os.fdopen( + open_scan_local_file_descriptor( + scan_dir, Path(safe).relative_to(scan_dir).as_posix(), "Accepted worker checkpoint" + ), + "rb", + ) as handle: + checkpoint_bytes = handle.read() + if json.loads(checkpoint_bytes) == semantic: + return safe, hashlib.sha256(checkpoint_bytes).hexdigest() + if head: + raise SystemExit( + "The accepted worker result does not match its current checkpoint head." + ) + # Legacy/direct file producers may have no checkpoint. Use the existing native + # checkpoint store; typed artifact writers already supplied the matching copy. + digest = hashlib.sha256(contents).hexdigest() + destination = directory / f"{digest}.json" + if destination.exists(): + raise SystemExit("An existing worker checkpoint does not match its accepted content.") + write_scan_local_bytes(scan_dir, destination.relative_to(scan_dir).as_posix(), contents) + return str(destination), digest + + +def record_worker_attempt( + connection: sqlite3.Connection, + scan: sqlite3.Row, + worker: sqlite3.Row, + timestamp: str, + *, + observed_thread_id: str | None = None, + error: str | None = None, + end_reason: str | None = None, +) -> None: + if worker["status"] == "queued" or worker["attempt"] < 1: + return + connection.execute( + """ + UPDATE deep_scan_attempts + SET status = 'replaced', completed_at = ?, end_reason = 'replacement_attempt' + WHERE worker_id = ? AND attempt < ? AND completed_at IS NULL + """, + (timestamp, worker["id"], worker["attempt"]), + ) + status = worker["status"] + if end_reason in DEEP_SCAN_REPLACEABLE_FAILURE_KINDS: + status = "failed" + if status != "running": + error = worker["error_message"] + if status == "running" and error: + status = "failed" + completed = timestamp if status != "running" else None + reason = end_reason or ( + "execution_or_artifact_error" if status == "failed" else status if completed else None + ) + accepted_path = accepted_sha = None + if status == "succeeded" and worker["result_manifest_path"]: + accepted_path, accepted_sha = snapshot_accepted_result(scan, worker) + connection.execute( + """ + INSERT INTO deep_scan_attempts ( + scan_id, worker_id, attempt, status, started_at, completed_at, + end_reason, error_message, accepted_result_path, accepted_result_sha256 + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(worker_id, attempt) DO UPDATE SET + status = CASE WHEN deep_scan_attempts.completed_at IS NULL THEN excluded.status + ELSE deep_scan_attempts.status END, + completed_at = COALESCE(deep_scan_attempts.completed_at, excluded.completed_at), + end_reason = COALESCE(deep_scan_attempts.end_reason, excluded.end_reason), + error_message = COALESCE(excluded.error_message, deep_scan_attempts.error_message), + accepted_result_path = COALESCE(excluded.accepted_result_path, + deep_scan_attempts.accepted_result_path), + accepted_result_sha256 = COALESCE(excluded.accepted_result_sha256, + deep_scan_attempts.accepted_result_sha256) + """, + ( + scan["id"], + worker["id"], + worker["attempt"], + status, + timestamp, + completed, + reason, + error, + accepted_path, + accepted_sha, + ), + ) + if observed_thread_id: + connection.execute( + """ + INSERT OR IGNORE INTO deep_scan_attempt_sessions ( + scan_id, worker_id, attempt, sdk_thread_id, observed_at + ) VALUES (?, ?, ?, ?, ?) + """, + (scan["id"], worker["id"], worker["attempt"], observed_thread_id, timestamp), + ) + + +def worker_result_reference( + connection: sqlite3.Connection, scan: sqlite3.Row, worker: sqlite3.Row +) -> tuple[str, str]: + accepted = connection.execute( + "SELECT accepted_result_path, accepted_result_sha256 FROM deep_scan_attempts " + "WHERE worker_id = ? AND attempt = ?", + (worker["id"], worker["attempt"]), + ).fetchone() + if accepted is not None and accepted["accepted_result_path"]: + return accepted["accepted_result_path"], accepted["accepted_result_sha256"] + # Old accepted workers have no attempt history; freeze their current accepted result on claim. + return snapshot_accepted_result(scan, worker) + + def upsert_deep_scan_worker( connection: sqlite3.Connection, args: argparse.Namespace ) -> dict[str, Any]: @@ -1205,11 +1617,14 @@ def upsert_deep_scan_worker( scan, args.artifact_dir, "Worker artifact directory", kind="directory" ) result_manifest_path = ( - deep_scan_path( - scan, - args.result_manifest_path, - "Worker result manifest path", - kind="file", + ( + deep_scan_output_path( + scan, args.result_manifest_path, "Worker result manifest path" + ) + if terminal_repeat + else deep_scan_path( + scan, args.result_manifest_path, "Worker result manifest path", kind="file" + ) ) if args.result_manifest_path else None @@ -1268,8 +1683,18 @@ def upsert_deep_scan_worker( timestamp, ), ) + record_worker_attempt( + connection, + scan, + require_deep_scan_worker(connection, worker_id), + timestamp, + observed_thread_id=optional_text(args.sdk_thread_id, maximum=512), + error=optional_text(args.error_message, maximum=2400), + end_reason=args.replaceable_failure_kind, + ) + result = deep_scan_result(connection, scan_id) connection.commit() - return deep_scan_result(connection, scan_id) + return result if existing["scan_id"] != scan_id or existing["kind"] != args.kind: raise SystemExit("Deep Scan worker identity does not match its persisted run and kind.") @@ -1292,8 +1717,15 @@ def upsert_deep_scan_worker( and repeated_error != existing["error_message"] ): raise SystemExit("Deep Scan worker terminal state is immutable.") + receipt = connection.execute( + "SELECT receipt_json FROM deep_scan_attempts WHERE worker_id = ? AND attempt = ?", + (worker_id, existing["attempt"]), + ).fetchone() + result = deep_scan_result(connection, scan_id) + if receipt is not None and receipt["receipt_json"]: + result["deepScan"]["workerReceipt"] = json.loads(receipt["receipt_json"]) connection.commit() - return deep_scan_result(connection, scan_id) + return result attempt = args.attempt if args.attempt is not None else existing["attempt"] if attempt < existing["attempt"]: raise SystemExit("Deep Scan worker attempt cannot decrease.") @@ -1389,7 +1821,27 @@ def upsert_deep_scan_worker( worker_id, ), ) - return deep_scan_result(connection, scan_id) + record_worker_attempt( + connection, + scan, + require_deep_scan_worker(connection, worker_id), + timestamp, + observed_thread_id=optional_text(args.sdk_thread_id, maximum=512), + error=optional_text(args.error_message, maximum=2400), + end_reason=args.replaceable_failure_kind, + ) + result = deep_scan_result(connection, scan_id) + if args.status in {"succeeded", "failed", "canceled"}: + receipt = next( + worker for worker in result["deepScan"]["workers"] if worker["id"] == worker_id + ) + result["deepScan"]["workerReceipt"] = receipt + connection.execute( + "UPDATE deep_scan_attempts SET receipt_json = ? WHERE worker_id = ? AND attempt = ?", + (json.dumps(receipt), worker_id, attempt), + ) + connection.commit() + return result def claim_deep_scan_dedup( @@ -1402,7 +1854,8 @@ def claim_deep_scan_dedup( raise SystemExit("Dedup input worker IDs must be unique.") connection.execute("BEGIN IMMEDIATE") with connection: - run, scan = require_running_deep_scan(connection, scan_id) + run = require_deep_scan_run(connection, scan_id) + scan = dependencies().require_scan(connection, scan_id) require_current_coordinator(run, args) prompt_path = deep_scan_path(scan, args.prompt_path, "Dedup prompt path", kind="file") artifact_dir = deep_scan_path( @@ -1431,9 +1884,11 @@ def claim_deep_scan_dedup( and existing["artifact_dir"] == artifact_dir and persisted_inputs == input_ids ): + result = deep_scan_result(connection, scan_id) connection.commit() - return deep_scan_result(connection, scan_id) + return result raise SystemExit("Dedup worker ID is already used by a different reducer claim.") + require_running_deep_scan(connection, scan_id) active_reducer = connection.execute( """ SELECT 1 FROM deep_scan_workers @@ -1476,15 +1931,8 @@ def claim_deep_scan_dedup( ).fetchone() is None ) - successful_reducer = connection.execute( - """ - SELECT 1 FROM deep_scan_workers - WHERE scan_id = ? AND kind = 'dedup' AND status = 'succeeded' - LIMIT 1 - """, - (scan_id,), - ).fetchone() - minimum_inputs = 1 if successful_reducer is not None or capped_singleton else 2 + previous = final_deep_scan_reducer(connection, scan_id) + minimum_inputs = 1 if previous is not None or capped_singleton else 2 if len(input_ids) < minimum_inputs: raise SystemExit( "The first Deep Scan dedup requires two buffered discovery results." @@ -1501,14 +1949,36 @@ def claim_deep_scan_dedup( """, (worker_id, scan_id, prompt_path, artifact_dir, timestamp, timestamp), ) + previous_path, previous_sha = ( + worker_result_reference(connection, scan, previous) if previous else (None, None) + ) + connection.execute( + """ + INSERT INTO deep_scan_merge_claims ( + worker_id, scan_id, previous_worker_id, previous_result_path, previous_result_sha256 + ) VALUES (?, ?, ?, ?, ?) + """, + (worker_id, scan_id, previous["id"] if previous else None, previous_path, previous_sha), + ) for input_order, input_id in enumerate(input_ids): + discovery = require_deep_scan_worker(connection, input_id) + accepted_path, accepted_sha = worker_result_reference(connection, scan, discovery) connection.execute( """ INSERT INTO deep_scan_dedup_inputs ( - scan_id, dedup_worker_id, discovery_worker_id, input_order - ) VALUES (?, ?, ?, ?) + scan_id, dedup_worker_id, discovery_worker_id, input_order, + result_manifest_path, result_manifest_sha256, attempt + ) VALUES (?, ?, ?, ?, ?, ?, ?) """, - (scan_id, worker_id, input_id, input_order), + ( + scan_id, + worker_id, + input_id, + input_order, + accepted_path, + accepted_sha, + discovery["attempt"], + ), ) connection.execute( f""" @@ -1530,7 +2000,8 @@ def claim_deep_scan_dedup( """, (timestamp, scan_id), ) - return deep_scan_result(connection, scan_id) + result = deep_scan_result(connection, scan_id) + return result def commit_deep_scan_dedup( @@ -1556,8 +2027,14 @@ def commit_deep_scan_dedup_locked( if worker["scan_id"] != scan_id or worker["kind"] != "dedup": raise SystemExit("Dedup worker does not belong to this Deep Scan.") if worker["status"] == "succeeded": + receipt = connection.execute( + "SELECT receipt_json FROM deep_scan_merge_claims WHERE worker_id = ?", (worker_id,) + ).fetchone() + result = deep_scan_result(connection, scan_id) + if receipt is not None and receipt["receipt_json"]: + result["deepScan"]["committedMerge"] = json.loads(receipt["receipt_json"]) connection.commit() - return deep_scan_result(connection, scan_id) + return result require_running_deep_scan(connection, scan_id) if worker["status"] not in {"queued", "running"}: raise SystemExit("Only an active dedup worker can commit a result.") @@ -1603,6 +2080,23 @@ def commit_deep_scan_dedup_locked( ) if not inputs or any(row["merge_state"] != "merging" for row in inputs): raise SystemExit("Dedup inputs are not in the claimed merging state.") + claim = connection.execute( + "SELECT * FROM deep_scan_merge_claims WHERE worker_id = ?", (worker_id,) + ).fetchone() + references = [ + (row["result_manifest_path"], row["result_manifest_sha256"]) + for row in connection.execute( + "SELECT * FROM deep_scan_dedup_inputs WHERE dedup_worker_id = ? ORDER BY input_order", + (worker_id,), + ) + ] + if claim is not None and claim["previous_result_path"]: + references.append((claim["previous_result_path"], claim["previous_result_sha256"])) + for path, digest in references: + if path is not None and digest is not None: + safe_path = deep_scan_path(scan, path, "Claimed reducer input", kind="file") + if hashlib.sha256(Path(safe_path).read_bytes()).hexdigest() != digest: + raise SystemExit("A claimed Deep Scan reducer input changed after acceptance.") if candidate_ledger_path and canonical_candidate_ledger_path: canonical_path = Path(canonical_candidate_ledger_path) publication_copy = canonical_path.with_name( @@ -1646,6 +2140,27 @@ def commit_deep_scan_dedup_locked( """, (no_new_streak, timestamp, scan_id), ) + committed_worker = require_deep_scan_worker(connection, worker_id) + record_worker_attempt( + connection, + scan, + committed_worker, + timestamp, + observed_thread_id=committed_worker["sdk_thread_id"], + ) + accepted_path, accepted_sha = worker_result_reference(connection, scan, committed_worker) + result = deep_scan_result(connection, scan_id) + result["deepScan"]["committedMerge"] = { + "workerId": worker_id, + "resultManifestPath": accepted_path, + "resultManifestSha256": accepted_sha, + "newFindings": args.new_findings_count, + } + connection.execute( + "INSERT INTO deep_scan_merge_claims (worker_id, scan_id, receipt_json) VALUES (?, ?, ?) " + "ON CONFLICT(worker_id) DO UPDATE SET receipt_json = excluded.receipt_json", + (worker_id, scan_id, json.dumps(result["deepScan"]["committedMerge"])), + ) connection.commit() except BaseException: connection.rollback() @@ -1658,10 +2173,20 @@ def commit_deep_scan_dedup_locked( finish_staged_file(promotion) if publication_copy is not None: publication_copy.unlink(missing_ok=True) - return deep_scan_result(connection, scan_id) + return result -def finish_deep_scan(connection: sqlite3.Connection, args: argparse.Namespace) -> dict[str, Any]: +def finish_deep_scan( + connection: sqlite3.Connection, args: argparse.Namespace, select_finalization: bool = False +) -> dict[str, Any]: + if select_finalization: + import sys + + args = argparse.Namespace( + **vars(args), + select_finalization=True, + finalization_result_path=json.load(sys.stdin)["resultPath"], + ) scan_id = require_uuid(args.scan_id, "scan-id") with dependencies().scan_completion_lock(scan_id): return finish_deep_scan_locked(connection, args, scan_id) @@ -1675,21 +2200,89 @@ def finish_deep_scan_locked( ] if len(set(omitted_worker_ids)) != len(omitted_worker_ids): raise SystemExit("Omitted Deep Scan worker IDs must be unique.") + selecting = getattr(args, "select_finalization", False) promotion: tuple[Path, Path, Path | None] | None = None connection.execute("BEGIN IMMEDIATE") try: run = require_deep_scan_run(connection, scan_id) require_current_coordinator(run, args) scan = dependencies().require_scan(connection, scan_id) + if selecting and run["workflow_version"] != "deep-security-scan/v2": + raise SystemExit("Selected finalization requires the supported v2 workflow.") + finalization = deep_scan_finalization_input(run) + if finalization is not None and ( + args.terminal_reason != finalization["terminalReason"] + or omitted_worker_ids != finalization["omittedWorkerIds"] + ): + raise SystemExit( + "Deep Scan finalization must retain its selected reason and omissions." + ) manifest_path = ( deep_scan_output_path(scan, args.manifest_path, "Deep Scan coordinator manifest path") - if args.staged_manifest_path + if args.staged_manifest_path or selecting else deep_scan_path( scan, args.manifest_path, "Deep Scan coordinator manifest path", kind="file" ) ) standard_scan_manifest = manifest_path == str(Path(scan["scan_dir"]) / "scan-manifest.json") + failure_capped = False + if ( + standard_scan_manifest + and not selecting + and args.terminal_reason == "capped" + and (run["status"] == "running" or omitted_worker_ids) + ): + for artifact_name in ("scan-manifest.json", "findings.json", "coverage.json"): + deep_scan_path( + scan, + str(Path(scan["scan_dir"]) / artifact_name), + f"Canonical parent {artifact_name}", + kind="file", + ) + coverage = _read_scan_local_json( + Path(scan["scan_dir"]), "coverage.json", "Canonical parent coverage.json" + ) + deferred = coverage.get("deferred") + failure_capped = ( + isinstance(deferred, list) + and any( + isinstance(item, dict) + and isinstance(item.get("reason"), str) + and item["reason"].startswith("Deep Scan stopped before completion: ") + for item in deferred + ) + and connection.execute( + """ + SELECT 1 FROM deep_scan_workers + WHERE scan_id = ? AND kind = 'dedup' AND status = 'succeeded' + LIMIT 1 + """, + (scan_id,), + ).fetchone() + is not None + ) + if failure_capped and run["status"] == "running": + require_running_deep_scan(connection, scan_id) + connection.execute( + """ + UPDATE deep_scan_workers + SET merge_state = 'buffered', updated_at = ? + WHERE scan_id = ? AND kind = 'discovery' AND status = 'succeeded' + AND merge_state = 'merging' + AND id IN ( + SELECT inputs.discovery_worker_id + FROM deep_scan_dedup_inputs AS inputs + JOIN deep_scan_workers AS reducers + ON reducers.id = inputs.dedup_worker_id + AND reducers.scan_id = inputs.scan_id + WHERE inputs.scan_id = ? + AND reducers.kind = 'dedup' + AND reducers.status IN ('failed', 'canceled') + ) + """, + (dependencies().now(), scan_id, scan_id), + ) buffered_worker_ids = [ row["id"] for row in connection.execute( @@ -1742,7 +2335,10 @@ def finish_deep_scan_locked( "Deep Scan cannot finish capped before reaching its configured maximum." ) canonical_artifacts = None - if standard_scan_manifest: + if selecting: + if not standard_scan_manifest: + raise SystemExit("Selected Deep Scan finalization requires the parent manifest.") + elif standard_scan_manifest: for artifact_name in ("scan-manifest.json", "findings.json", "coverage.json"): deep_scan_path( scan, @@ -1791,6 +2387,17 @@ def finish_deep_scan_locked( f"Deep Scan {args.terminal_reason} completion must exactly identify all buffered discovery " "workers with --omitted-worker-id." ) + if selecting: + selection = selected_deep_scan_finalization( + connection, run, scan, args, omitted_worker_ids, zero_discovery_deadline + ) + connection.execute( + "UPDATE deep_scan_runs SET finalization_input_json = ?, terminal_reason = ?, " + "phase = 'terminal', updated_at = ? WHERE scan_id = ?", + (json.dumps(selection), selection["terminalReason"], dependencies().now(), scan_id), + ) + connection.commit() + return deep_scan_result(connection, scan_id) if args.staged_manifest_path: staged_manifest_path = deep_scan_path( scan, @@ -1801,7 +2408,9 @@ def finish_deep_scan_locked( promotion = promote_staged_file(staged_manifest_path, manifest_path) with ( saved_results.preserve_deep_deadline_draft(dependencies(), connection, scan) - if zero_discovery_deadline and standard_scan_manifest + if zero_discovery_deadline + and standard_scan_manifest + and (finalization is None or "publicationSha256" not in finalization) else nullcontext() ): timestamp = dependencies().now() @@ -1826,6 +2435,167 @@ def finish_deep_scan_locked( return deep_scan_result(connection, scan_id) +def final_deep_scan_reducer(connection: sqlite3.Connection, scan_id: str) -> sqlite3.Row | None: + return connection.execute( + "SELECT workers.*, attempts.accepted_result_path, attempts.accepted_result_sha256 " + "FROM deep_scan_workers AS workers LEFT JOIN deep_scan_attempts AS attempts " + "ON attempts.worker_id = workers.id AND attempts.attempt = workers.attempt " + "LEFT JOIN deep_scan_merge_claims AS own_claim " + "ON own_claim.worker_id = workers.id AND own_claim.scan_id = workers.scan_id " + "WHERE workers.scan_id = ? AND workers.kind = 'dedup' AND workers.status = 'succeeded' " + "AND NOT EXISTS (SELECT 1 FROM deep_scan_merge_claims AS claims " + "JOIN deep_scan_workers AS successor ON successor.id = claims.worker_id " + "WHERE claims.previous_worker_id = workers.id AND claims.scan_id = workers.scan_id " + "AND successor.scan_id = workers.scan_id AND successor.kind = 'dedup' " + "AND successor.status = 'succeeded') " + # Committed claims supersede legacy aggregates; legacy-only runs retain their order. + "ORDER BY (own_claim.worker_id IS NOT NULL) DESC, " + "workers.completed_at DESC, workers.rowid DESC LIMIT 1", + (scan_id,), + ).fetchone() + + +def selected_deep_scan_finalization( + connection: sqlite3.Connection, + run: sqlite3.Row, + scan: sqlite3.Row, + args: argparse.Namespace, + omitted_worker_ids: list[str], + zero_discovery_deadline: bool, +) -> dict[str, Any]: + """Select the committed attempt's immutable aggregate before publication.""" + if run["finalization_input_json"] is not None: + return json.loads(run["finalization_input_json"]) + result_path = getattr(args, "finalization_result_path", None) + relative: str | None = None + digest: str | None = None + if result_path is None: + if not zero_discovery_deadline: + raise SystemExit("Deep Scan finalization requires its accepted reducer result.") + else: + accepted = final_deep_scan_reducer(connection, scan["id"]) + if ( + accepted is None + or result_path + not in (accepted["result_manifest_path"], accepted["accepted_result_path"]) + or not accepted["accepted_result_path"] + or not accepted["accepted_result_sha256"] + ): + raise SystemExit( + "Deep Scan finalization requires its committed accepted reducer reference." + ) + scan_dir = Path(scan["scan_dir"]) + source = Path( + deep_scan_path( + scan, accepted["accepted_result_path"], "Selected Deep Scan result", kind="file" + ) + ) + relative = source.relative_to(scan_dir).as_posix() + digest = accepted["accepted_result_sha256"] + selection = { + "version": 1, + "resultPath": relative, + "resultSha256": digest, + "terminalReason": args.terminal_reason, + "omittedWorkerIds": omitted_worker_ids, + "selectedAt": dependencies().now(), + } + return selection + + +def budget_unmerged_workers(connection: sqlite3.Connection, scan_id: str) -> list[sqlite3.Row]: + return connection.execute( + "SELECT workers.*, attempts.accepted_result_path, attempts.accepted_result_sha256 " + "FROM deep_scan_workers AS workers JOIN deep_scan_attempts AS attempts " + "ON attempts.worker_id = workers.id AND attempts.attempt = workers.attempt " + "WHERE workers.scan_id = ? AND workers.kind = 'discovery' " + "AND workers.status = 'succeeded' AND workers.merge_state IN ('buffered', 'merging') " + "ORDER BY workers.completion_sequence, workers.id", + (scan_id,), + ).fetchall() + + +def prepare_budget_exhausted_deep_scan( + connection: sqlite3.Connection, + scan: sqlite3.Row, + scan_dir: Path, + warning: str, + write_draft: Callable[[dict[str, Any] | None, list[sqlite3.Row]], PreparedScanFinalization], +) -> PreparedScanFinalization: + """Finish local budget publication without starting or promoting scan work.""" + snapshots = _snapshot_published_outputs(scan_dir) + connection.execute("BEGIN IMMEDIATE") + try: + # The completion lock and this transaction preserve any committed selector. + run, scan = require_running_deep_scan(connection, scan["id"]) + unmerged = budget_unmerged_workers(connection, scan["id"]) + if run["finalization_input_json"] is not None: + selection = json.loads(run["finalization_input_json"]) + else: + reducer = final_deep_scan_reducer(connection, scan["id"]) + if reducer is not None and not reducer["accepted_result_path"]: + raise SystemExit("Budget completion requires the committed reducer reference.") + selection = selected_deep_scan_finalization( + connection, + run, + scan, + argparse.Namespace( + finalization_result_path=reducer["accepted_result_path"] if reducer else None, + terminal_reason="capped", + ), + [row["id"] for row in unmerged], + reducer is None, + ) + accepted = None + if selection["resultPath"] is not None: + descriptor = open_scan_local_file_descriptor( + scan_dir, selection["resultPath"], "Accepted reducer" + ) + with os.fdopen(descriptor, "rb") as source: + contents = source.read() + if hashlib.sha256(contents).hexdigest() != selection["resultSha256"]: + raise SystemExit("The accepted reducer changed before budget completion.") + accepted = json.loads(contents, parse_constant=reject_non_finite_json) + if ( + not isinstance(accepted, dict) + or accepted.get("scanId") != scan["id"] + or accepted.get("complete", True) is not True + or not isinstance(accepted.get("sourceCoverage"), dict) + ): + raise SystemExit("Budget completion requires the accepted complete reducer.") + prepared = write_draft(accepted, unmerged) + selection["publicationSha256"] = _selected_publication_digest(prepared) + timestamp = dependencies().now() + cancel_active_workers(connection, scan["id"], timestamp) + connection.execute( + "UPDATE deep_scan_runs SET status = 'succeeded', phase = 'terminal', " + "terminal_reason = ?, cancel_requested = 1, error_message = ?, " + "manifest_path = ?, finalization_input_json = ?, completed_at = ?, updated_at = ? " + "WHERE scan_id = ?", + ( + selection["terminalReason"], + warning, + str(scan_dir / "scan-manifest.json"), + json.dumps(selection), + timestamp, + timestamp, + scan["id"], + ), + ) + warnings = json.loads(scan["completion_warnings_json"]) + if warning not in warnings: + connection.execute( + "UPDATE scans SET completion_warnings_json = ? WHERE id = ?", + (json.dumps([*warnings, warning]), scan["id"]), + ) + connection.commit() + return prepared + except BaseException: + connection.rollback() + _restore_published_outputs(scan_dir, snapshots) + raise + + def fail_deep_scan(connection: sqlite3.Connection, args: argparse.Namespace) -> dict[str, Any]: scan_id = require_uuid(args.scan_id, "scan-id") with dependencies().scan_completion_lock(scan_id): @@ -2006,6 +2776,11 @@ def cancel_from_parent_scan(connection: sqlite3.Connection, scan_id: str, timest def cancel_active_workers(connection: sqlite3.Connection, scan_id: str, timestamp: str) -> None: + connection.execute( + "UPDATE deep_scan_attempts SET status = 'canceled', completed_at = ?, " + "end_reason = 'scan_stopped' WHERE scan_id = ? AND completed_at IS NULL", + (timestamp, scan_id), + ) connection.execute( """ UPDATE deep_scan_workers diff --git a/plugins/codex-security/scripts/finalize_scan_contract.py b/plugins/codex-security/scripts/finalize_scan_contract.py index ae8fdbd349..3c68cf1fc6 100644 --- a/plugins/codex-security/scripts/finalize_scan_contract.py +++ b/plugins/codex-security/scripts/finalize_scan_contract.py @@ -3011,6 +3011,8 @@ def _prepare_scan_finalization( allow_missing_writeups: bool = False, draft_documents: tuple[dict[str, Any], dict[str, Any], dict[str, Any]] | None = None, finding_scope: list[tuple[str, bool]] | None = None, + require_complete: bool = False, + accepted_findings: bool = False, ) -> PreparedScanFinalization: """Prepare validated documents and warnings without writing any output files.""" @@ -3023,6 +3025,10 @@ def _prepare_scan_finalization( else _read_scan_local_json(scan_dir, "scan-manifest.json", "scan-manifest.json") ) scan = _require_dict(manifest, "scan", "manifest") + if require_complete and scan.get("complete") is False: + raise RecoverableContractError( + "The latest saved scan draft is incomplete; continue the scan before completing it." + ) if scan.get("sealedAt") is None and scan.get("artifacts") == []: del scan["artifacts"] was_sealed = scan.get("sealedAt") is not None or scan.get("artifacts") is not None @@ -3080,8 +3086,12 @@ def _prepare_scan_finalization( if not was_sealed and scan["status"] == "completed": try: _populate_unsealed_finding_identities(manifest, findings) - _validate_findings(manifest, findings) - _validate_current_findings(findings, schema_dir) + if accepted_findings: + # SHA-verified Deep selections retain the historical artifact read contract. + findings_for_validation = _legacy_sealed_findings_for_validation(findings) + _validate_findings(manifest, findings_for_validation) + if not accepted_findings: + _validate_current_findings(findings, schema_dir) except ContractError as exc: raise RecoverableContractError( "Scan result contains invalid findings; repair the finding records " diff --git a/plugins/codex-security/scripts/report_projection.py b/plugins/codex-security/scripts/report_projection.py index 552f6e106d..3e1667cfe6 100644 --- a/plugins/codex-security/scripts/report_projection.py +++ b/plugins/codex-security/scripts/report_projection.py @@ -489,7 +489,14 @@ def _target_scope_lines(target: dict[str, Any]) -> list[str]: def _surface_notes(surface: dict[str, Any]) -> str: - notes = surface.get("notes", "No additional canonical notes were recorded.") + notes = surface.get( + "notes", surface.get("reason", "No additional canonical notes were recorded.") + ) + if "notes" in surface and "reason" in surface and surface["reason"] != notes: + notes += f" {surface['reason']}" + source = _coverage_source(surface) + if source: + notes = f"{source}. {notes}" receipt_refs = surface.get("receiptRefs", []) if not isinstance(receipt_refs, list) or not receipt_refs: return _cell(notes) @@ -499,6 +506,16 @@ def _surface_notes(surface: dict[str, Any]) -> str: return _cell(f"{notes} Evidence: {evidence}") +def _coverage_source(item: dict[str, Any]) -> str: + provenance = item.get("provenance", {}) + if not isinstance(provenance, dict) or not provenance.get("workerId"): + return "" + source = f"Review {provenance['workerId']}" + if provenance.get("attempt") is not None: + source += f", attempt {provenance['attempt']}" + return source + + def _remediation_section(finding: dict[str, Any]) -> list[str]: remediation = _text(finding.get("remediation"), "No canonical remediation was recorded.") lines = ["", "#### Remediation", "", remediation] @@ -957,6 +974,22 @@ def build_report_markdown( f"[Open the structural hardening portfolio]({hardening_portfolio_path})", ] ) + reviews = coverage.get("reviews", []) + if reviews: + lines.extend( + [ + "", + "## Source Review Coverage", + "", + "| Review | Attempt |", + "| --- | --- |", + ] + ) + for review in reviews: + if isinstance(review, dict): + lines.append( + f"| {_cell(review.get('workerId'))} | {_cell(str(review.get('attempt', 'unknown')))} |" + ) surfaces = coverage.get("surfaces", []) if surfaces: lines.extend( @@ -994,6 +1027,7 @@ def build_report_markdown( questions.extend( { "question": item.get("reason", "Deferred review requires follow-up."), + "provenance": item.get("provenance", {}), "followUpPrompt": " ".join( ( f"Review deferred unit {item.get('id', 'unknown')} and close its stated proof gap.", @@ -1015,6 +1049,9 @@ def build_report_markdown( if not isinstance(question, dict): continue lines.append(f"- {_text(question.get('question'), 'Unspecified open question.')}") + source = _coverage_source(question) + if source: + lines.append(f" - {_text(source, '')}.") prompt = _text(question.get("followUpPrompt"), "") if prompt: lines.append(f" - Follow-up prompt: {prompt}") diff --git a/plugins/codex-security/scripts/workbench/saved_coverage.py b/plugins/codex-security/scripts/workbench/saved_coverage.py index d4a439ee97..1059283a90 100644 --- a/plugins/codex-security/scripts/workbench/saved_coverage.py +++ b/plugins/codex-security/scripts/workbench/saved_coverage.py @@ -98,6 +98,7 @@ def _prepare_worker_coverage_recovery( source_order: dict[str, tuple[int, int]], *, enabled: bool, + preserve_workers: set[str] | None = None, ) -> tuple[ bool, list[tuple[dict[str, Any], str, dict[str, Any], dict[str, Any], dict[str, Any] | None]] ]: @@ -173,6 +174,10 @@ def _prepare_worker_coverage_recovery( row["receiptRefs"] = refs if field == "surfaces": row.setdefault("receiptRefs", []) + # Accepted reviews keep their host projection while historical + # worker paths remain available for receipt repair. + if preserve_workers and owner in preserve_workers: + continue aggregate_rows = _aggregate_worker_coverage_rows(rows, owner, field) projected[field].extend(aggregate_rows) if field in {"surfaces", "deferred"}: @@ -232,3 +237,56 @@ def _prepare_worker_coverage_recovery( if _coverage_comparison_row(row, field) not in projected[field] ] return True, surface_links + + +def _merge_reviewed_surface_receipts( + coverage: dict[str, Any], + worker_row_owners: dict[int, str], + reviewed_workers: set[str], + reviewed_attempts: set[tuple[Any, Any]], +) -> None: + # Receipt recovery may run before the accepted reducer copy is emitted. + # Combine its repaired evidence after every source has been accumulated. + surfaces = coverage.get("surfaces", []) + merged_surfaces: set[int] = set() + for retained in surfaces if isinstance(surfaces, list) else []: + if not isinstance(retained, dict): + continue + owner = worker_row_owners.get(id(retained)) + if owner not in reviewed_workers: + continue + for existing in surfaces: + provenance = existing.get("provenance") if isinstance(existing, dict) else None + if not ( + id(existing) not in worker_row_owners + and isinstance(provenance, dict) + and provenance.get("workerId") == owner + and (owner, provenance.get("attempt")) in reviewed_attempts + and isinstance(provenance.get("sourceId"), str) + and retained.get("id") == f"{owner}/{provenance['sourceId']}" + and isinstance(existing.get("receiptRefs"), list) + and isinstance(retained.get("receiptRefs"), list) + ): + continue + source_provenance = retained.get("provenance") + projected_provenance = ( + dict(source_provenance) if isinstance(source_provenance, dict) else {} + ) + for key in ("workerId", "attempt", "sourceId", "candidateId"): + projected_provenance.pop(key, None) + if key in provenance: + projected_provenance[key] = provenance[key] + projected = { + **retained, + "id": existing.get("id"), + "provenance": projected_provenance, + "receiptRefs": existing["receiptRefs"], + } + if projected == existing: + for ref in retained["receiptRefs"]: + if ref not in existing["receiptRefs"]: + existing["receiptRefs"].append(ref) + merged_surfaces.add(id(retained)) + break + if merged_surfaces: + coverage["surfaces"] = [row for row in surfaces if id(row) not in merged_surfaces] diff --git a/plugins/codex-security/scripts/workbench/saved_sources.py b/plugins/codex-security/scripts/workbench/saved_sources.py new file mode 100644 index 0000000000..f0e0b25ac5 --- /dev/null +++ b/plugins/codex-security/scripts/workbench/saved_sources.py @@ -0,0 +1,557 @@ +"""Read and freeze saved scan source identities and checkpoint order.""" + +from __future__ import annotations + +import hashlib +import json +import re +import stat +from collections.abc import Iterator +from pathlib import Path +from typing import Any + +from finalize_scan_contract import ( + ContractError, + _read_scan_local_json, + _read_scan_local_json_with_metadata, + write_scan_local_bytes, +) + + +def _encoded(value: Any) -> bytes: + return json.dumps( + value, ensure_ascii=True, allow_nan=False, sort_keys=True, separators=(",", ":") + ).encode() + + +def _digest(value: Any) -> str: + return hashlib.sha256(_encoded(value)).hexdigest() + + +def _children(scan_dir: Path, relative: str) -> list[str]: + cursor = scan_dir + for part in Path(relative).parts: + if part in {"..", "."}: + return [] + cursor = cursor / part + try: + if not stat.S_ISDIR(cursor.lstat().st_mode): + return [] + except FileNotFoundError: + return [] + return sorted(child.name for child in cursor.iterdir()) + + +def _saved_workers(connection: Any, scan_id: str) -> list[dict[str, Any]]: + rows = connection.execute( + "SELECT worker.*, attempt.accepted_result_path FROM deep_scan_workers AS worker " + "LEFT JOIN deep_scan_attempts AS attempt " + "ON attempt.worker_id = worker.id AND attempt.attempt = worker.attempt " + "WHERE worker.scan_id = ? ORDER BY worker.created_at, worker.id", + (scan_id,), + ).fetchall() + return [ + { + **dict(row), + "result_manifest_path": row["accepted_result_path"] or row["result_manifest_path"], + } + for row in rows + ] + + +def _accepted_source_digests(connection: Any, scan_id: str) -> dict[str, str]: + return { + row["accepted_result_path"]: row["accepted_result_sha256"] + for row in connection.execute( + "SELECT accepted_result_path, accepted_result_sha256 FROM deep_scan_attempts " + "WHERE scan_id = ? AND accepted_result_path IS NOT NULL " + "AND accepted_result_sha256 IS NOT NULL", + (scan_id,), + ) + } + + +def _latest_successful_reducer(workers: list[Any]) -> Any | None: + return max( + ( + worker + for worker in workers + if worker["kind"] == "dedup" + and worker["status"] == "succeeded" + and worker["result_manifest_path"] + ), + key=lambda worker: (worker["completed_at"] or "", worker["id"]), + default=None, + ) + + +def _checkpoint_paths(scan_dir: Path, directory: str) -> list[str]: + return [ + f"{directory}/{name}" + for name in _children(scan_dir, directory) + if re.fullmatch(r"[0-9a-f]{64}\.json", name) + ] + + +def _worker_outputs(scan_dir: Path, worker: Any) -> list[tuple[str, int]]: + output = Path(worker["artifact_dir"]).relative_to(scan_dir) + attempts = (output.parent if output.name == "output" else output) / "attempts" + archived = [ + ((attempts / name).as_posix(), int(name.split("-")[1])) + for name in _children(scan_dir, attempts.as_posix()) + if re.fullmatch(r"attempt-\d+", name) + ] + attempt = int(worker["attempt"] or 0) if "attempt" in worker.keys() else 0 + if worker["kind"] == "discovery" and not attempt: + attempt = max((attempt for _, attempt in archived), default=0) + 1 + return [(output.as_posix(), attempt), *archived] + + +def _saved_result_paths(scan_dir: Path, workers: list[Any]) -> Iterator[tuple[str, str | None]]: + latest_reducer = _latest_successful_reducer(workers) + yield "checkpoint-head.json", None + for directory in ("checkpoint-heads", "checkpoints"): + yield from ((path, None) for path in _checkpoint_paths(scan_dir, directory)) + for worker in workers: + if worker["kind"] not in {"dedup", "discovery"}: + continue + try: + outputs = _worker_outputs(scan_dir, worker) + except (TypeError, ValueError): + continue + for directory, _ in outputs: + if worker["kind"] == "discovery": + yield f"{directory}/checkpoint-head.json", worker["kind"] + yield from ( + (path, worker["kind"]) + for path in _checkpoint_paths(scan_dir, f"{directory}/checkpoint-heads") + ) + checkpoint_paths = _checkpoint_paths(scan_dir, f"{directory}/checkpoints") + if worker["kind"] == "discovery" or checkpoint_paths: + yield f"{directory}/result.json", worker["kind"] + yield from ((path, worker["kind"]) for path in checkpoint_paths) + if worker["result_manifest_path"] and ( + worker["kind"] == "discovery" + or (latest_reducer is not None and worker["id"] == latest_reducer["id"]) + ): + try: + yield ( + Path(worker["result_manifest_path"]).relative_to(scan_dir).as_posix(), + worker["kind"], + ) + except ValueError: + continue + + +def _checkpoint_head_directory(relative: str) -> Path | None: + path = Path(relative) + if path.name == "checkpoint-head.json": + return path.parent + if path.parent.name == "checkpoint-heads": + return path.parent.parent + return None + + +def _capture_saved_source( + scan_dir: Path, + relative: str, + scan_id: str, + *, + kind: str | None = None, + snapshot_head: bool = True, + write: bool = True, + accepted_source_digests: dict[str, str] | None = None, +) -> dict[str, tuple[str, int]]: + if not snapshot_head or Path(relative).name != "checkpoint-head.json": + _, digest, observed = _read_saved_result( + scan_dir, relative, scan_id, kind=kind, accepted_source_digests=accepted_source_digests + ) + return {relative: (digest, observed)} + head, _, observed = _read_saved_result( + scan_dir, relative, scan_id, accepted_source_digests=accepted_source_digests + ) + observation = {"checkpoint": head["checkpoint"], "observedAtNs": str(observed)} + directory = Path(relative).parent + selected = (directory / "checkpoints" / observation["checkpoint"]).as_posix() + _, selected_digest, selected_time = _read_saved_result( + scan_dir, selected, scan_id, accepted_source_digests=accepted_source_digests + ) + digest = _digest(observation) + snapshot = (directory / "checkpoint-heads" / f"{digest}.json").as_posix() + # Capture the selected file even if the worker created it after directory enumeration. + if write and not (scan_dir / snapshot).exists(): + write_scan_local_bytes(scan_dir, snapshot, _encoded(observation)) + return { + snapshot: (digest, int(observation["observedAtNs"])), + selected: (selected_digest, selected_time), + } + + +def _is_source_order_snapshot(relative: str) -> bool: + path = Path(relative) + return path.parent == Path("source-order") and bool( + re.fullmatch(r"[0-9a-f]{64}\.json", path.name) + ) + + +def _read_saved_result( + scan_dir: Path, + relative: str, + scan_id: str, + *, + kind: str | None = None, + accepted_source_digests: dict[str, str] | None = None, +) -> tuple[dict[str, Any], str, int]: + draft, contents, metadata = _read_scan_local_json_with_metadata( + scan_dir, relative, "Saved scan checkpoint" + ) + expected = (accepted_source_digests or {}).get(str(scan_dir / relative)) + if expected is not None and hashlib.sha256(contents).hexdigest() != expected: + raise ContractError("checkpoint changed after acceptance") + directory = _checkpoint_head_directory(relative) + if directory is not None: + checkpoint = draft.get("checkpoint") + if not isinstance(checkpoint, str) or not re.fullmatch(r"[0-9a-f]{64}\.json", checkpoint): + raise ContractError("checkpoint head does not name a saved checkpoint") + _read_saved_result( + scan_dir, + (directory / "checkpoints" / checkpoint).as_posix(), + scan_id, + accepted_source_digests=accepted_source_digests, + ) + if Path(relative).name == "checkpoint-head.json": + return draft, _digest([draft, metadata.st_mtime_ns]), metadata.st_mtime_ns + observed = draft.get("observedAtNs") + if not isinstance(observed, str) or not re.fullmatch(r"-?[0-9]+", observed): + raise ContractError("checkpoint head has no observation time") + return draft, _digest(draft), int(observed) + if draft.get("scanId") != scan_id: + raise ContractError("checkpoint belongs to a different scan") + if not _is_source_order_snapshot(relative) and ( + not isinstance(draft.get("findings"), list) + or not isinstance( + draft.get("sourceCoverage", draft.get("coverage", {})) + if kind == "dedup" + else draft.get("coverage"), + dict, + ) + ): + raise ContractError("checkpoint has no semantic findings or coverage") + return draft, _digest(draft), metadata.st_mtime_ns + + +def _frozen_source_times(scan_dir: Path, scan_id: str, sources: dict[str, str]) -> dict[str, int]: + times: dict[str, int] = {} + snapshots = [path for path in sources if _is_source_order_snapshot(path)] + for path in snapshots: + record, digest, _ = _read_saved_result(scan_dir, path, scan_id) + if digest != sources[path] or not isinstance(record.get("sources"), dict): + raise ContractError("saved source ordering changed after the scan stopped") + for relative, observation in record["sources"].items(): + if ( + not isinstance(observation, dict) + or relative not in sources + or observation.get("digest") != sources[relative] + or not isinstance(observation.get("observedAtNs"), str) + or not re.fullmatch(r"-?[0-9]+", observation["observedAtNs"]) + ): + raise ContractError("saved source ordering does not match its frozen sources") + observed = int(observation["observedAtNs"]) + if relative in times and times[relative] != observed: + raise ContractError("saved source ordering has conflicting observations") + times[relative] = observed + if snapshots and sources.keys() - set(snapshots) - times.keys(): + raise ContractError("saved source ordering is incomplete") + return times + + +def _freeze_source_times( + scan_dir: Path, scan_id: str, sources: dict[str, str], times: dict[str, int] +) -> None: + # Identical result rewrites must not change the order of frozen review evidence. + observations = { + path: {"digest": digest, "observedAtNs": str(times[path])} + for path, digest in sources.items() + if not _is_source_order_snapshot(path) + } + if not observations: + return + record = {"scanId": scan_id, "sources": observations} + digest = _digest(record) + path = f"source-order/{digest}.json" + if not (scan_dir / path).exists(): + write_scan_local_bytes(scan_dir, path, _encoded(record)) + if _read_saved_result(scan_dir, path, scan_id)[1] != digest: + raise ContractError("saved source ordering does not match its digest") + sources[path] = digest + + +def _parent_scan_draft( + scan_id: str, + parent_scan: dict[str, Any], + findings: dict[str, Any], + coverage: dict[str, Any], +) -> dict[str, Any]: + parent = { + "scanId": scan_id, + "findings": findings.get("findings"), + "coverage": coverage, + **{ + key: parent_scan[key] + for key in ("scope", "threatModel", "complete") + if key in parent_scan + }, + } + if not isinstance(parent["findings"], list): + raise ContractError("Saved parent draft has no findings array") + return parent + + +def _worker_checkpoint_head( + scan_dir: Path, + directory: str, + scan_id: str, + accepted_source_digests: dict[str, str] | None = None, +) -> str | None: + relative = f"{directory}/checkpoint-head.json" + try: + (scan_dir / relative).lstat() + except FileNotFoundError: + return None + head = _read_scan_local_json(scan_dir, relative, "Saved worker checkpoint head") + name = head.get("checkpoint") + if not isinstance(name, str) or not re.fullmatch(r"[0-9a-f]{64}\.json", name): + raise ContractError("Saved worker checkpoint head is invalid.") + checkpoint = f"{directory}/checkpoints/{name}" + # A committed head precedes replacement of result.json. Do not fall back to + # that older result if the selected checkpoint cannot be read. + _read_saved_result( + scan_dir, checkpoint, scan_id, accepted_source_digests=accepted_source_digests + ) + return checkpoint + + +def _worker_checkpoint_heads( + scan_dir: Path, + workers: list[Any], + scan_id: str, + accepted_source_digests: dict[str, str] | None = None, +) -> dict[str, str]: + heads: dict[str, str] = {} + for worker in workers: + if worker["kind"] != "discovery": + continue + try: + output = Path(worker["artifact_dir"]).relative_to(scan_dir) + except (TypeError, ValueError): + continue + attempts = (output.parent if output.name == "output" else output) / "attempts" + directories = [output] + [ + attempts / name + for name in _children(scan_dir, attempts.as_posix()) + if re.fullmatch(r"attempt-\d+", name) + ] + for directory in directories: + relative = directory.as_posix() + head = _worker_checkpoint_head(scan_dir, relative, scan_id, accepted_source_digests) + if head is not None: + heads[relative] = head + return heads + + +def _read_saved_parent_result( + scan_dir: Path, scan_id: str +) -> tuple[dict[str, Any], dict[str, Any]]: + manifest = _read_scan_local_json(scan_dir, "scan-manifest.json", "Saved parent manifest") + findings = _read_scan_local_json(scan_dir, "findings.json", "Saved parent findings") + coverage = _read_scan_local_json(scan_dir, "coverage.json", "Saved parent coverage") + parent_scan = manifest.get("scan") + if not isinstance(parent_scan, dict): + raise ContractError("Saved parent manifest has no scan object") + if (parent_scan.get("sealedAt") or parent_scan.get("artifacts")) and ( + parent_scan.get("id", scan_id) != scan_id + or findings.get("scanId", scan_id) != scan_id + or coverage.get("scanId", scan_id) != scan_id + ): + raise ContractError("Saved parent documents belong to a different scan") + return manifest, _parent_scan_draft(scan_id, parent_scan, findings, coverage) + + +def _source_digests(value: Any, error: str) -> dict[str, str]: + if not isinstance(value, dict) or not all( + isinstance(relative, str) and isinstance(digest, str) for relative, digest in value.items() + ): + raise ContractError(error) + return value + + +def _retained_source_state(value: Any) -> tuple[dict[str, str], str | None]: + if isinstance(value, dict) and isinstance(value.get("sources"), dict): + sources = _source_digests( + value["sources"], "Saved stopped-scan source digests are malformed." + ) + model_source = value.get("threatModelSource") + if not isinstance(model_source, str) or model_source not in sources: + raise ContractError("Saved stopped-scan model source is outside its checkpoint set.") + return sources, model_source + return _source_digests(value, "Saved stopped-scan source digests are malformed."), None + + +def _encode_retained_sources(sources: dict[str, str], model_source: list[str]) -> str: + state = {"sources": sources, "threatModelSource": model_source[0]} if model_source else sources + return json.dumps(state, sort_keys=True) + + +def _saved_results_changed(db: Any, connection: Any, scan: Any) -> bool: + try: + scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"])) + manifest_path = db.artifact_path(scan_dir, db.ARTIFACTS["manifest"], required=False) + workers = _saved_workers(connection, scan["id"]) + accepted_digests = _accepted_source_digests(connection, scan["id"]) + paths = dict(_saved_result_paths(scan_dir, workers)) + frozen_sources = scan["retained_source_digests_json"] + + def has_saved_source() -> bool: + for path in paths: + try: + _read_saved_result( + scan_dir, + path, + scan["id"], + kind=paths[path], + accepted_source_digests=accepted_digests, + ) + return True + except (ContractError, OSError, ValueError): + continue + return False + + if manifest_path is None: + if frozen_sources is not None: + return bool(_retained_source_state(json.loads(frozen_sources))[0]) + return has_saved_source() + if scan["seal_manifest_digest"] is None: + try: + _read_saved_parent_result(scan_dir, scan["id"]) + return True + except (ContractError, OSError, ValueError): + pass + return has_saved_source() + manifest = _read_scan_local_json( + scan_dir, + manifest_path.relative_to(scan_dir).as_posix(), + "Saved scan manifest", + ) + manifest_scan = manifest.get("scan") + if not isinstance(manifest_scan, dict): + return True + published_sources = _source_digests( + manifest_scan.get("preservedSources", {}), + "Published scan source digests are malformed.", + ) + if _worker_checkpoint_heads( + scan_dir, workers, scan["id"], accepted_digests + ) != manifest_scan.get("preservedCheckpointHeads", {}): + return True + current_sources = dict(published_sources) + paths.update({path: None for path in published_sources if _is_source_order_snapshot(path)}) + for path in paths: + try: + captured = _capture_saved_source( + scan_dir, + path, + scan["id"], + kind=paths[path], + accepted_source_digests=accepted_digests, + snapshot_head=path not in published_sources, + write=False, + ) + current_sources.update({path: value[0] for path, value in captured.items()}) + except (ContractError, OSError, ValueError): + continue + return current_sources != published_sources + except (ContractError, OSError, SystemExit, ValueError): + return False + + +def _recovery_source_digests( + db: Any, connection: Any, scan: Any +) -> tuple[dict[str, str], bool, dict[str, str]]: + scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"])) + frozen_sources: dict[str, str] | None = None + include_parent = True + raw_frozen_sources = scan["retained_source_digests_json"] + if raw_frozen_sources is not None: + frozen_sources, _ = _retained_source_state(json.loads(raw_frozen_sources)) + include_parent = False + + manifest_path = db.artifact_path(scan_dir, db.ARTIFACTS["manifest"], required=False) + if manifest_path is not None: + manifest = _read_scan_local_json( + scan_dir, + manifest_path.relative_to(scan_dir).as_posix(), + "Saved scan manifest", + ) + manifest_scan = manifest.get("scan") + if not isinstance(manifest_scan, dict): + raise ContractError("Saved scan manifest has no scan object") + if scan["seal_manifest_digest"] is not None or ( + manifest_scan.get("sealedAt") is not None or manifest_scan.get("artifacts") is not None + ): + if "preservedSources" in manifest_scan: + published_sources = _source_digests( + manifest_scan["preservedSources"], + "Published scan source digests are malformed.", + ) + include_parent = not published_sources + if published_sources: + if frozen_sources is not None and frozen_sources != published_sources: + raise ContractError( + "Stopped scan sources changed after terminal publication." + ) + frozen_sources = published_sources + elif frozen_sources is None: + frozen_sources = {} + else: + include_parent = True + + workers = _saved_workers(connection, scan["id"]) + accepted_digests = _accepted_source_digests(connection, scan["id"]) + checkpoint_heads = _worker_checkpoint_heads(scan_dir, workers, scan["id"], accepted_digests) + paths = dict(_saved_result_paths(scan_dir, workers)) + recovery_sources = dict(frozen_sources or {}) + source_times = _frozen_source_times(scan_dir, scan["id"], recovery_sources) + for relative, expected_digest in recovery_sources.items(): + try: + _, digest, observed = _read_saved_result( + scan_dir, + relative, + scan["id"], + kind=paths.get(relative), + accepted_source_digests=accepted_digests, + ) + except (ContractError, OSError, ValueError) as exc: + raise ContractError("Frozen stopped-scan checkpoint set is incomplete.") from exc + if digest != expected_digest: + raise ContractError("checkpoint changed after the scan stopped") + if not _is_source_order_snapshot(relative): + source_times.setdefault(relative, observed) + + for relative in paths.keys() - recovery_sources.keys(): + try: + captured = _capture_saved_source( + scan_dir, + relative, + scan["id"], + kind=paths[relative], + accepted_source_digests=accepted_digests, + ) + except (ContractError, OSError, ValueError): + continue + for path, (digest, observed) in captured.items(): + if path in recovery_sources and recovery_sources[path] != digest: + raise ContractError("checkpoint changed after the scan stopped") + recovery_sources[path] = digest + source_times.setdefault(path, observed) + _freeze_source_times(scan_dir, scan["id"], recovery_sources, source_times) + return recovery_sources, include_parent, checkpoint_heads diff --git a/plugins/codex-security/scripts/workbench_cli.py b/plugins/codex-security/scripts/workbench_cli.py index 7d3c86ee25..daf4c34e20 100644 --- a/plugins/codex-security/scripts/workbench_cli.py +++ b/plugins/codex-security/scripts/workbench_cli.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse +import json import sys from pathlib import Path @@ -30,7 +31,7 @@ def add_user_context(parser: argparse.ArgumentParser, *, required: bool = False) context.add_argument("--user-context-stdin", action="store_true") -def parse_args(description: str) -> argparse.Namespace: +def parse_args(description: str, *, execution_settings: bool = False) -> argparse.Namespace: parser = argparse.ArgumentParser(description=description) subparsers = parser.add_subparsers(dest="command", required=True) @@ -310,6 +311,14 @@ def add_command(name: str, *required: str) -> argparse.ArgumentParser: subparsers.add_parser("finding-workflow") subparsers.add_parser("local-dedupe") args = parser.parse_args() + if execution_settings: + args.require_execution_settings = True + if args.command == "begin-deep-scan": + # Private MCP creation input shares stdin with the user's unchanged context. + payload = json.load(sys.stdin) + args.execution_settings = payload["executionSettings"] + args.user_context = payload.get("userContext") + args.user_context_stdin = False if getattr(args, "user_context_stdin", False): args.user_context = sys.stdin.buffer.read().decode("utf-8") args.user_context_stdin = False diff --git a/plugins/codex-security/scripts/workbench_db.py b/plugins/codex-security/scripts/workbench_db.py index a80f4c7638..7d944dd1ce 100644 --- a/plugins/codex-security/scripts/workbench_db.py +++ b/plugins/codex-security/scripts/workbench_db.py @@ -46,13 +46,12 @@ from finalize_scan_contract import ( PRODUCER_NAME, ContractError, + PreparedScanFinalization, RecoverableContractError, _prepare_scan_finalization, _write_prepared_scan_finalization, finalize_scan, - finding_candidate_id, open_scan_local_file_descriptor, - write_scan_local_bytes, ) from finding_preview import bounded_finding_details from workbench import handoff @@ -132,13 +131,16 @@ bounded_output_text, optional_text, parse_scan_cost, + parse_scan_recipe, path_within_scope, reject_non_finite_json, require_close_note, require_occurrence, + require_target, require_uuid, sqlite_busy, user_context_argument, + validate_completed_scan_artifacts, ) FINDING_ARTIFACT_DIRECTORIES_LIMIT = 80 @@ -266,16 +268,6 @@ def apply_migrations(connection: sqlite3.Connection, *, immediate: bool = False) ) -def require_target(value: str) -> Path: - expanded = Path(value).expanduser() - if not expanded.is_absolute(): - raise SystemExit("Scan target must be an absolute local directory path.") - target = expanded.resolve() - if not target.is_dir(): - raise SystemExit(f"Scan target is not a readable local directory: {target}") - return target - - def inspect_target(target_path: str) -> dict[str, Any]: target = require_target(target_path) return { @@ -1210,18 +1202,22 @@ def complete_budget_exhausted_scan( connection: sqlite3.Connection, args: argparse.Namespace ) -> dict[str, Any]: scan_id = require_uuid(args.scan_id, "scan-id") - cost_json = parse_scan_cost(args.cost_json) + cost_json = parse_scan_cost(args.cost_json, allow_lower_bound=True) if cost_json is None: raise SystemExit("Budget-exhausted scan completion requires the measured scan cost.") with scan_completion_lock(scan_id): scan = require_scan(connection, scan_id) if scan["status"] != "running" or scan["mode"] != "deep" or scan["recipe_json"] is None: raise SystemExit("Only a running CLI Deep Scan can complete after its cost limit.") + handoff.require_current_continuation( + scan, None, error_message="Scan completion is owned by another continuation." + ) recipe = json.loads(scan["recipe_json"], parse_constant=reject_non_finite_json) if not isinstance(recipe, dict) or recipe.get("mode") != "deep": raise SystemExit("Budget-exhausted scan completion requires a Deep Scan launch recipe.") cost = json.loads(cost_json) - measured = cost.get("cost", cost) + lower_bound = set(cost) == {"lowerBound"} + measured = cost["lowerBound"] if lower_bound else cost.get("cost", cost) limit = recipe.get("maxCostUsd") if ( not isinstance(limit, (int, float)) @@ -1230,11 +1226,13 @@ def complete_budget_exhausted_scan( or measured.get("estimatedUsd", 0) <= limit ): raise SystemExit("Deep Scan has not exceeded its configured cost limit.") - run = connection.execute( - "SELECT status, terminal_reason, manifest_path FROM deep_scan_runs WHERE scan_id = ?", - (scan_id,), - ).fetchone() - if ( + run = deep_scan.find_supported_deep_scan_run(connection, scan_id) + before_selection = ( + run is not None + and run["status"] == "running" + and run["workflow_version"] == "deep-security-scan/v2" + ) + if not before_selection and ( run is None or run["status"] != "succeeded" or run["terminal_reason"] not in {"saturated", "capped"} @@ -1247,16 +1245,88 @@ def complete_budget_exhausted_scan( scan_dir = require_canonical_scan_directory(Path(scan["scan_dir"])) candidates = ( [] - if run["manifest_path"] == str(scan_dir / "scan-manifest.json") + if before_selection or run["manifest_path"] == str(scan_dir / "scan-manifest.json") else budget_exhausted_candidates(scan, scan_dir) ) warning = optional_text(args.message, maximum=2400) if warning is None: + retained = ( + "saved work was retained" + if before_selection + else "completed discovery was preserved" + ) warning = ( f"Deep Scan reached its cost limit after an estimated " - f"${measured['estimatedUsd']:.6g}; completed discovery was preserved." + f"${measured['estimatedUsd']:.6g}; {retained}." + ) + selection = deep_scan.deep_scan_finalization_input(run) + draft_documents = None + sealed_replay = False + try: + accepted_findings = saved_results.has_verified_selected_result( + _WORKBENCH_DB_CONTEXT, connection, scan ) - budget_exhausted_draft(scan, scan_dir, candidates, warning) + if selection is not None: + if "publicationSha256" in selection: + draft_documents = tuple( + saved_results._read_scan_local_json(scan_dir, name, name) + for name in ("scan-manifest.json", "findings.json", "coverage.json") + ) + manifest = draft_documents[0] + sealed = manifest["scan"].get("sealedAt") is not None or bool( + manifest["scan"].get("artifacts") + ) + validated = _prepare_scan_finalization( + scan_dir, + expected_coverage_mode=expected_coverage_mode(scan), + completion_binding=workbench_completion_binding(scan, now(), manifest), + draft_documents=None if sealed else draft_documents, + accepted_findings=accepted_findings, + ) + saved_results.require_selected_publication( + _WORKBENCH_DB_CONTEXT, connection, scan, validated + ) + sealed_replay = validated[5] + if before_selection: + prepared = deep_scan.prepare_budget_exhausted_deep_scan( + connection, + scan, + scan_dir, + warning, + lambda accepted, unmerged: budget_exhausted_draft( + scan, + scan_dir, + [], + warning, + before_selection=True, + accepted_result=accepted, + unmerged_workers=unmerged, + draft_documents=draft_documents, + accepted_findings=accepted_findings or accepted is not None, + ), + ) + else: + prepared = budget_exhausted_draft( + scan, + scan_dir, + candidates, + warning, + unmerged_workers=deep_scan.budget_unmerged_workers(connection, scan_id), + draft_documents=draft_documents, + accepted_findings=accepted_findings, + ) + if selection is not None: + if sealed_replay: + # Bind replay to the validated snapshot, even if disk lost its seal. + saved_results.require_selected_publication( + _WORKBENCH_DB_CONTEXT, connection, scan, prepared + ) + else: + saved_results.record_selected_publication( + _WORKBENCH_DB_CONTEXT, connection, scan, prepared + ) + except ContractError as exc: + raise SystemExit(str(exc)) from exc warnings = json.loads(scan["completion_warnings_json"]) if warning not in warnings: connection.execute( @@ -1264,7 +1334,15 @@ def complete_budget_exhausted_scan( (json.dumps([*warnings, warning]), scan_id), ) connection.commit() - return complete_scan_locked(connection, scan_id, None, cost_json) + # A priced subtotal proves the stop but is not the saved total estimate. + # Sealed replay must verify current disk bytes again at completion. + return complete_scan_locked( + connection, + scan_id, + None, + None if lower_bound else cost_json, + prepared=None if prepared[5] else prepared, + ) def budget_exhausted_candidates(scan: sqlite3.Row, scan_dir: Path) -> list[dict[str, Any]]: @@ -1344,147 +1422,47 @@ def budget_exhausted_draft( scan_dir: Path, candidates: list[dict[str, Any]], warning: str, -) -> None: - documents: dict[str, dict[str, Any]] = {} - for name in ("scan-manifest.json", "findings.json", "coverage.json"): - path = artifact_path(scan_dir, name, required=False) - if path is not None: - documents[name] = read_json_object(path) - if documents and len(documents) != 3: - raise SystemExit("Budget-exhausted scan contains an incomplete canonical scan draft.") - - if documents: - manifest = documents["scan-manifest.json"] - findings = documents["findings.json"] - coverage = documents["coverage.json"] - if not isinstance(manifest.get("scan"), dict) or not isinstance( - findings.get("findings"), list - ): - raise SystemExit("Budget-exhausted scan contains an invalid canonical scan draft.") - for key in ("surfaces", "explicitExclusions", "deferred"): - if not isinstance(coverage.get(key), list): - raise SystemExit("Budget-exhausted scan contains invalid canonical coverage.") - if manifest["scan"].get("sealedAt") is not None or manifest["scan"].get("artifacts"): - raise SystemExit("Budget-exhausted scan cannot replace an already sealed scan draft.") - else: - contract = scan_contract(scan) - target_contract = contract["target"] - target: dict[str, Any] = { - "kind": target_contract["allowedKinds"][0], - "targetId": target_contract["targetId"], - "displayName": target_contract["displayName"], - } - if scan["target_revision"] != "unversioned": - target["revision"] = scan["target_revision"] - if "requiredSnapshotDigest" in target_contract: - target["snapshotDigest"] = target_contract["requiredSnapshotDigest"] - manifest = { - "scan": { - "target": target, - "scope": {"limitations": [warning]}, - } - } - findings = {"findings": []} - coverage = { - "inventoryStrategy": ( - "scoped_path" if expected_coverage_mode(scan) == "scoped_path" else "repository" - ), - "surfaces": [], - "explicitExclusions": [], - "deferred": [], - } + *, + before_selection: bool = False, + accepted_result: dict[str, Any] | None = None, + unmerged_workers: list[sqlite3.Row] = (), + draft_documents: tuple[dict[str, Any], dict[str, Any], dict[str, Any]] | None = None, + accepted_findings: bool = False, +) -> PreparedScanFinalization: + documents = saved_results.budget_exhausted_draft( + _WORKBENCH_DB_CONTEXT, + scan, + scan_dir, + candidates, + warning, + scan_contract(scan), + before_selection=before_selection, + accepted_result=accepted_result, + unmerged_workers=unmerged_workers, + draft_documents=draft_documents, + ) + manifest = documents[0] + sealed = manifest["scan"].get("sealedAt") is not None or bool(manifest["scan"].get("artifacts")) + return _prepare_scan_finalization( + scan_dir, + expected_coverage_mode=expected_coverage_mode(scan), + completion_binding=workbench_completion_binding(scan, now(), manifest), + draft_documents=None if sealed else documents, + finding_scope=requested_finding_scope(scan), + accepted_findings=accepted_findings, + ) - findings_by_candidate = { - candidate_id - for finding in findings["findings"] - if isinstance(finding, dict) - and isinstance(candidate_id := finding_candidate_id(finding), str) - } - existing_deferred = { - item.get("candidateId", item.get("id")) - for item in coverage["deferred"] - if isinstance(item, dict) and isinstance(item.get("candidateId", item.get("id")), str) - } - existing_surfaces = { - item.get("id") - for item in coverage["surfaces"] - if isinstance(item, dict) and isinstance(item.get("id"), str) + +def completed_scan_context( + connection: sqlite3.Connection, scan_id: str, coverage: dict[str, Any] +) -> dict[str, Any]: + context = scan_context(connection, scan_id) + context["coverageSummary"] = { + "surfaceCount": len(coverage["surfaces"]), + "deferredCount": len(coverage.get("deferred", [])), + "explicitExclusionCount": len(coverage.get("explicitExclusions", [])), } - for candidate in candidates: - candidate_id = candidate["candidate_id"] - if candidate_id in findings_by_candidate or candidate_id in existing_deferred: - continue - paths = list(dict.fromkeys(location["path"] for location in candidate["locations"])) - surface_id = f"candidate-{candidate_id}" - validation = candidate.get("validation") - validation = validation.get("disposition") if isinstance(validation, dict) else None - attack = candidate.get("attack_path") - attack = attack.get("decision") if isinstance(attack, dict) else None - disposition = ( - "needs_follow_up" - if validation == "deferred" or attack == "deferred" - else "not_applicable" - if validation == "not_applicable" - else "rejected" - if validation == "suppressed" or attack == "ignore" - else "needs_follow_up" - ) - if surface_id not in existing_surfaces: - coverage["surfaces"].append( - { - "id": surface_id, - "label": candidate["summary"], - "disposition": disposition, - "notes": candidate["evidence"], - "receiptRefs": [], - } - ) - existing_surfaces.add(surface_id) - if disposition != "needs_follow_up": - continue - coverage["deferred"].append( - { - "id": candidate_id, - "candidateId": candidate_id, - "reason": ( - "Validation was deferred because the scan reached its cost limit: " - f"{candidate['summary']}. Evidence: {candidate['evidence']}" - ), - "paths": paths, - "surfaceIds": [surface_id], - } - ) - if not any( - isinstance(item, dict) - and isinstance(reason := item.get("reason"), str) - and ( - reason == "Validation was deferred because the scan reached its cost limit." - or reason.startswith( - "Validation was deferred because the scan reached its cost limit: " - ) - ) - for item in coverage["deferred"] - ): - coverage["deferred"].append( - { - "id": "scan-cost-limit", - "reason": "Validation was deferred because the scan reached its cost limit.", - } - ) - coverage.pop("completeness", None) - for name, payload in ( - ("findings.json", findings), - ("coverage.json", coverage), - ("scan-manifest.json", manifest), - ): - try: - write_scan_local_bytes( - scan_dir, - name, - (json.dumps(payload, allow_nan=False, indent=2, sort_keys=True) + "\n").encode(), - ) - except (ContractError, OSError, TypeError, ValueError) as exc: - raise SystemExit(f"Budget-exhausted scan draft could not be saved: {exc}") from exc + return context def complete_scan_locked( @@ -1495,12 +1473,14 @@ def complete_scan_locked( *, prepare_only: bool = False, thread_id: str | None = None, + prepared: PreparedScanFinalization | None = None, turn_id: str | None = None, ) -> dict[str, Any]: scan = require_scan(connection, scan_id) if scan["status"] == "complete": + deep_scan.find_supported_deep_scan_run(connection, scan_id) return saved_results.refresh_completed_scan( - _WORKBENCH_DB_CONTEXT, connection, scan, cost_json + _WORKBENCH_DB_CONTEXT, connection, scan, cost_json, prepare_only=prepare_only ) if scan["status"] != "running": raise SystemExit("Only a running scan can be completed.") @@ -1523,15 +1503,19 @@ def add_warning() -> None: scan_dir = require_canonical_scan_directory(Path(scan["scan_dir"])) completion_timestamp = now() current_manifest_path = artifact_path(scan_dir, ARTIFACTS["manifest"], required=False) - current_manifest = None - if current_manifest_path is not None: + current_manifest = prepared[2] if prepared else None + if current_manifest_path is not None and prepared is None: current_manifest = read_json_object(current_manifest_path) already_sealed = ( - current_manifest_path is not None - and isinstance(current_manifest.get("scan"), dict) - and ( - current_manifest["scan"].get("sealedAt") is not None - or current_manifest["scan"].get("artifacts") is not None + prepared[5] + if prepared + else ( + current_manifest_path is not None + and isinstance(current_manifest.get("scan"), dict) + and ( + current_manifest["scan"].get("sealedAt") is not None + or current_manifest["scan"].get("artifacts") is not None + ) ) ) completion_binding = workbench_completion_binding(scan, completion_timestamp, current_manifest) @@ -1558,46 +1542,51 @@ def add_warning() -> None: scan["mode"] != "deep" and current_manifest_path is not None and not already_sealed ) wrote = False + accepted = False try: with ( saved_results.preserve_parent_head_on_error(scan_dir) if merge_parent_draft else nullcontext() ): - draft_documents = ( - saved_results.merge_saved_results( + if prepared is None: + draft_documents = ( + saved_results.merge_saved_results( + scan_dir, + scan["id"], + completion_binding, + connection.execute( + "SELECT * FROM deep_scan_workers WHERE scan_id = ? ORDER BY created_at, id", + (scan["id"],), + ).fetchall(), + warnings, + stopped=False, + reason="", + ) + if merge_parent_draft + else None + ) + accepted = saved_results.has_verified_selected_result( + _WORKBENCH_DB_CONTEXT, connection, scan + ) + prepared = _prepare_scan_finalization( scan_dir, - scan["id"], - completion_binding, - connection.execute( - "SELECT * FROM deep_scan_workers WHERE scan_id = ? ORDER BY created_at, id", - (scan["id"],), - ).fetchall(), - warnings, - stopped=False, - reason="", + expected_coverage_mode=expected_coverage_mode(scan), + completion_binding=completion_binding, + recover_unsealed=scan["mode"] != "deep", + allow_missing_writeups=True, + draft_documents=draft_documents, + finding_scope=requested_finding_scope(scan) if scan["mode"] == "deep" else None, + require_complete=True, + accepted_findings=accepted, ) - if merge_parent_draft - else None - ) - completion_manifest = draft_documents[0] if draft_documents else current_manifest - if ( - completion_manifest is not None - and isinstance(completion_manifest.get("scan"), dict) - and completion_manifest["scan"].get("complete") is False - ): + if prepared[2]["scan"].get("complete") is False: raise RecoverableContractError( "The latest saved scan draft is incomplete; continue the scan before completing it." ) - prepared = _prepare_scan_finalization( - scan_dir, - expected_coverage_mode=expected_coverage_mode(scan), - completion_binding=completion_binding, - recover_unsealed=scan["mode"] != "deep", - allow_missing_writeups=True, - draft_documents=draft_documents, - finding_scope=requested_finding_scope(scan) if scan["mode"] == "deep" else None, - ) + saved_results.require_selected_publication( + _WORKBENCH_DB_CONTEXT, connection, scan, prepared + ) warnings.extend(warning for warning in prepared.warnings if warning not in warnings) add_warning() if prepared.warnings: @@ -1609,14 +1598,18 @@ def add_warning() -> None: (json.dumps(warnings), scan_id), ) wrote = True - manifest, findings, _ = _write_prepared_scan_finalization( + manifest, findings, coverage = _write_prepared_scan_finalization( prepared, projection_warnings=warnings ) except ContractError as exc: - # Only a coordinator-selected parent is fenced for output-write replay. - selected_deep_parent = scan["mode"] == "deep" and deep_scan.require_deep_scan_run( - connection, scan_id - )["manifest_path"] == str(scan_dir / ARTIFACTS["manifest"]) + # A canonical parent or committed selection remains fenced for replay. + run = ( + deep_scan.require_deep_scan_run(connection, scan_id) if scan["mode"] == "deep" else None + ) + selected_deep_parent = run is not None and ( + run["manifest_path"] == str(scan_dir / ARTIFACTS["manifest"]) + or deep_scan.deep_scan_finalization_input(run) is not None + ) if (wrote and not selected_deep_parent) or ( scan["mode"] == "deep" and not wrote @@ -1643,6 +1636,7 @@ def add_warning() -> None: raise SystemExit("Only a running scan can be prepared for completion.") context = scan_context(connection, scan["id"]) context["targetWarnings"] = target_warnings + context["acceptedFindings"] = accepted return context cost_fields = scan_usage.stored_scan_cost_fields(cost_json) @@ -1672,7 +1666,7 @@ def add_warning() -> None: scan = require_scan(connection, scan["id"]) if scan["status"] == "complete": connection.commit() - return scan_context(connection, scan["id"]) + return completed_scan_context(connection, scan["id"], coverage) if scan["status"] != "running": raise SystemExit("Only a running scan can be completed.") deep_scan.require_deep_scan_ready_for_parent_completion(connection, scan) @@ -1719,7 +1713,7 @@ def add_warning() -> None: ) if updated.rowcount != 1: raise SystemExit("Only a running scan can be completed.") - context = scan_context(connection, scan["id"]) + context = completed_scan_context(connection, scan["id"], coverage) context["targetWarnings"] = target_warnings return context @@ -1929,55 +1923,6 @@ def set_scan_cost_limit(connection: sqlite3.Connection, args: argparse.Namespace return {"scanId": scan["id"], "maxCostUsd": limit} -def parse_scan_recipe(value: str, repository: Path) -> dict[str, Any]: - try: - recipe = json.loads(value, parse_constant=reject_non_finite_json) - except (TypeError, UnicodeError, ValueError) as exc: - raise SystemExit("Scan launch recipe must be a valid JSON object.") from exc - if not isinstance(recipe, dict): - raise SystemExit("Scan launch recipe must be a JSON object.") - requested_repository = recipe.get("repository") - if ( - not isinstance(requested_repository, str) - or require_target(requested_repository) != repository - ): - raise SystemExit("Scan launch recipe repository must match the scanned repository.") - if recipe.get("mode") not in {"standard", "deep"}: - raise SystemExit("Scan launch recipe mode must be standard or deep.") - if not isinstance(recipe.get("config"), dict): - raise SystemExit("Scan launch recipe config must be a JSON object.") - target = recipe.get("target") - if not isinstance(target, dict) or target.get("kind") not in { - "repository", - "paths", - "refs", - "working_tree", - }: - raise SystemExit("Scan launch recipe target must identify a supported scan target.") - paths = target.get("paths") - if not isinstance(paths, list) or not all(isinstance(path, str) for path in paths): - raise SystemExit("Scan launch recipe target paths must be an array of strings.") - if target["kind"] == "paths" and not paths: - raise SystemExit("A scoped scan launch recipe must include at least one target path.") - if target["kind"] != "paths" and paths: - raise SystemExit("Only scoped scan launch recipes can include target paths.") - for path in paths: - candidate = PurePosixPath(path) - if ( - not path - or candidate.is_absolute() - or ".." in candidate.parts - or "\\" in path - or not (repository / candidate).exists() - or not (repository / candidate).resolve().is_relative_to(repository) - ): - raise SystemExit("Scan launch recipe target paths must exist inside the repository.") - if target["kind"] in {"refs", "working_tree"}: - if not isinstance(target.get("base"), str) or not isinstance(target.get("head"), str): - raise SystemExit("Diff scan launch recipes require resolved base and head revisions.") - return recipe - - def coverage_for_comparison(scan: sqlite3.Row) -> dict[str, Any]: return saved_results.coverage_for_comparison(_WORKBENCH_DB_CONTEXT, scan) @@ -2880,8 +2825,7 @@ def scan_result( **scan_usage.stored_scan_cost_fields(scan["cost_json"]), "contract": scan_contract(scan), "continuationThreadId": scan["continuation_thread_id"], - "threadIds": scan_usage._scan_root_thread_ids(connection, scan, None), - "executionThreadIds": scan_usage._scan_execution_thread_ids(connection, scan), + **scan_usage.scan_execution_fields(connection, scan), "failureMessage": scan["failure_message"], "findings": [ finding_result(connection, scan, row, related=relations.get(row["id"], [])) @@ -3404,10 +3348,16 @@ def read_json_object(path: Path) -> dict[str, Any]: _WORKBENCH_PUBLICATION_CONTEXT = _WORKBENCH_DB_CONTEXT = SimpleNamespace(**globals()) -def main(*, before_archive: Callable[[], None] | None = None) -> None: +def main( + *, + select_finalization: bool = False, + with_execution_settings: bool = False, + release_coordinator: bool = False, + before_archive: Callable[[], None] | None = None, +) -> None: # Workbench callers send UTF-8 even when Windows uses a legacy code page. sys.stdin.reconfigure(encoding="utf-8") - args = parse_args(__doc__) + args = parse_args(__doc__, execution_settings=with_execution_settings) command_context = SimpleNamespace( **globals(), preserve_stopped_results=preserve_stopped_results_after_transition, @@ -3447,13 +3397,10 @@ def main(*, before_archive: Callable[[], None] | None = None) -> None: "create-workspace": create_workspace, "save-workspace": save_workspace, "start-scan": start_scan, - "begin-deep-scan": deep_scan.begin_deep_scan, "get-deep-scan": deep_scan.get_deep_scan, - "claim-deep-scan-coordinator": deep_scan.claim_deep_scan_coordinator, "upsert-deep-scan-worker": deep_scan.upsert_deep_scan_worker, "claim-deep-scan-dedup": deep_scan.claim_deep_scan_dedup, "commit-deep-scan-dedup": deep_scan.commit_deep_scan_dedup, - "finish-deep-scan": deep_scan.finish_deep_scan, "fail-deep-scan": deep_scan.fail_deep_scan, "record-deep-scan-publication-failure": deep_scan.record_deep_scan_publication_failure, "list-scans": scan_history.list_scans, @@ -3504,6 +3451,35 @@ def main(*, before_archive: Callable[[], None] | None = None) -> None: result = _start_prompt_driven_scan(connection, args, headless_standard=False) elif args.command == "start-headless-standard-scan": result = _start_prompt_driven_scan(connection, args, headless_standard=True) + elif args.command == "begin-deep-scan": + result = deep_scan.begin_deep_scan(connection, args) + if ( + result["deepScan"]["status"] == "succeeded" + and result["deepScan"]["finalizationInput"] is not None + ): + scan = require_scan(connection, result["deepScan"]["scanId"]) + if scan["status"] == "complete": + validate_completed_scan_artifacts(_WORKBENCH_DB_CONTEXT, scan) + elif args.command == "get-deep-scan": + result = deep_scan.get_deep_scan(connection, args) + elif args.command == "claim-deep-scan-coordinator": + result = ( + deep_scan.release_deep_scan_coordinator(connection, args) + if release_coordinator + else deep_scan.claim_deep_scan_coordinator(connection, args) + ) + elif args.command == "upsert-deep-scan-worker": + result = deep_scan.upsert_deep_scan_worker(connection, args) + elif args.command == "claim-deep-scan-dedup": + result = deep_scan.claim_deep_scan_dedup(connection, args) + elif args.command == "commit-deep-scan-dedup": + result = deep_scan.commit_deep_scan_dedup(connection, args) + elif args.command == "finish-deep-scan": + result = deep_scan.finish_deep_scan(connection, args, select_finalization) + elif args.command == "fail-deep-scan": + result = deep_scan.fail_deep_scan(connection, args) + elif args.command == "record-deep-scan-publication-failure": + result = deep_scan.record_deep_scan_publication_failure(connection, args) elif args.command == "get-scan": result = scan_context(connection, args.scan_id, args.occurrence_id) elif args.command == "rename-scan": @@ -3569,6 +3545,8 @@ def main(*, before_archive: Callable[[], None] | None = None) -> None: result = finding_workflow(connection, json.load(sys.stdin), now()) else: raise SystemExit(f"Unknown command: {args.command}") + if with_execution_settings: + deep_scan.include_execution_settings(connection, result) print(json.dumps(result, allow_nan=False, sort_keys=True)) diff --git a/plugins/codex-security/scripts/workbench_saved_results.py b/plugins/codex-security/scripts/workbench_saved_results.py index 429967a831..927e07c89e 100644 --- a/plugins/codex-security/scripts/workbench_saved_results.py +++ b/plugins/codex-security/scripts/workbench_saved_results.py @@ -10,7 +10,6 @@ import os import re import sqlite3 -import stat import sys from collections.abc import Callable, Iterator from contextlib import contextmanager @@ -20,6 +19,9 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) from finalize_scan_contract import ( ContractError, + PreparedScanFinalization, + RecoverableContractError, + _contract_json_bytes, _finding_strength, _populate_unsealed_artifact_envelope, _populate_unsealed_manifest_envelope, @@ -28,7 +30,6 @@ _read_saved_threat_model, _read_scan_local_json, _read_scan_local_json_bytes, - _read_scan_local_json_with_metadata, _recover_unsealed_findings, _remove_scan_local_file_if_exists, _require_portable_relative_path, @@ -44,13 +45,91 @@ write_threat_model_projection_if_possible, ) from workbench.saved_coverage import ( + _merge_reviewed_surface_receipts, _prepare_worker_coverage_recovery, _project_worker_coverage_row, _saved_coverage_id, ) +from workbench.saved_sources import ( + _accepted_source_digests as _accepted_source_digests, +) +from workbench.saved_sources import ( + _capture_saved_source as _capture_saved_source, +) +from workbench.saved_sources import ( + _checkpoint_head_directory as _checkpoint_head_directory, +) +from workbench.saved_sources import ( + _checkpoint_paths as _checkpoint_paths, +) +from workbench.saved_sources import ( + _children as _children, +) +from workbench.saved_sources import ( + _digest as _digest, +) +from workbench.saved_sources import ( + _encode_retained_sources as _encode_retained_sources, +) +from workbench.saved_sources import ( + _encoded as _encoded, +) +from workbench.saved_sources import ( + _freeze_source_times as _freeze_source_times, +) +from workbench.saved_sources import ( + _frozen_source_times as _frozen_source_times, +) +from workbench.saved_sources import ( + _is_source_order_snapshot as _is_source_order_snapshot, +) +from workbench.saved_sources import ( + _latest_successful_reducer as _latest_successful_reducer, +) +from workbench.saved_sources import ( + _parent_scan_draft as _parent_scan_draft, +) +from workbench.saved_sources import ( + _read_saved_parent_result as _read_saved_parent_result, +) +from workbench.saved_sources import ( + _read_saved_result as _read_saved_result, +) +from workbench.saved_sources import ( + _recovery_source_digests as _recovery_source_digests, +) +from workbench.saved_sources import ( + _retained_source_state as _retained_source_state, +) +from workbench.saved_sources import ( + _saved_result_paths as _saved_result_paths, +) +from workbench.saved_sources import ( + _saved_results_changed as _saved_results_changed, +) +from workbench.saved_sources import ( + _saved_workers as _saved_workers, +) +from workbench.saved_sources import ( + _source_digests as _source_digests, +) +from workbench.saved_sources import ( + _worker_checkpoint_head as _worker_checkpoint_head, +) +from workbench.saved_sources import ( + _worker_checkpoint_heads as _worker_checkpoint_heads, +) +from workbench.saved_sources import ( + _worker_outputs as _worker_outputs, +) from workbench_constants import PHASES +from workbench_source_coverage import retain_unmerged_review_coverage from workbench_target import committed_diff_snapshot_digest -from workbench_validation import path_within_scope +from workbench_validation import ( + has_verified_selected_result, + latest_publication_reducer, + path_within_scope, +) _PUBLISHED_OUTPUTS = ( "findings.json", @@ -97,13 +176,16 @@ def refresh_completed_scan( connection: sqlite3.Connection, scan: sqlite3.Row, cost_json: str | None, + *, + prepare_only: bool = False, ) -> dict[str, Any]: warnings = json.loads(scan["completion_warnings_json"]) scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"])) db.require_recorded_manifest_digest(scan, scan_dir) db.verify_manifest_binding(scan, db.read_json_object(scan_dir / db.ARTIFACTS["manifest"])) try: - manifest, _, _ = finalize_scan( + accepted = prepare_only and has_verified_selected_result(db, connection, scan) + manifest, _, coverage = finalize_scan( scan_dir, expected_coverage_mode=db.expected_coverage_mode(scan), projection_warnings=warnings, @@ -121,422 +203,25 @@ def refresh_completed_scan( "UPDATE scans SET completion_warnings_json = ? WHERE id = ?", (json.dumps(warnings), scan["id"]), ) - return db.scan_context(connection, scan["id"]) - - -def _encoded(value: Any) -> bytes: - return json.dumps( - value, ensure_ascii=True, allow_nan=False, sort_keys=True, separators=(",", ":") - ).encode() - - -def _digest(value: Any) -> str: - return hashlib.sha256(_encoded(value)).hexdigest() - - -def _children(scan_dir: Path, relative: str) -> list[str]: - cursor = scan_dir - for part in Path(relative).parts: - if part in {"..", "."}: - return [] - cursor = cursor / part - try: - if not stat.S_ISDIR(cursor.lstat().st_mode): - return [] - except FileNotFoundError: - return [] - return sorted(child.name for child in cursor.iterdir()) - - -def _latest_successful_reducer(workers: list[Any]) -> Any | None: - return max( - ( - worker - for worker in workers - if worker["kind"] == "dedup" - and worker["status"] == "succeeded" - and worker["result_manifest_path"] - ), - key=lambda worker: (worker["completed_at"] or "", worker["id"]), - default=None, - ) - - -def _checkpoint_paths(scan_dir: Path, directory: str) -> list[str]: - return [ - f"{directory}/{name}" - for name in _children(scan_dir, directory) - if re.fullmatch(r"[0-9a-f]{64}\.json", name) - ] - - -def _worker_outputs(scan_dir: Path, worker: Any) -> list[tuple[str, int]]: - output = Path(worker["artifact_dir"]).relative_to(scan_dir) - attempts = (output.parent if output.name == "output" else output) / "attempts" - archived = [ - ((attempts / name).as_posix(), int(name.split("-")[1])) - for name in _children(scan_dir, attempts.as_posix()) - if re.fullmatch(r"attempt-\d+", name) - ] - attempt = int(worker["attempt"] or 0) if "attempt" in worker.keys() else 0 - if worker["kind"] == "discovery" and not attempt: - attempt = max((attempt for _, attempt in archived), default=0) + 1 - return [(output.as_posix(), attempt), *archived] - - -def _saved_result_paths(scan_dir: Path, workers: list[Any]) -> Iterator[tuple[str, str | None]]: - latest_reducer = _latest_successful_reducer(workers) - yield "checkpoint-head.json", None - for directory in ("checkpoint-heads", "checkpoints"): - yield from ((path, None) for path in _checkpoint_paths(scan_dir, directory)) - for worker in workers: - if worker["kind"] not in {"dedup", "discovery"}: - continue - try: - outputs = _worker_outputs(scan_dir, worker) - except (TypeError, ValueError): - continue - for directory, _ in outputs: - if worker["kind"] == "discovery": - yield f"{directory}/checkpoint-head.json", worker["kind"] - yield from ( - (path, worker["kind"]) - for path in _checkpoint_paths(scan_dir, f"{directory}/checkpoint-heads") - ) - checkpoint_paths = _checkpoint_paths(scan_dir, f"{directory}/checkpoints") - if worker["kind"] == "discovery" or checkpoint_paths: - yield f"{directory}/result.json", worker["kind"] - yield from ((path, worker["kind"]) for path in checkpoint_paths) - if worker["result_manifest_path"] and ( - worker["kind"] == "discovery" - or (latest_reducer is not None and worker["id"] == latest_reducer["id"]) - ): - try: - yield ( - Path(worker["result_manifest_path"]).relative_to(scan_dir).as_posix(), - worker["kind"], - ) - except ValueError: - continue - - -def _checkpoint_head_directory(relative: str) -> Path | None: - path = Path(relative) - if path.name == "checkpoint-head.json": - return path.parent - if path.parent.name == "checkpoint-heads": - return path.parent.parent - return None - - -def _capture_saved_source( - scan_dir: Path, - relative: str, - scan_id: str, - *, - kind: str | None = None, - snapshot_head: bool = True, - write: bool = True, -) -> dict[str, tuple[str, int]]: - if not snapshot_head or Path(relative).name != "checkpoint-head.json": - _, digest, observed = _read_saved_result(scan_dir, relative, scan_id, kind=kind) - return {relative: (digest, observed)} - head, _, observed = _read_saved_result(scan_dir, relative, scan_id) - observation = {"checkpoint": head["checkpoint"], "observedAtNs": str(observed)} - directory = Path(relative).parent - selected = (directory / "checkpoints" / observation["checkpoint"]).as_posix() - _, selected_digest, selected_time = _read_saved_result(scan_dir, selected, scan_id) - digest = _digest(observation) - snapshot = (directory / "checkpoint-heads" / f"{digest}.json").as_posix() - # Capture the selected file even if the worker created it after directory enumeration. - if write and not (scan_dir / snapshot).exists(): - write_scan_local_bytes(scan_dir, snapshot, _encoded(observation)) - return { - snapshot: (digest, int(observation["observedAtNs"])), - selected: (selected_digest, selected_time), - } - - -def _is_source_order_snapshot(relative: str) -> bool: - path = Path(relative) - return path.parent == Path("source-order") and bool( - re.fullmatch(r"[0-9a-f]{64}\.json", path.name) - ) + context = db.completed_scan_context(connection, scan["id"], coverage) + if prepare_only: + context["acceptedFindings"] = accepted + return context -def _read_saved_result( - scan_dir: Path, relative: str, scan_id: str, *, kind: str | None = None -) -> tuple[dict[str, Any], str, int]: - draft, _, metadata = _read_scan_local_json_with_metadata( - scan_dir, relative, "Saved scan checkpoint" - ) - directory = _checkpoint_head_directory(relative) - if directory is not None: - checkpoint = draft.get("checkpoint") - if not isinstance(checkpoint, str) or not re.fullmatch(r"[0-9a-f]{64}\.json", checkpoint): - raise ContractError("checkpoint head does not name a saved checkpoint") - _read_saved_result(scan_dir, (directory / "checkpoints" / checkpoint).as_posix(), scan_id) - if Path(relative).name == "checkpoint-head.json": - return draft, _digest([draft, metadata.st_mtime_ns]), metadata.st_mtime_ns - observed = draft.get("observedAtNs") - if not isinstance(observed, str) or not re.fullmatch(r"-?[0-9]+", observed): - raise ContractError("checkpoint head has no observation time") - return draft, _digest(draft), int(observed) - if draft.get("scanId") != scan_id: - raise ContractError("checkpoint belongs to a different scan") - if not _is_source_order_snapshot(relative) and ( - not isinstance(draft.get("findings"), list) - or not isinstance(draft.get("coverage", {} if kind == "dedup" else None), dict) - ): - raise ContractError("checkpoint has no semantic findings or coverage") - return draft, _digest(draft), metadata.st_mtime_ns - - -def _frozen_source_times(scan_dir: Path, scan_id: str, sources: dict[str, str]) -> dict[str, int]: - times: dict[str, int] = {} - snapshots = [path for path in sources if _is_source_order_snapshot(path)] - for path in snapshots: - record, digest, _ = _read_saved_result(scan_dir, path, scan_id) - if digest != sources[path] or not isinstance(record.get("sources"), dict): - raise ContractError("saved source ordering changed after the scan stopped") - for relative, observation in record["sources"].items(): - if ( - not isinstance(observation, dict) - or relative not in sources - or observation.get("digest") != sources[relative] - or not isinstance(observation.get("observedAtNs"), str) - or not re.fullmatch(r"-?[0-9]+", observation["observedAtNs"]) - ): - raise ContractError("saved source ordering does not match its frozen sources") - observed = int(observation["observedAtNs"]) - if relative in times and times[relative] != observed: - raise ContractError("saved source ordering has conflicting observations") - times[relative] = observed - if snapshots and sources.keys() - set(snapshots) - times.keys(): - raise ContractError("saved source ordering is incomplete") - return times - - -def _freeze_source_times( - scan_dir: Path, scan_id: str, sources: dict[str, str], times: dict[str, int] +def validate_sealed_budget_draft( + db: Any, scan: sqlite3.Row, scan_dir: Path, manifest: dict[str, Any] ) -> None: - # Identical result rewrites must not change the order of frozen review evidence. - observations = { - path: {"digest": digest, "observedAtNs": str(times[path])} - for path, digest in sources.items() - if not _is_source_order_snapshot(path) - } - if not observations: - return - record = {"scanId": scan_id, "sources": observations} - digest = _digest(record) - path = f"source-order/{digest}.json" - if not (scan_dir / path).exists(): - write_scan_local_bytes(scan_dir, path, _encoded(record)) - if _read_saved_result(scan_dir, path, scan_id)[1] != digest: - raise ContractError("saved source ordering does not match its digest") - sources[path] = digest - - -def _parent_scan_draft( - scan_id: str, - parent_scan: dict[str, Any], - findings: dict[str, Any], - coverage: dict[str, Any], -) -> dict[str, Any]: - parent = { - "scanId": scan_id, - "findings": findings.get("findings"), - "coverage": coverage, - **{ - key: parent_scan[key] - for key in ("scope", "threatModel", "complete") - if key in parent_scan - }, - } - if not isinstance(parent["findings"], list): - raise ContractError("Saved parent draft has no findings array") - return parent - - -def _read_saved_parent_result( - scan_dir: Path, scan_id: str -) -> tuple[dict[str, Any], dict[str, Any]]: - manifest = _read_scan_local_json(scan_dir, "scan-manifest.json", "Saved parent manifest") - findings = _read_scan_local_json(scan_dir, "findings.json", "Saved parent findings") - coverage = _read_scan_local_json(scan_dir, "coverage.json", "Saved parent coverage") - parent_scan = manifest.get("scan") - if not isinstance(parent_scan, dict): - raise ContractError("Saved parent manifest has no scan object") - if (parent_scan.get("sealedAt") or parent_scan.get("artifacts")) and ( - parent_scan.get("id", scan_id) != scan_id - or findings.get("scanId", scan_id) != scan_id - or coverage.get("scanId", scan_id) != scan_id - ): - raise ContractError("Saved parent documents belong to a different scan") - return manifest, _parent_scan_draft(scan_id, parent_scan, findings, coverage) - - -def _source_digests(value: Any, error: str) -> dict[str, str]: - if not isinstance(value, dict) or not all( - isinstance(relative, str) and isinstance(digest, str) for relative, digest in value.items() - ): - raise ContractError(error) - return value - - -def _retained_source_state(value: Any) -> tuple[dict[str, str], str | None]: - if isinstance(value, dict) and isinstance(value.get("sources"), dict): - sources = _source_digests( - value["sources"], "Saved stopped-scan source digests are malformed." - ) - model_source = value.get("threatModelSource") - if not isinstance(model_source, str) or model_source not in sources: - raise ContractError("Saved stopped-scan model source is outside its checkpoint set.") - return sources, model_source - return _source_digests(value, "Saved stopped-scan source digests are malformed."), None - - -def _encode_retained_sources(sources: dict[str, str], model_source: list[str]) -> str: - state = {"sources": sources, "threatModelSource": model_source[0]} if model_source else sources - return json.dumps(state, sort_keys=True) - - -def _saved_results_changed(db: Any, connection: Any, scan: Any) -> bool: + # The seal can reach disk before parent completion commits. Validate + # it without changing bytes; the existing finalizer commits replay. try: - scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"])) - manifest_path = db.artifact_path(scan_dir, db.ARTIFACTS["manifest"], required=False) - workers = connection.execute( - "SELECT id, kind, status, completed_at, artifact_dir, result_manifest_path " - "FROM deep_scan_workers WHERE scan_id = ?", - (scan["id"],), - ).fetchall() - paths = dict(_saved_result_paths(scan_dir, workers)) - frozen_sources = scan["retained_source_digests_json"] - - def has_saved_source() -> bool: - for path in paths: - try: - _read_saved_result(scan_dir, path, scan["id"], kind=paths[path]) - return True - except (ContractError, OSError, ValueError): - continue - return False - - if manifest_path is None: - if frozen_sources is not None: - return bool(_retained_source_state(json.loads(frozen_sources))[0]) - return has_saved_source() - if scan["seal_manifest_digest"] is None: - try: - _read_saved_parent_result(scan_dir, scan["id"]) - return True - except (ContractError, OSError, ValueError): - pass - return has_saved_source() - manifest = _read_scan_local_json( - scan_dir, - manifest_path.relative_to(scan_dir).as_posix(), - "Saved scan manifest", - ) - manifest_scan = manifest.get("scan") - if not isinstance(manifest_scan, dict): - return True - published_sources = _source_digests( - manifest_scan.get("preservedSources", {}), - "Published scan source digests are malformed.", - ) - current_sources = dict(published_sources) - paths.update({path: None for path in published_sources if _is_source_order_snapshot(path)}) - for path in paths: - try: - captured = _capture_saved_source( - scan_dir, - path, - scan["id"], - kind=paths[path], - snapshot_head=path not in published_sources, - write=False, - ) - current_sources.update({path: value[0] for path, value in captured.items()}) - except (ContractError, OSError, ValueError): - continue - return current_sources != published_sources - except (ContractError, OSError, SystemExit, ValueError): - return False - - -def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[str, str], bool]: - scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"])) - frozen_sources: dict[str, str] | None = None - include_parent = True - raw_frozen_sources = scan["retained_source_digests_json"] - if raw_frozen_sources is not None: - frozen_sources, _ = _retained_source_state(json.loads(raw_frozen_sources)) - include_parent = False - - manifest_path = db.artifact_path(scan_dir, db.ARTIFACTS["manifest"], required=False) - if manifest_path is not None: - manifest = _read_scan_local_json( + _prepare_scan_finalization( scan_dir, - manifest_path.relative_to(scan_dir).as_posix(), - "Saved scan manifest", + expected_coverage_mode=db.expected_coverage_mode(scan), + completion_binding=db.workbench_completion_binding(scan, scan["started_at"], manifest), ) - manifest_scan = manifest.get("scan") - if not isinstance(manifest_scan, dict): - raise ContractError("Saved scan manifest has no scan object") - if scan["seal_manifest_digest"] is not None or ( - manifest_scan.get("sealedAt") is not None or manifest_scan.get("artifacts") is not None - ): - if "preservedSources" in manifest_scan: - published_sources = _source_digests( - manifest_scan["preservedSources"], - "Published scan source digests are malformed.", - ) - include_parent = not published_sources - if published_sources: - if frozen_sources is not None and frozen_sources != published_sources: - raise ContractError( - "Stopped scan sources changed after terminal publication." - ) - frozen_sources = published_sources - elif frozen_sources is None: - frozen_sources = {} - else: - include_parent = True - - workers = connection.execute( - "SELECT id, kind, status, completed_at, artifact_dir, result_manifest_path " - "FROM deep_scan_workers WHERE scan_id = ?", - (scan["id"],), - ).fetchall() - paths = dict(_saved_result_paths(scan_dir, workers)) - recovery_sources = dict(frozen_sources or {}) - source_times = _frozen_source_times(scan_dir, scan["id"], recovery_sources) - for relative, expected_digest in recovery_sources.items(): - try: - _, digest, observed = _read_saved_result( - scan_dir, relative, scan["id"], kind=paths.get(relative) - ) - except (ContractError, OSError, ValueError) as exc: - raise ContractError("Frozen stopped-scan checkpoint set is incomplete.") from exc - if digest != expected_digest: - raise ContractError("checkpoint changed after the scan stopped") - if not _is_source_order_snapshot(relative): - source_times.setdefault(relative, observed) - - for relative in paths.keys() - recovery_sources.keys(): - try: - captured = _capture_saved_source(scan_dir, relative, scan["id"], kind=paths[relative]) - except (ContractError, OSError, ValueError): - continue - for path, (digest, observed) in captured.items(): - if path in recovery_sources and recovery_sources[path] != digest: - raise ContractError("checkpoint changed after the scan stopped") - recovery_sources[path] = digest - source_times.setdefault(path, observed) - _freeze_source_times(scan_dir, scan["id"], recovery_sources, source_times) - return recovery_sources, include_parent + except ContractError as exc: + raise SystemExit(str(exc)) from exc def scan_results_recovery_needed(db: Any, connection: Any, scan: Any) -> bool: @@ -637,6 +322,30 @@ def _ensure_finding_identity(finding: Any, *, candidate_only: bool = False) -> N finding["identity"] = {"anchor": anchor} +def _ensure_finding_identities(findings: list[dict[str, Any]]) -> None: + def key(finding: dict[str, Any]) -> bytes: + identity = finding["identity"] + return _encoded( + [finding.get("ruleId"), identity.get("anchor"), identity.get("instance", "")] + ) + + # Reserve authored identities even when they follow a generated sibling. + reserved = { + key(finding) + for finding in findings + if isinstance(finding, dict) and isinstance(finding.get("identity"), dict) + } + for finding in findings: + if not isinstance(finding, dict) or "identity" in finding: + continue + _ensure_finding_identity(finding) + suffix = 2 + while key(finding) in reserved: + finding["identity"]["instance"] = f"saved-{suffix}" + suffix += 1 + reserved.add(key(finding)) + + def _retained_findings(finding: dict[str, Any]) -> Iterator[dict[str, Any]]: """Yield canonical and historical findings without trusting candidate IDs.""" pending = [finding] @@ -1042,7 +751,9 @@ def merge_saved_results( stopped: bool, reason: str, frozen_source_digests: dict[str, str] | None = None, + checkpoint_heads: dict[str, str] | None = None, allow_frozen_legacy_parent: bool = False, + accepted_source_digests: dict[str, str] | None = None, frozen_model_source: str | None = None, selected_model_source: list[str] | None = None, project_worker_coverage: bool = False, @@ -1053,6 +764,10 @@ def merge_saved_results( source_times = _frozen_source_times(scan_dir, scan_id, frozen_source_digests or {}) except (ContractError, OSError, ValueError) as exc: raise ContractError("Frozen stopped-scan checkpoint set is incomplete.") from exc + if checkpoint_heads is None: + checkpoint_heads = _worker_checkpoint_heads( + scan_dir, workers, scan_id, accepted_source_digests + ) parent: dict[str, Any] | None = None parent_manifest: dict[str, Any] | None = None parent_is_canonical = False @@ -1145,6 +860,7 @@ def merge_saved_results( reducer_paths.add(latest_reducer) except ValueError: warnings.append("Skipped a reducer result outside the scan directory.") + accepted_reducer = latest_reducer def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None: if worker_id is not None: @@ -1223,7 +939,11 @@ def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None for relative, worker_id in paths.items(): try: draft, digest, observed = _read_saved_result( - scan_dir, relative, scan_id, kind="dedup" if relative in reducer_paths else None + scan_dir, + relative, + scan_id, + kind="dedup" if relative in reducer_paths else None, + accepted_source_digests=accepted_source_digests, ) if frozen_source_digests is not None and frozen_source_digests[relative] != digest: raise ContractError("checkpoint changed after the scan stopped") @@ -1233,8 +953,13 @@ def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None if _checkpoint_head_directory(relative) is not None: saved_heads[relative] = draft["checkpoint"] continue - # Recovery expects coverage, but reducer results only contain findings - # and context. Add an empty value after hashing the original result. + # The host supplies reducer coverage separately from model output. + # Preserve the digest of the original accepted document. + if relative in reducer_paths: + draft = { + **draft, + "coverage": draft.get("sourceCoverage", draft.get("coverage", {})), + } sources.append((relative, {"coverage": {}, **draft}, worker_id)) except (ContractError, OSError, ValueError) as exc: if (scan_dir / relative).exists(): @@ -1275,6 +1000,22 @@ def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None source_order.update(selected_observations) drafts_by_path = {relative: draft for relative, draft, _ in sources} + if "sourceCoverage" not in drafts_by_path.get(accepted_reducer, {}): + accepted_reducer = None + accepted_coverage = drafts_by_path.get(accepted_reducer, {}).get("coverage", {}) + reviewed_attempts = { + (review.get("workerId"), review.get("attempt")) + for review in accepted_coverage.get("reviews", []) + if isinstance(review, dict) + } + reviewed_workers = { + worker["id"] + for worker in workers + if {"status", "merge_state", "attempt"}.issubset(worker.keys()) + and worker["status"] == "succeeded" + and worker["merge_state"] == "merged" + and (worker["id"], worker["attempt"]) in reviewed_attempts + } latest_reducer_key = ( (reducer["completed_at"] or "", reducer["id"], int(reducer["attempt"] or 0)) if reducer is not None and latest_reducer in drafts_by_path @@ -1312,7 +1053,11 @@ def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None parent = drafts_by_path[latest_reducer] project_worker_coverage, surface_links = _prepare_worker_coverage_recovery( - sources, parent, source_order, enabled=project_worker_coverage + sources, + parent, + source_order, + enabled=project_worker_coverage, + preserve_workers=reviewed_workers, ) # Keep the actual retained row even when recovery adds previous findings. linked_surface_sources = {id(surface) for _, _, surface, _, _ in surface_links} @@ -1427,6 +1172,7 @@ def linked_surface(surface, outputs, named_outputs): and parent_manifest["scan"].get("sealedAt") and parent_manifest["scan"].get("status") == binding["status"] and parent_manifest["scan"].get("preservedSources") == source_digests + and parent_manifest["scan"].get("preservedCheckpointHeads", {}) == checkpoint_heads and all(warning in initial_warnings for warning in warnings) ): return None @@ -1459,6 +1205,7 @@ def linked_surface(surface, outputs, named_outputs): for key in ("sealedAt", "artifacts"): manifest["scan"].pop(key, None) manifest["scan"]["preservedSources"] = source_digests + manifest["scan"]["preservedCheckpointHeads"] = checkpoint_heads # Completion follows the selected parent, including legacy terminal drafts # that omit the optional marker, rather than an older canonical manifest. if parent is not None: @@ -1536,6 +1283,18 @@ def valid_finding(value: Any) -> bool: ) return bool(document["findings"]) + all_sources = ([("parent", parent, None)] if parent else []) + sources + + def coverage_candidate( + owner: str | None, item: dict[str, Any], identity: Any = None + ) -> tuple[str | None, Any]: + provenance = item.get("provenance") + if owner is None and isinstance(provenance, dict): + return provenance.get("workerId"), provenance.get( + "candidateId", identity if identity is not None else item.get("candidateId") + ) + return owner, identity if identity is not None else item.get("candidateId") + source_order["parent"] = (0, parent_modified) deferred_rows = { relative: _deferred_rows(draft["coverage"]) for relative, draft, _ in all_sources @@ -1562,7 +1321,10 @@ def valid_finding(value: Any) -> bool: ambiguous_deferred.add((owner, identity)) by_id[identity] = row current_drafts = ([("parent", parent, None)] if parent else []) + [ - source for source in sources if source[0] in current_results | selected_observations.keys() + source + for source in sources + if source[0] in current_results | selected_observations.keys() + or source[0] == accepted_reducer ] # Generic closures belong to one logical scan or worker, just like candidates. # Keep them when recovering a terminal checkpoint without its canonical write. @@ -1682,7 +1444,8 @@ def valid_finding(value: Any) -> bool: and isinstance(item.get("candidateId"), str) and item.get("disposition") in {"rejected", "not_applicable"} ): - outcomes.append((relative, owner, item["candidateId"], item["disposition"])) + candidate_owner, candidate_id = coverage_candidate(owner, item) + outcomes.append((relative, candidate_owner, candidate_id, item["disposition"])) ordered_candidates.update( (owner, candidate_id) for relative, owner, candidate_id, _ in outcomes @@ -1817,6 +1580,7 @@ def coverage_row_is_retained( item: Any, worker_id: str | None, selection: tuple[bool, set[str], bool, bool], + relative: str, ) -> bool: selected_coverage_superseded, selected_candidates, superseded, retain_pending = selection # A selected terminal checkpoint replaces ordinary coverage, while @@ -1831,14 +1595,19 @@ def coverage_row_is_retained( ): return False # A selected outcome must retain its evidence even if its result write failed. - if superseded and not ( - isinstance(item, dict) - and ( - (field == "deferred" and retain_pending) - or ( - isinstance(item.get("candidateId"), str) - and item["candidateId"] in selected_candidates - and item.get("disposition") == resolved.get((worker_id, item["candidateId"])) + if ( + superseded + and relative != accepted_reducer + and not ( + isinstance(item, dict) + and ( + (field == "deferred" and retain_pending) + or ( + isinstance(item.get("candidateId"), str) + and item["candidateId"] in selected_candidates + and item.get("disposition") + == resolved.get((worker_id, item["candidateId"])) + ) ) ) ): @@ -1851,7 +1620,7 @@ def coverage_row_is_retained( else item.get("candidateId"), str, ) - and (worker_id, identity) in resolved + and coverage_candidate(worker_id, item, identity) in resolved and (field == "deferred" or item.get("disposition") == "needs_follow_up") ): return False @@ -1901,7 +1670,7 @@ def coverage_row_is_retained( selection = source_coverage_selection(relative, owner) for row in deferred_rows[relative]: if ( - coverage_row_is_retained("deferred", row, owner, selection) + coverage_row_is_retained("deferred", row, owner, selection, relative) and id(row) not in replaced_deferred and not ( isinstance(row, dict) @@ -1916,7 +1685,7 @@ def coverage_row_is_retained( for row in rows if isinstance(rows, list) else []: if owner is None or not isinstance(row, dict): continue - emitted = coverage_row_is_retained("surfaces", row, owner, selection) + emitted = coverage_row_is_retained("surfaces", row, owner, selection, relative) ambiguous = ( isinstance(row.get("id"), str) and (owner, row["id"]) in ambiguous_surface_ids @@ -1967,7 +1736,7 @@ def coverage_row_is_retained( receipt_warnings=warnings, fallback_surfaces=fallback_surfaces, surface_is_retained=lambda relative, owner, row: coverage_row_is_retained( - "surfaces", row, owner, source_coverage_selection(relative, owner) + "surfaces", row, owner, source_coverage_selection(relative, owner), relative ), eligible_receipt_sources={ relative @@ -1997,6 +1766,8 @@ def coverage_row_is_retained( # Reopened work survives a superseded checkpoint, but current candidate # outcomes still apply. Parent closures cannot remove another worker's row. for owner, item in reopened_rows: + if owner in reviewed_workers: + continue if (identity := _deferred_candidate_id(item, owner, ambiguous_deferred)) is not None and ( owner, identity, @@ -2009,7 +1780,8 @@ def coverage_row_is_retained( surfaces = draft["coverage"].get("surfaces", []) for surface in surfaces if isinstance(surfaces, list) else []: if ( - isinstance(surface, dict) + owner not in reviewed_workers + and isinstance(surface, dict) and isinstance(surface.get("id"), str) and (owner, surface["id"]) in ambiguous_surface_ids and surface.get("disposition") == "needs_follow_up" @@ -2204,10 +1976,21 @@ def coverage_row_is_retained( continue finding_positions[key] = len(findings) findings.append(finding) - if superseded and not selected_candidates and not retain_pending: + if worker_id in reviewed_workers or ( + superseded + and relative != accepted_reducer + and not selected_candidates + and not retain_pending + ): continue - for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"): - if superseded and field not in {"surfaces", "explicitExclusions", "deferred"}: + for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions", "reviews"): + if ( + superseded + and relative != accepted_reducer + and field not in {"surfaces", "explicitExclusions", "deferred"} + ): + continue + if field == "reviews" and field not in draft["coverage"]: continue items = draft["coverage"].get(field, []) if field == "deferred": @@ -2221,7 +2004,7 @@ def coverage_row_is_retained( continue for item in items: source_item = item - if not coverage_row_is_retained(field, item, worker_id, selection): + if not coverage_row_is_retained(field, item, worker_id, selection, relative): continue if id(item) in replaced_rows.get(field, ()): continue @@ -2277,6 +2060,10 @@ def coverage_row_is_retained( if field == "surfaces" and id(source_item) in linked_surface_sources: linked_surface_outputs[id(source_item)] = retained + _merge_reviewed_surface_receipts( + coverage, worker_row_owners, reviewed_workers, reviewed_attempts + ) + identities: dict[str, str] = {} for finding in findings: if not valid_finding(finding): @@ -2399,6 +2186,218 @@ def coverage_row_is_retained( return manifest, {"findings": findings}, coverage +def budget_exhausted_draft( + db: Any, + scan: sqlite3.Row, + scan_dir: Path, + candidates: list[dict[str, Any]], + warning: str, + contract: dict[str, Any], + *, + before_selection: bool = False, + accepted_result: dict[str, Any] | None = None, + unmerged_workers: list[sqlite3.Row] = (), + draft_documents: tuple[dict[str, Any], dict[str, Any], dict[str, Any]] | None = None, +) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]: + names = ("scan-manifest.json", "findings.json", "coverage.json") + documents = dict(zip(names, copy.deepcopy(draft_documents))) if draft_documents else {} + if draft_documents is None: + for name in names: + path = db.artifact_path(scan_dir, name, required=False) + if path is not None: + documents[name] = db.read_json_object(path) + interrupted_documents = {} + if documents and len(documents) != 3: + if not before_selection or set(documents) not in ( + {"findings.json"}, + {"findings.json", "coverage.json"}, + ): + raise SystemExit("Budget-exhausted scan contains an incomplete canonical scan draft.") + interrupted_documents = documents + documents = {} + + if documents: + manifest = documents["scan-manifest.json"] + findings = documents["findings.json"] + coverage = documents["coverage.json"] + if not isinstance(manifest.get("scan"), dict) or not isinstance( + findings.get("findings"), list + ): + raise SystemExit("Budget-exhausted scan contains an invalid canonical scan draft.") + for key in ("surfaces", "explicitExclusions", "deferred"): + if not isinstance(coverage.get(key), list): + raise SystemExit("Budget-exhausted scan contains invalid canonical coverage.") + if manifest["scan"].get("sealedAt") is not None or manifest["scan"].get("artifacts"): + if before_selection: + raise SystemExit( + "Budget-exhausted scan cannot replace an already sealed scan draft." + ) + # Completion verifies sealed bytes directly from disk. + return manifest, findings, coverage + else: + target_contract = contract["target"] + target: dict[str, Any] = { + "kind": target_contract["allowedKinds"][0], + "targetId": target_contract["targetId"], + "displayName": target_contract["displayName"], + } + if scan["target_revision"] != "unversioned": + target["revision"] = scan["target_revision"] + if "requiredSnapshotDigest" in target_contract: + target["snapshotDigest"] = target_contract["requiredSnapshotDigest"] + manifest = { + "scan": { + "target": target, + "scope": {"limitations": [warning]}, + } + } + findings = {"findings": []} + coverage = { + "inventoryStrategy": ( + "scoped_path" if db.expected_coverage_mode(scan) == "scoped_path" else "repository" + ), + "surfaces": [], + "explicitExclusions": [], + "deferred": [], + } + + if before_selection: + # Unmerged discoveries remain in their accepted artifacts. Only the + # committed reducer may contribute findings to this partial result. + findings = { + "findings": copy.deepcopy(accepted_result["findings"]) if accepted_result else [] + } + _ensure_finding_identities(findings["findings"]) + if accepted_result is not None: + coverage = accepted_result["sourceCoverage"] + if "threatModel" in accepted_result: + manifest["scan"]["threatModel"] = accepted_result["threatModel"] + if "scope" in accepted_result: + manifest["scan"]["scope"].update(copy.deepcopy(accepted_result["scope"])) + limitations = manifest["scan"]["scope"].setdefault("limitations", []) + if warning not in limitations: + limitations.append(warning) + + for worker in unmerged_workers: + retain_unmerged_budget_coverage(scan, scan_dir, coverage, worker) + + findings_by_candidate = { + candidate_id + for finding in findings["findings"] + if isinstance(finding, dict) + and isinstance(candidate_id := finding_candidate_id(finding), str) + } + existing_deferred = { + item.get("candidateId", item.get("id")) + for item in coverage["deferred"] + if isinstance(item, dict) and isinstance(item.get("candidateId", item.get("id")), str) + } + existing_surfaces = { + item.get("id") + for item in coverage["surfaces"] + if isinstance(item, dict) and isinstance(item.get("id"), str) + } + for candidate in candidates: + candidate_id = candidate["candidate_id"] + if candidate_id in findings_by_candidate or candidate_id in existing_deferred: + continue + paths = list(dict.fromkeys(location["path"] for location in candidate["locations"])) + surface_id = f"candidate-{candidate_id}" + validation = candidate.get("validation") + validation = validation.get("disposition") if isinstance(validation, dict) else None + attack = candidate.get("attack_path") + attack = attack.get("decision") if isinstance(attack, dict) else None + disposition = ( + "needs_follow_up" + if validation == "deferred" or attack == "deferred" + else "not_applicable" + if validation == "not_applicable" + else "rejected" + if validation == "suppressed" or attack == "ignore" + else "needs_follow_up" + ) + if surface_id not in existing_surfaces: + coverage["surfaces"].append( + { + "id": surface_id, + "label": candidate["summary"], + "disposition": disposition, + "notes": candidate["evidence"], + "receiptRefs": [], + } + ) + existing_surfaces.add(surface_id) + if disposition != "needs_follow_up": + continue + coverage["deferred"].append( + { + "id": candidate_id, + "candidateId": candidate_id, + "reason": ( + "Validation was deferred because the scan reached its cost limit: " + f"{candidate['summary']}. Evidence: {candidate['evidence']}" + ), + "paths": paths, + "surfaceIds": [surface_id], + } + ) + if not any( + isinstance(item, dict) + and isinstance(reason := item.get("reason"), str) + and ( + reason == "Validation was deferred because the scan reached its cost limit." + or reason.startswith( + "Validation was deferred because the scan reached its cost limit: " + ) + ) + for item in coverage["deferred"] + ): + coverage["deferred"].append( + { + "id": "scan-cost-limit", + "reason": "Validation was deferred because the scan reached its cost limit.", + } + ) + coverage.pop("completeness", None) + outputs = ( + ("findings.json", findings), + ("coverage.json", coverage), + ("scan-manifest.json", manifest), + ) + # Process death can leave a prefix of these derived writes before SQLite + # commits. Recover only bytes whose documents match the accepted inputs. + if any( + interrupted_documents[name] != payload + for name, payload in outputs + if name in interrupted_documents + ): + raise SystemExit("Budget-exhausted scan contains an incomplete canonical scan draft.") + for name, payload in outputs: + try: + write_scan_local_bytes( + scan_dir, + name, + (json.dumps(payload, allow_nan=False, indent=2, sort_keys=True) + "\n").encode(), + ) + except (ContractError, OSError, TypeError, ValueError) as exc: + raise SystemExit(f"Budget-exhausted scan draft could not be saved: {exc}") from exc + return manifest, findings, coverage + + +def retain_unmerged_budget_coverage( + scan: sqlite3.Row, scan_dir: Path, coverage: dict[str, Any], worker: sqlite3.Row +) -> None: + """Keep each unmerged review's obligations; its findings remain evidence only.""" + relative = Path(worker["accepted_result_path"]).relative_to(scan_dir).as_posix() + draft, _, _ = _read_saved_result( + scan_dir, + relative, + scan["id"], + accepted_source_digests={worker["accepted_result_path"]: worker["accepted_result_sha256"]}, + ) + retain_unmerged_review_coverage(scan_dir, coverage, worker, draft) + + def coverage_for_comparison(db: Any, scan: Any) -> dict[str, Any]: if scan["seal_manifest_digest"] is None: raise SystemExit("Only sealed scans can be compared.") @@ -2522,6 +2521,7 @@ def preserve_scan_results_locked( scan_id: str, *, recovery_source_digests: dict[str, str] | None = None, + recovery_checkpoint_heads: dict[str, str] | None = None, include_parent_with_recovery: bool = False, ) -> bool: """Publish or verify retained terminal results through the workbench host.""" @@ -2529,6 +2529,7 @@ def preserve_scan_results_locked( if scan["status"] != "failed": return False frozen_source_digests: dict[str, str] | None = None + checkpoint_heads = recovery_checkpoint_heads model_source: list[str] = [] saved_model_source: str | None = None raw_frozen_sources = scan["retained_source_digests_json"] @@ -2540,10 +2541,17 @@ def preserve_scan_results_locked( model_source.append(saved_model_source) if recovery_source_digests is not None: frozen_source_digests = recovery_source_digests + elif raw_frozen_sources is not None: + raw_checkpoint_heads = scan["retained_checkpoint_heads_json"] + checkpoint_heads = ( + json.loads(raw_checkpoint_heads) if raw_checkpoint_heads is not None else {} + ) scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"])) deep_run = connection.execute( - "SELECT status FROM deep_scan_runs WHERE scan_id = ?", (scan_id,) + "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan_id,) ).fetchone() + if deep_run is not None: + db.deep_scan.require_supported_deep_scan(deep_run) outcome = ( "canceled" if scan["canceled_at"] @@ -2591,11 +2599,15 @@ def record_publication(manifest: dict[str, Any], findings: dict[str, Any]) -> No db.index_findings(connection, scan_id, findings, scan["completed_at"]) connection.execute( "UPDATE scans SET seal_manifest_digest = ?, retained_source_digests_json = ?, " + "retained_checkpoint_heads_json = ?, " "completion_warnings_json = ?, " "updated_at = ? WHERE id = ? AND status = 'failed'", ( digest, _encode_retained_sources(retained_sources, model_source), + json.dumps( + manifest["scan"].get("preservedCheckpointHeads", {}), sort_keys=True + ), json.dumps(list(dict.fromkeys(warnings))), timestamp, scan_id, @@ -2625,11 +2637,13 @@ def record_publication(manifest: dict[str, Any], findings: dict[str, Any]) -> No db.verify_manifest_binding(scan, existing) if existing_scan.get("status") == outcome: existing_sources = existing_scan.get("preservedSources") + existing_heads = existing_scan.get("preservedCheckpointHeads", {}) if frozen_source_digests is None: frozen_source_digests = _source_digests( existing_sources, "Stopped scan source digests could not be frozen." ) - if existing_sources == frozen_source_digests: + checkpoint_heads = existing_heads + if existing_sources == frozen_source_digests and existing_heads == checkpoint_heads: if ( raw_frozen_sources is not None and scan["seal_manifest_digest"] is not None @@ -2651,10 +2665,7 @@ def record_publication(manifest: dict[str, Any], findings: dict[str, Any]) -> No scan_dir, scan_id, binding, - connection.execute( - "SELECT * FROM deep_scan_workers WHERE scan_id = ? ORDER BY created_at, id", - (scan_id,), - ).fetchall(), + _saved_workers(connection, scan_id), warnings, stopped=True, reason=( @@ -2662,6 +2673,8 @@ def record_publication(manifest: dict[str, Any], findings: dict[str, Any]) -> No f"{scan['failure_message'] or ''}" ).strip(), frozen_source_digests=frozen_source_digests, + checkpoint_heads=checkpoint_heads, + accepted_source_digests=_accepted_source_digests(connection, scan_id), frozen_model_source=model_source[0] if model_source else None, selected_model_source=model_source, project_worker_coverage=scan["mode"] == "deep", @@ -2695,10 +2708,13 @@ def record_publication(manifest: dict[str, Any], findings: dict[str, Any]) -> No ) with connection: connection.execute( - "UPDATE scans SET retained_source_digests_json = ? " + "UPDATE scans SET retained_source_digests_json = ?, retained_checkpoint_heads_json = ? " "WHERE id = ? AND retained_source_digests_json IS ?", ( _encode_retained_sources(retained_sources, model_source), + json.dumps( + documents[0]["scan"].get("preservedCheckpointHeads", {}), sort_keys=True + ), scan_id, raw_frozen_sources, ), @@ -2733,12 +2749,15 @@ def recover_scan_results(db: Any, connection: Any, args: Any) -> dict[str, Any]: raise SystemExit("Only a stopped scan can recover terminal results.") if scan["canceled_at"] is not None: raise SystemExit("Canceled scans cannot recover terminal results.") - recovery_source_digests, include_parent = _recovery_source_digests(db, connection, scan) + recovery_source_digests, include_parent, checkpoint_heads = _recovery_source_digests( + db, connection, scan + ) if not preserve_scan_results_locked( db, connection, scan_id, recovery_source_digests=recovery_source_digests, + recovery_checkpoint_heads=checkpoint_heads, include_parent_with_recovery=include_parent, ): raise SystemExit("No saved stopped-scan results were available to recover.") @@ -2823,64 +2842,160 @@ def save_scan_artifact(db: Any, connection: Any, args: Any) -> dict[str, Any]: return {"scanId": scan_id, "path": str(scan_dir / output)} +def _read_staged_scan_draft(scan_dir: Path, draft_path: str) -> dict[str, Any]: + try: + relative = Path(draft_path).relative_to(scan_dir).as_posix() + except ValueError as exc: + raise SystemExit("Scan draft must be inside the registered scan drafts directory.") from exc + if not re.fullmatch(r"drafts/[0-9a-fA-F-]+\.json", relative): + raise SystemExit("Scan draft must be inside the registered scan drafts directory.") + return _read_scan_local_json(scan_dir, relative, "Staged scan draft") + + +def _selected_publication_digest(prepared: Any, legacy_completeness: str | None = None) -> str: + # Completion time is chosen at sealing, after publication. Everything else + # must remain the host projection of the same accepted aggregate. + manifest = copy.deepcopy(prepared[2]) + for field in ("completedAt", "sealedAt"): + manifest["scan"].pop(field, None) + coverage = prepared.coverage + if legacy_completeness is not None: + # Reconstruct the historical projection and its coverage artifact binding. + coverage = {**coverage, "completeness": legacy_completeness} + coverage_digest = hashlib.sha256( + _contract_json_bytes("coverage.json", coverage) + ).hexdigest() + for artifact in manifest["scan"]["artifacts"]: + if artifact["path"] == manifest["scan"]["coverageRef"]: + artifact["sha256"] = coverage_digest + return _digest([manifest, prepared.findings, coverage]) + + +def _require_selected_result(scan: Any, selection: dict[str, Any]) -> None: + relative = selection["resultPath"] + if relative is not None: + _read_saved_result( + Path(scan["scan_dir"]), + relative, + scan["id"], + kind="dedup", + accepted_source_digests={ + str(Path(scan["scan_dir"]) / relative): selection["resultSha256"] + }, + ) + + +def record_selected_publication( + db: Any, connection: Any, scan: Any, prepared: PreparedScanFinalization +) -> None: + run = db.deep_scan.require_deep_scan_run(connection, scan["id"]) + selection = db.deep_scan.deep_scan_finalization_input(run) + if selection is None: + return + _require_selected_result(scan, selection) + selection["publicationSha256"] = _selected_publication_digest(prepared) + with connection: + connection.execute( + "UPDATE deep_scan_runs SET finalization_input_json = ? WHERE scan_id = ?", + (json.dumps(selection), scan["id"]), + ) + + +def require_selected_publication(db: Any, connection: Any, scan: Any, prepared: Any) -> None: + if scan["mode"] != "deep": + return + run = db.deep_scan.require_deep_scan_run(connection, scan["id"]) + selection = db.deep_scan.deep_scan_finalization_input(run) + if selection is None or "publicationSha256" not in selection: + return + try: + _require_selected_result(scan, selection) + except ContractError as exc: + raise RecoverableContractError(str(exc)) from exc + digest = selection.get("publicationSha256") + if digest == _selected_publication_digest(prepared): + return + # Old receipts still bind every field, including on normalized seal replay. + if "completeness" not in prepared.coverage and any( + digest == _selected_publication_digest(prepared, completeness) + for completeness in ("complete", "partial", "unknown") + ): + return + raise RecoverableContractError( + "The selected Deep Scan publication changed or is missing; " + "republish its accepted result before completing the scan." + ) + + def _require_current_deep_publication( db: Any, connection: Any, scan: Any, draft: dict[str, Any] -) -> None: +) -> bool: scan_id = scan["id"] run = connection.execute( "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan_id,) ).fetchone() publication = draft.get("deepScanPublication") + selection = db.deep_scan.deep_scan_finalization_input(run) if run is not None else None if ( - publication is None + selection is None + and publication is None and draft["manifest"]["scan"].get("complete") is False and (run is None or run["status"] == "running") ): - return + return False if run is None: run = db.deep_scan.require_deep_scan_run(connection, scan_id) # Finished discovery-only runs hand remaining publication back to the parent. if ( - publication is None + selection is None + and publication is None and run["status"] != "running" and run["manifest_path"] != str(Path(scan["scan_dir"]) / "scan-manifest.json") ): - return + return False db.deep_scan.require_current_coordinator( run, argparse.Namespace( coordinator_generation=publication.get("coordinatorGeneration") if publication else None ), ) - # Once finish selects the canonical parent, only completion may replay it. - if publication is None: - if run["status"] != "running" and run["manifest_path"] == str( - Path(scan["scan_dir"]) / "scan-manifest.json" - ): - raise SystemExit("Deep Scan is terminal; drafts cannot replace its publication.") - return - - # Match the durable reducer sequence used by coordinator recovery. - def reducer_order(worker: Any) -> tuple[int, str]: - match = re.fullmatch(r"dedup-([0-9]+)", Path(worker["prompt_path"]).parent.name) - return (int(match[1]) if match else 0, worker["id"]) - - reducer = max( - ( - worker - for worker in connection.execute( - "SELECT * FROM deep_scan_workers WHERE scan_id = ?", (scan_id,) + accepted_findings = False + if selection is not None: + if publication is None: + raise SystemExit("Deep Scan publication requires its committed selection.") + scan = db.require_scan(connection, scan_id) + accepted_findings = has_verified_selected_result(db, connection, scan) + if "publicationSha256" in selection: + prepared = _prepare_scan_finalization( + Path(scan["scan_dir"]), + expected_coverage_mode=db.expected_coverage_mode(scan), + completion_binding=db.workbench_completion_binding(scan, db.now()), + draft_documents=(draft["manifest"], draft["findings"], draft["coverage"]), + accepted_findings=accepted_findings, ) - if worker["kind"] == "dedup" - and worker["status"] == "succeeded" - and worker["result_manifest_path"] - ), - key=reducer_order, - default=None, - ) - selected_result = reducer["result_manifest_path"] if reducer is not None else None + require_selected_publication(db, connection, scan, prepared) + selected_result = ( + str(Path(scan["scan_dir"]) / selection["resultPath"]) + if selection["resultPath"] is not None + else None + ) + else: + # Generation-one runs predate host publication metadata. Keep their existing + # draft path; adopted coordinators must carry their generation and selection. + if publication is None: + if run["status"] != "running" and run["manifest_path"] == str( + Path(scan["scan_dir"]) / "scan-manifest.json" + ): + raise SystemExit("Deep Scan is terminal; drafts cannot replace its publication.") + return False + + reducer = latest_publication_reducer(db, connection, scan_id) + selected_result = reducer["result_manifest_path"] if reducer is not None else None if publication["resultPath"] != selected_result: raise SystemExit("Deep Scan aggregate belongs to a superseded publication selection.") + if selection is None: + accepted_findings = has_verified_selected_result(db, connection, scan, reducer=reducer) + return accepted_findings def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]: @@ -2902,15 +3017,7 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]: raise SystemExit( "scan_draft_conflict: canonical scan results changed; reconcile the saved checkpoint again." ) - try: - relative = Path(args.draft_path).relative_to(scan_dir).as_posix() - except ValueError as exc: - raise SystemExit( - "Scan draft must be inside the registered scan drafts directory." - ) from exc - if not re.fullmatch(r"drafts/[0-9a-fA-F-]+\.json", relative): - raise SystemExit("Scan draft must be inside the registered scan drafts directory.") - draft = _read_scan_local_json(scan_dir, relative, "Staged scan draft") + draft = _read_staged_scan_draft(scan_dir, args.draft_path) manifest, findings, coverage = draft["manifest"], draft["findings"], draft["coverage"] binding = db.workbench_completion_binding(scan, db.now()) # Save scan IDs without sealing the draft. @@ -2946,8 +3053,11 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]: ).fetchone() is not None ) + run = None + accepted_findings = False if scan["mode"] == "deep" and not checkpoint_only: - _require_current_deep_publication(db, connection, scan, draft) + accepted_findings = _require_current_deep_publication(db, connection, scan, draft) + run = db.deep_scan.find_supported_deep_scan_run(connection, scan_id) if checkpoint_contents is not None: checkpoint_digest = hashlib.sha256(checkpoint_contents).hexdigest() write_scan_local_bytes( @@ -2987,6 +3097,15 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]: (str(scan_dir / "scan-manifest.json"), scan_id), ) model_warning = write_threat_model_projection_if_possible(scan_dir, manifest) + if run is not None and db.deep_scan.deep_scan_finalization_input(run) is not None: + prepared = _prepare_scan_finalization( + scan_dir, + expected_coverage_mode=db.expected_coverage_mode(scan), + completion_binding=binding, + draft_documents=(manifest, findings, coverage), + accepted_findings=accepted_findings, + ) + record_selected_publication(db, connection, scan, prepared) # Accepted Standard drafts are evidence of review or report assembly, # even when the parent omitted its explicit progress call. model_only_checkpoint = ( diff --git a/plugins/codex-security/scripts/workbench_scan_history.py b/plugins/codex-security/scripts/workbench_scan_history.py index 11e705d468..33b1fe15f8 100644 --- a/plugins/codex-security/scripts/workbench_scan_history.py +++ b/plugins/codex-security/scripts/workbench_scan_history.py @@ -79,26 +79,40 @@ def cli_scan_resume( ): raise SystemExit("Resume requires the original owning CLI session.") run = connection.execute( - "SELECT status, cancel_requested FROM deep_scan_runs WHERE scan_id = ?", (scan["id"],) + "SELECT status, cancel_requested, finalization_input_json FROM deep_scan_runs " + "WHERE scan_id = ?", + (scan["id"],), ).fetchone() if run is not None and ( - run["status"] not in {"running", "succeeded"} or run["cancel_requested"] + run["status"] not in {"running", "succeeded"} + or ( + run["cancel_requested"] + and not (run["status"] == "succeeded" and run["finalization_input_json"] is not None) + ) ): raise SystemExit("This Deep Scan has stopped and cannot resume.") - try: - repository = require_scan_target_identity(scan) - except SystemExit as exc: - raise SystemExit( - "Cannot resume: the original checkout is missing or was replaced." - ) from exc - if scan_target_identity(repository, None) != ( - scan["target_revision"], - scan["target_snapshot_digest"], - scan["target_device"], - scan["target_inode"], - ): - raise SystemExit("Cannot resume: the original checkout revision or contents changed.") - recipe = wb.parse_scan_recipe(scan["recipe_json"], repository) + selected = run is not None and run["finalization_input_json"] is not None + if selected: + # Publication uses the frozen selection and saved contract. The normal + # completion command reports checkout drift without restarting analysis. + repository = Path(scan["target_path"]) + else: + try: + repository = require_scan_target_identity(scan) + except SystemExit as exc: + raise SystemExit( + "Cannot resume: the original checkout is missing or was replaced." + ) from exc + if scan_target_identity(repository, None) != ( + scan["target_revision"], + scan["target_snapshot_digest"], + scan["target_device"], + scan["target_inode"], + ): + raise SystemExit("Cannot resume: the original checkout revision or contents changed.") + recipe = wb.parse_scan_recipe( + scan["recipe_json"], repository, validate_current_target=not selected + ) scan_dir = wb.require_canonical_scan_directory(Path(scan["scan_dir"])) progress = connection.execute( "SELECT scope_file_count FROM scan_progress WHERE scan_id = ?", (scan["id"],) @@ -109,6 +123,7 @@ def cli_scan_resume( "scanDir": str(scan_dir), "scanId": scan["id"], "scopeFileCount": progress["scope_file_count"], + "selectedFinalization": selected, "startedAt": scan["started_at"], "targetId": scan["target_id"], "targetRevision": scan["target_revision"], diff --git a/plugins/codex-security/scripts/workbench_scan_usage.py b/plugins/codex-security/scripts/workbench_scan_usage.py index 301ff9cbef..602cd5f311 100644 --- a/plugins/codex-security/scripts/workbench_scan_usage.py +++ b/plugins/codex-security/scripts/workbench_scan_usage.py @@ -10,6 +10,7 @@ import sqlite3 import sys import uuid +from collections import deque from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path @@ -91,14 +92,64 @@ def collect_scan_usage( ) -> dict[str, Any]: """Count attributable receipts and flag observable gaps inside the scan window.""" - roots = _scan_root_thread_ids(connection, scan, thread_id) + completion_thread_id = thread_id + attribution = scan_execution_attribution(connection, scan) + if attribution and attribution.get("legacy"): + attribution = None + roots = ( + list( + dict.fromkeys( + [ + *( + [attribution["owner"]["threadId"]] + if attribution["owner"].get("threadId") + else [] + ), + *attribution["executionThreadIds"], + ] + ) + ) + if attribution + else _scan_root_thread_ids(connection, scan, thread_id) + ) parent_ids = set(_scan_root_thread_ids(connection, scan, thread_id, include_workers=False)) worker_directories = _scan_worker_directories(connection, scan) if not roots and not worker_directories: return _unavailable_usage("scan_thread_unavailable") - state_database = _codex_state_database() - if state_database is None: + warnings: set[str] = set() + current_database = _codex_state_database() + worker_codex_home = None + if scan["mode"] == "deep": + # Only the workbench-bound original settings select a worker log home. + from deep_scan_workbench import ( + recorded_deep_scan_execution_settings, + validate_deep_scan_execution_settings, + ) + + run = connection.execute( + "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan["id"],) + ).fetchone() + saved = recorded_deep_scan_execution_settings(run) if run is not None else None + if saved is not None: + try: + settings = validate_deep_scan_execution_settings(saved) + worker_codex_home = Path(settings["codexHome"]) + except SystemExit: + # Legacy scans may have no recorded home. Keep usage best effort. + pass + groups = [(current_database, roots)] + worker_roots: set[str] = set() + if worker_codex_home is not None: + worker_roots = set( + _scan_root_thread_ids(connection, scan, None, include_owner_threads=False) + ) + # Workers retain their Codex home, but inherit an explicit current + # SQLite home. Their earlier and resumed sessions can be in either index. + worker_database = _codex_state_database(worker_codex_home) + if worker_database != current_database: + groups.append((worker_database, [root for root in roots if root in worker_roots])) + if not any(database is not None for database, _ in groups) and not worker_roots: return _unavailable_usage("codex_state_unavailable") started_at = _timestamp(scan["started_at"]) @@ -107,21 +158,61 @@ def collect_scan_usage( if started_at is None: return _unavailable_usage("scan_window_unavailable") - warnings: set[str] = set() - try: - sessions, missing_thread_ids = _discover_rollout_sessions( - state_database, - roots, - warnings, - worker_directories=worker_directories, - started_at=started_at, - completed_at=stopped_at, + execution_thread_ids = set(attribution["executionThreadIds"]) if attribution else set() + sessions: dict[str, list[RolloutSession]] = {} + missing_thread_ids: set[str] = set() + seen_thread_ids: set[str] = set() + for state_database, group_roots in groups: + if not group_roots and not worker_directories: + continue + try: + if state_database is None: + raise FileNotFoundError("Codex state is unavailable") + discovered, missing = _discover_rollout_sessions( + state_database, + group_roots, + warnings, + descendant_roots=set(attribution["executionThreadIds"]) if attribution else None, + worker_directories=worker_directories, + started_at=started_at, + completed_at=stopped_at, + ) + except (OSError, sqlite3.Error, ValueError): + warnings.add("codex_state_unavailable") + missing_thread_ids.update(group_roots) + continue + missing_thread_ids.update(missing) + for session in discovered: + if session.parent_thread_id is None and session.thread_id not in roots: + execution_thread_ids.add(session.thread_id) + copies = sessions.setdefault(session.thread_id, []) + if session not in copies: + copies.append(session) + seen_thread_ids.add(session.thread_id) + + if worker_codex_home is not None and worker_roots: + # An external SQLite location can change on recovery. Native rollouts + # still live in the recorded worker home; they retain their lineage. + for session in _discover_recorded_worker_sessions(worker_codex_home, worker_roots): + copies = sessions.setdefault(session.thread_id, []) + if session not in copies: + copies.append(session) + seen_thread_ids.add(session.thread_id) + + # Absence from one known index is not missing usage when another has it. + missing_thread_ids.difference_update(seen_thread_ids) + if not missing_thread_ids: + warnings.difference_update( + {"scan_root_unavailable", "codex_state_unavailable", "rollout_unavailable"} ) - except (OSError, sqlite3.Error, ValueError): - return _unavailable_usage("codex_state_unavailable") if not sessions: - return _unavailable_usage("scan_thread_unavailable", warnings=warnings) + return _unavailable_usage( + "codex_state_unavailable" + if "codex_state_unavailable" in warnings + else "scan_thread_unavailable", + warnings=warnings, + ) by_role = {role: {**_empty_token_usage(), "threadCount": 0} for role in ("parent", "workers")} rate_limits: dict[str, dict[str, Any]] = {} @@ -129,7 +220,38 @@ def collect_scan_usage( accepted_thread_ids: set[str] = set() measured_thread_ids: set[str] = set() excluded_thread_ids: set[str] = set() - for session in sessions: + model_usage: dict[str | None, dict[str, int]] = {} + children: dict[str | None, list[str]] = {} + for thread_id, copies in sessions.items(): + children.setdefault(copies[0].parent_thread_id, []).append(thread_id) + pending = deque( + thread_id + for thread_id, copies in sessions.items() + if copies[0].parent_thread_id not in sessions + ) + ordered_thread_ids: list[str] = [] + while pending: + thread_id = pending.popleft() + ordered_thread_ids.append(thread_id) + pending.extend(children.get(thread_id, [])) + # Keep unresolved lineage in the existing accounting checks as well. + ordered = set(ordered_thread_ids) + ordered_thread_ids.extend(thread_id for thread_id in sessions if thread_id not in ordered) + for thread_id in ordered_thread_ids: + copies = sessions[thread_id] + session = copies[0] + owner_turn_id = None + if ( + attribution + and session.thread_id not in execution_thread_ids + and session.parent_thread_id is None + ): + owner = attribution["owner"] + if session.thread_id != owner.get("threadId") or not owner.get("turnId"): + missing_thread_ids.add(session.thread_id) + warnings.add("scan_owner_turn_unavailable") + continue + owner_turn_id = owner["turnId"] if session.parent_thread_id in excluded_thread_ids: excluded_thread_ids.add(session.thread_id) continue @@ -141,11 +263,15 @@ def collect_scan_usage( warnings.add("thread_lineage_incomplete") continue try: - usage = _read_rollout_usage( - session, + usage = _read_rollout_copies_usage( + copies, started_at=started_at, completed_at=stopped_at, - completed_turn_id=completed_turn_id if session.thread_id == thread_id else None, + owner_turn_id=owner_turn_id, + completed_turn_id=( + completed_turn_id if session.thread_id == completion_thread_id else None + ), + model_usage=model_usage, is_scan_owner=( scan["mode"] == "deep" and session.thread_id == scan["deep_scan_owner_thread_id"] @@ -214,9 +340,67 @@ def collect_scan_usage( result["missingThreadCount"] = len(missing_thread_ids) if warnings: result["warnings"] = sorted(warnings) + if model_usage and (attribution or any(model is not None for model in model_usage)): + result["modelUsage"] = [{"model": model, **usage} for model, usage in model_usage.items()] return result +def _read_rollout_copies_usage( + copies: list[RolloutSession], + *, + started_at: datetime, + completed_at: datetime | None, + owner_turn_id: str | None, + completed_turn_id: str | None = None, + model_usage: dict[str | None, dict[str, int]], + is_scan_owner: bool = False, +) -> RolloutUsage: + readings = [] + for session in copies: + local_models: dict[str | None, dict[str, int]] = {} + try: + usage, parsed_events = _read_rollout_usage_with_extent( + session, + started_at=started_at, + completed_at=completed_at, + owner_turn_id=owner_turn_id, + completed_turn_id=completed_turn_id, + model_usage=local_models, + is_scan_owner=is_scan_owner, + ) + except (OSError, UnicodeError, ValueError): + continue + readings.append((usage, local_models, parsed_events)) + if not readings: + raise ValueError("No readable rollout copy.") + attributable = [ + reading + for reading in readings + if not reading[0].warnings.intersection( + { + "thread_identity_mismatch", + "thread_ownership_unavailable", + "thread_outside_scan_window", + "token_usage_unavailable", + } + ) + ] + # Restored indexes can reference a prefix and its complete continuation. + # Equal totals can precede later cache corrections or model receipts. Prefer + # the parsed continuation, keeping all accounting from that one copy. + usage, selected_models, _ = max( + attributable or readings, + key=lambda reading: ( + reading[0].tokens["totalTokens"], + reading[2], + "rollout_record_incomplete" not in reading[0].warnings, + ), + ) + for model, tokens in selected_models.items(): + _add_token_usage(model_usage.setdefault(model, _empty_token_usage()), tokens) + return usage + + def _scan_root_thread_ids( connection: sqlite3.Connection, scan: sqlite3.Row, @@ -242,12 +426,15 @@ def _scan_root_thread_ids( row["sdk_thread_id"] for row in connection.execute( """ - SELECT DISTINCT sdk_thread_id - FROM deep_scan_worker_threads - WHERE scan_id = ? + SELECT sdk_thread_id FROM deep_scan_attempt_sessions WHERE scan_id = ? + UNION + SELECT sdk_thread_id FROM deep_scan_workers + WHERE scan_id = ? AND sdk_thread_id IS NOT NULL + UNION + SELECT sdk_thread_id FROM deep_scan_worker_threads WHERE scan_id = ? ORDER BY sdk_thread_id """, - (scan["id"],), + (scan["id"], scan["id"], scan["id"]), ) ) roots: dict[str, None] = {} @@ -267,6 +454,104 @@ def _scan_execution_thread_ids(connection: sqlite3.Connection, scan: sqlite3.Row ) +def capture_scan_usage_owner(connection: sqlite3.Connection, scan: sqlite3.Row) -> dict[str, Any]: + """Bind the active native turn once; joining a scan does not bind later conversation work.""" + roots = _scan_root_thread_ids(connection, scan, None) + owner = roots[0] if roots else None + result = { + "threadId": owner, + "turnId": None, + "startedAt": scan["started_at"], + "dedicated": scan["recipe_json"] is not None, + } + database = _codex_state_database() + if owner is None or database is None: + return result + try: + sessions, _ = _discover_rollout_sessions(database, [owner], set(), descendant_roots=set()) + if not sessions: + return result + with sessions[0].path.open("rb") as source: + for line in source: + if not line.endswith(b"\n"): + continue + event = json.loads(line) + payload = event.get("payload") + if not isinstance(payload, dict): + continue + if event.get("type") == "turn_context" or ( + event.get("type") == "event_msg" and payload.get("type") == "task_started" + ): + turn_id = payload.get("turn_id") + if isinstance(turn_id, str): + result["turnId"] = turn_id + elif event.get("type") == "event_msg" and payload.get("type") == "task_complete": + result["turnId"] = None + except (OSError, ValueError, sqlite3.Error): + # Accounting availability must not prevent a scan from starting. + pass + return result + + +def scan_execution_attribution( + connection: sqlite3.Connection, scan: sqlite3.Row +) -> dict[str, Any] | None: + if scan["mode"] != "deep": + return None + run = connection.execute( + "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan["id"],) + ).fetchone() + if run is None: + return None + owner_json = run["usage_owner_json"] if "usage_owner_json" in run.keys() else None + legacy = False + if owner_json is None: + legacy = True + roots = _scan_root_thread_ids(connection, scan, None) + owner = { + "threadId": roots[0] if roots else None, + "turnId": None, + "startedAt": scan["started_at"], + "dedicated": scan["recipe_json"] is not None, + } + else: + owner = json.loads(owner_json) + executions = _scan_execution_thread_ids(connection, scan) + if owner.get("dedicated") and owner.get("threadId") not in executions: + executions.append(owner["threadId"]) + from deep_scan_workbench import ( + DEEP_SCAN_WORKFLOW_VERSION, + recorded_deep_scan_execution_settings, + validate_deep_scan_execution_settings, + ) + + saved_settings = recorded_deep_scan_execution_settings(run) + worker_home = {} + if saved_settings is not None: + worker_home["workerCodexHome"] = validate_deep_scan_execution_settings(saved_settings)[ + "codexHome" + ] + elif run["workflow_version"] == DEEP_SCAN_WORKFLOW_VERSION: + worker_home["workerCodexHome"] = None + return { + "formatVersion": 1, + **({"legacy": True} if legacy else {}), + "executionThreadIds": executions, + "owner": owner, + "startedAt": scan["started_at"], + "completedAt": scan["completed_at"], + **worker_home, + } + + +def scan_execution_fields(connection: sqlite3.Connection, scan: sqlite3.Row) -> dict[str, Any]: + return { + "threadIds": _scan_root_thread_ids(connection, scan, None), + "executionThreadIds": _scan_execution_thread_ids(connection, scan), + "executionAttribution": scan_execution_attribution(connection, scan), + } + + def _scan_worker_directories(connection: sqlite3.Connection, scan: sqlite3.Row) -> set[str]: if scan["mode"] != "deep": return set() @@ -282,15 +567,17 @@ def _scan_worker_directories(connection: sqlite3.Connection, scan: sqlite3.Row) } -def _codex_state_database() -> Path | None: - configured_database = os.environ.get("CODEX_STATE_DB", "").strip() +def _codex_state_database(worker_codex_home: Path | None = None) -> Path | None: + configured_home = os.environ.get("CODEX_HOME", "") + current_home = Path(configured_home) if configured_home.strip() else Path.home() / ".codex" + codex_home = worker_codex_home if worker_codex_home is not None else current_home + same_home = worker_codex_home is None or codex_home.resolve() == current_home.resolve() + configured_database = os.environ.get("CODEX_STATE_DB", "").strip() if same_home else "" if configured_database: path = Path(configured_database).expanduser() return path.resolve() if path.is_file() and os.access(path, os.R_OK) else None - configured_home = os.environ.get("CODEX_HOME", "") - codex_home = Path(configured_home) if configured_home.strip() else Path.home() / ".codex" - configured_sqlite_home = os.environ.get("CODEX_SQLITE_HOME", "") + configured_sqlite_home = os.environ.get("CODEX_SQLITE_HOME", "") if same_home else "" search_roots = [ *([Path(configured_sqlite_home)] if configured_sqlite_home.strip() else []), codex_home, @@ -324,9 +611,10 @@ def _discover_rollout_sessions( roots: list[str], warnings: set[str], *, - worker_directories: set[str], - started_at: datetime, - completed_at: datetime | None, + descendant_roots: set[str] | None = None, + worker_directories: set[str] | None = None, + started_at: datetime | None = None, + completed_at: datetime | None = None, ) -> tuple[list[RolloutSession], set[str]]: database = sqlite3.connect( f"file:{quote(str(state_database), safe='')}?mode=ro", @@ -346,7 +634,12 @@ def _discover_rollout_sessions( sessions: dict[str, RolloutSession] = {} missing_thread_ids: set[str] = set() root_ids = dict.fromkeys(roots) - started_millisecond = started_at.replace(microsecond=started_at.microsecond // 1000 * 1000) + worker_directories = worker_directories or set() + started_millisecond = ( + started_at.replace(microsecond=started_at.microsecond // 1000 * 1000) + if started_at is not None + else None + ) completed_timestamp = int(completed_at.timestamp()) if completed_at is not None else None directory_expression = "cwd" if worker_directories and os.name == "nt": @@ -397,6 +690,8 @@ def _discover_rollout_sessions( ): continue sessions[root] = RolloutSession(root, None, path) + if descendant_roots is not None and root in roots and root not in descendant_roots: + continue descendants = database.execute( """ WITH RECURSIVE descendants( @@ -458,6 +753,69 @@ def _discover_rollout_sessions( database.close() +def _discover_recorded_worker_sessions(codex_home: Path, roots: set[str]) -> list[RolloutSession]: + codex_home = codex_home.resolve() + recorded: dict[str, list[RolloutSession]] = {} + children: dict[str, set[str]] = {} + for candidate in sorted( + candidate + for directory in ("sessions", "archived_sessions") + for candidate in (codex_home / directory).rglob("*.jsonl") + ): + path = _rollout_path(str(candidate)) + if path is None: + continue + try: + with path.open("rb") as stream: + metadata = json.loads(stream.readline()) + except (OSError, UnicodeError, ValueError): + continue + if not isinstance(metadata, dict) or metadata.get("type") != "session_meta": + continue + payload = metadata.get("payload") + if not isinstance(payload, dict): + continue + thread_id = payload.get("id") or payload.get("session_id") + if not isinstance(thread_id, str): + continue + parent_id = _session_parent_thread_id(payload) + recorded.setdefault(thread_id, []).append(RolloutSession(thread_id, parent_id, path)) + if parent_id is not None: + children.setdefault(parent_id, set()).add(thread_id) + + sessions: list[RolloutSession] = [] + included = set(roots) + pending = deque(sorted(roots)) + while pending: + thread_id = pending.popleft() + for session in recorded.get(thread_id, []): + sessions.append( + RolloutSession( + thread_id, + None if thread_id in roots else session.parent_thread_id, + session.path, + ) + ) + for child_id in sorted(children.get(thread_id, set()) - included): + included.add(child_id) + pending.append(child_id) + return sessions + + +def _require_state_columns( + connection: sqlite3.Connection, + table: str, + required: set[str], +) -> None: + statements = { + "threads": "PRAGMA table_info(threads)", + "thread_spawn_edges": "PRAGMA table_info(thread_spawn_edges)", + } + columns = {str(row["name"]) for row in connection.execute(statements[table])} + if not required.issubset(columns): + raise ValueError("Codex state graph does not expose the required thread columns.") + + def _rollout_path(value: object) -> Path | None: if not isinstance(value, str) or not value: return None @@ -486,19 +844,53 @@ def _read_rollout_usage( *, started_at: datetime, completed_at: datetime | None, + owner_turn_id: str | None = None, + model_usage: dict[str | None, dict[str, int]] | None = None, completed_turn_id: str | None = None, is_scan_owner: bool = False, ) -> RolloutUsage: + usage, _ = _read_rollout_usage_with_extent( + session, + started_at=started_at, + completed_at=completed_at, + owner_turn_id=owner_turn_id, + completed_turn_id=completed_turn_id, + model_usage=model_usage, + is_scan_owner=is_scan_owner, + ) + return usage + + +def _read_rollout_usage_with_extent( + session: RolloutSession, + *, + started_at: datetime, + completed_at: datetime | None, + owner_turn_id: str | None = None, + completed_turn_id: str | None = None, + model_usage: dict[str | None, dict[str, int]] | None = None, + is_scan_owner: bool = False, +) -> tuple[RolloutUsage, int]: + parsed_events = 0 result = RolloutUsage(_empty_token_usage(), set(), None, {}) - total = result.tokens + total = _empty_token_usage() + counter_total = _empty_token_usage() warnings = result.warnings previous = _empty_token_usage() boundary_reached = False - legacy_usage = _empty_token_usage() - response_usage: dict[str, int] | None = None - observed_response_usage = _empty_token_usage() - response_ids: set[str] = set() + session_started_at: datetime | None = None current_turn_id: str | None = None + current_model: str | None = None + response_ids: set[str] = set() + response_usage_observed = False + response_tokens = 0 + expected_response_tokens = 0 + response_epoch_owned = False + local_models: dict[str | None, dict[str, int]] = {} + finished_total = _empty_token_usage() + finished_models: dict[str | None, dict[str, int]] = {} + observed_response_usage = _empty_token_usage() + native_receipts = False completed_turn_measured = False direct_control_session = False started_turns: set[str] = set() @@ -506,6 +898,42 @@ def _read_rollout_usage( closed_turns: set[str] = set() measured_turns: set[str] = set() + def finish_turn() -> None: + nonlocal total, counter_total, local_models, response_usage_observed, native_receipts + # Turn receipts supersede overlapping legacy counters. Older receipts + # lack a turn total, so retain the measured counter remainder as unknown-model usage. + remainder = ( + {key: max(0, value - total[key]) for key, value in counter_total.items()} + if not native_receipts + else _empty_token_usage() + ) + remainder["cachedInputTokens"] = min( + remainder["cachedInputTokens"], remainder["inputTokens"] + ) + remainder["cacheWriteInputTokens"] = min( + remainder["cacheWriteInputTokens"], + remainder["inputTokens"] - remainder["cachedInputTokens"], + ) + remainder["reasoningOutputTokens"] = min( + remainder["reasoningOutputTokens"], remainder["outputTokens"] + ) + remainder["totalTokens"] = remainder["inputTokens"] + remainder["outputTokens"] + if remainder["totalTokens"] > 0: + if response_usage_observed: + warnings.add("token_receipts_incomplete") + _add_token_usage(total, remainder) + _add_token_usage(local_models.setdefault(None, _empty_token_usage()), remainder) + if response_usage_observed: + warnings.discard("token_counter_regressed") + _add_token_usage(finished_total, total) + for model, usage in local_models.items(): + _add_token_usage(finished_models.setdefault(model, _empty_token_usage()), usage) + total = _empty_token_usage() + counter_total = _empty_token_usage() + local_models = {} + response_usage_observed = False + native_receipts = False + with session.path.open("rb") as source: for line_number, raw_line in enumerate(source, start=1): if not raw_line.endswith(b"\n"): @@ -524,19 +952,26 @@ def _read_rollout_usage( warnings.add("rollout_record_invalid") continue payload = event.get("payload") + if isinstance(payload, dict): + parsed_events += 1 + if event.get("type") in {"session_meta", "turn_context"} and isinstance(payload, dict): + model = payload.get("model") + if isinstance(model, str) and model: + current_model = model if line_number == 1: if event.get("type") != "session_meta" or not isinstance(payload, dict): warnings.add("thread_identity_mismatch") - return result + return result, parsed_events + session_started_at = _timestamp(payload.get("timestamp")) recorded_id = payload.get("id") or payload.get("session_id") if recorded_id != session.thread_id: warnings.add("thread_identity_mismatch") - return result + return result, parsed_events recorded_parent = _session_parent_thread_id(payload) if session.parent_thread_id is not None: if recorded_parent != session.parent_thread_id: warnings.add("thread_identity_mismatch") - return result + return result, parsed_events boundary_reached = ( session.parent_thread_id is None and not recorded_parent @@ -554,18 +989,18 @@ def _read_rollout_usage( if not isinstance(payload, dict): continue if not boundary_reached: - if _is_owned_task_start(session.thread_id, event, payload): + if _is_owned_task_start(session.thread_id, event, payload, session_started_at): task_started_at = _timestamp(event.get("timestamp")) if task_started_at is None: warnings.add("thread_ownership_unavailable") - return result + return result, parsed_events if task_started_at < started_at or ( completed_at is not None and task_started_at > completed_at and payload.get("turn_id") != completed_turn_id ): warnings.add("thread_outside_scan_window") - return result + return result, parsed_events boundary_reached = True elif event.get("type") == "event_msg" and payload.get("type") == "token_count": inherited_usage = _token_snapshot(payload) @@ -587,30 +1022,37 @@ def _read_rollout_usage( ) ): direct_control_session = False + if event.get("type") == "turn_context" or ( + event.get("type") == "event_msg" and payload.get("type") == "task_started" + ): + next_turn_id = payload.get("turn_id") if ( completed_turn_id is not None and current_turn_id == completed_turn_id - and payload.get("turn_id") != completed_turn_id + and next_turn_id != completed_turn_id ): break - _add_token_usage( - total, response_usage if response_usage is not None else legacy_usage - ) - legacy_usage = _empty_token_usage() - response_usage = None - if payload.get("turn_id") != current_turn_id: + if next_turn_id != current_turn_id: + finish_turn() observed_response_usage = _empty_token_usage() - current_turn_id = payload.get("turn_id") - timestamp = _timestamp(event.get("timestamp")) - if not isinstance(current_turn_id, str) or not current_turn_id or timestamp is None: - warnings.add("turn_record_invalid") - elif ( - completed_at is None - or timestamp <= completed_at - or current_turn_id == completed_turn_id + current_turn_id = next_turn_id + if event.get("type") == "event_msg" and ( + owner_turn_id is None or current_turn_id == owner_turn_id ): - started_turns.add(current_turn_id) - open_turns.add(current_turn_id) + timestamp = _timestamp(event.get("timestamp")) + if ( + not isinstance(current_turn_id, str) + or not current_turn_id + or timestamp is None + ): + warnings.add("turn_record_invalid") + elif ( + completed_at is None + or timestamp <= completed_at + or current_turn_id == completed_turn_id + ): + started_turns.add(current_turn_id) + open_turns.add(current_turn_id) continue if event.get("type") == "event_msg" and payload.get("type") in { "task_complete", @@ -624,7 +1066,7 @@ def _read_rollout_usage( or timestamp is None ): warnings.add("turn_record_invalid") - elif ( + elif (owner_turn_id is None or terminal_turn_id == owner_turn_id) and ( completed_at is None or timestamp <= completed_at or terminal_turn_id == completed_turn_id @@ -653,25 +1095,33 @@ def _read_rollout_usage( or timestamp <= completed_at or (completed_turn_id is not None and current_turn_id == completed_turn_id) ) + and (owner_turn_id is None or current_turn_id == owner_turn_id) and "compaction_response_id" not in payload - and response_usage is None + and not response_usage_observed ): warnings.add("compaction_usage_unavailable") + # Later receipt counters describe a new compaction epoch and cannot + # account for missing receipts from the earlier one. + if response_epoch_owned and expected_response_tokens > response_tokens: + warnings.add("token_receipts_incomplete") + response_tokens = 0 + expected_response_tokens = 0 + response_epoch_owned = False continue if event.get("type") == "token_usage_record": + response_id = payload.get("response_id") + usage = _usage_counts(payload.get("usage")) + timestamp = _timestamp(event.get("timestamp")) if not isinstance(payload.get("thread_id"), str): warnings.add("token_record_invalid") continue if payload.get("thread_id") != session.thread_id: continue - timestamp = _timestamp(event.get("timestamp")) - usage = _usage_counts(payload.get("usage")) - response_id = payload.get("response_id") if ( - timestamp is None - or usage is None - or not isinstance(response_id, str) + not isinstance(response_id, str) or not response_id + or usage is None + or timestamp is None ): warnings.add("token_record_invalid") continue @@ -680,17 +1130,23 @@ def _read_rollout_usage( ) if completed_at is not None and timestamp > completed_at and not is_completion_turn: continue - if response_usage is None: - response_usage = _empty_token_usage() if current_turn_id is None: current_turn_id = payload.get("turn_id") - if isinstance(current_turn_id, str): + if isinstance(current_turn_id, str) and ( + owner_turn_id is None or current_turn_id == owner_turn_id + ): open_turns.add(current_turn_id) if response_id in response_ids: continue response_ids.add(response_id) - _add_token_usage(observed_response_usage, usage) turn_usage = _usage_counts(payload.get("turn_token_usage")) + _add_token_usage(observed_response_usage, usage) + cumulative = _usage_counts(payload.get("thread_token_usage")) + if cumulative is not None and turn_usage is None: + expected_response_tokens = max( + expected_response_tokens, cumulative["totalTokens"] + ) + response_tokens += usage["totalTokens"] receipt_turn_id = payload.get("turn_id") if isinstance(receipt_turn_id, str): measured_turns.add(receipt_turn_id) @@ -698,17 +1154,38 @@ def _read_rollout_usage( if turn_usage is not None: observed_response_usage = turn_usage continue + if ( + owner_turn_id is not None + and payload.get("turn_id", current_turn_id) != owner_turn_id + ): + continue + if current_turn_id is None: + current_turn_id = payload.get("turn_id") + if turn_usage is not None: + native_receipts = True + if any( + turn_usage[key] > observed_response_usage[key] + for key in TOKEN_FIELDS.values() + ): + warnings.add("token_record_incomplete") + else: + response_epoch_owned = True if is_completion_turn: completed_turn_measured = True - if turn_usage is not None and any( - turn_usage[key] > observed_response_usage[key] for key in TOKEN_FIELDS.values() - ): - warnings.add("token_record_incomplete") + if not response_usage_observed: + response_usage_observed = True + total = _empty_token_usage() + local_models = {} result.tokens_updated_at = max(result.tokens_updated_at or timestamp, timestamp) - _add_token_usage(response_usage, usage) + model = payload.get("model", current_model) + if not isinstance(model, str): + model = None + _add_token_usage(total, usage) + _add_token_usage(local_models.setdefault(model, _empty_token_usage()), usage) continue if event.get("type") != "event_msg" or payload.get("type") != "token_count": continue + # Native rate-limit updates can carry no token usage. timestamp = _timestamp(event.get("timestamp")) in_window = ( timestamp is not None @@ -719,7 +1196,7 @@ def _read_rollout_usage( or (completed_turn_id is not None and current_turn_id == completed_turn_id) ) ) - if in_window: + if in_window and (owner_turn_id is None or current_turn_id == owner_turn_id): rate_limits = _rate_limit_snapshot(payload.get("rate_limits"), timestamp) if rate_limits is not None: key = rate_limits["limitId"] @@ -735,16 +1212,18 @@ def _read_rollout_usage( if timestamp is None or snapshot is None: warnings.add("token_record_invalid") continue - delta = { - key: value - previous[key] if value >= previous[key] else value - for key, value in snapshot.items() - } + if snapshot["totalTokens"] < previous["totalTokens"]: + warnings.add("token_counter_regressed") + continue + delta = {key: max(0, value - previous[key]) for key, value in snapshot.items()} previous = snapshot is_completion_turn = ( completed_turn_id is not None and current_turn_id == completed_turn_id ) if completed_at is not None and timestamp > completed_at and not is_completion_turn: continue + if owner_turn_id is not None and current_turn_id != owner_turn_id: + continue if isinstance(current_turn_id, str): measured_turns.add(current_turn_id) if timestamp < started_at: @@ -752,11 +1231,24 @@ def _read_rollout_usage( if is_completion_turn: completed_turn_measured = True result.tokens_updated_at = max(result.tokens_updated_at or timestamp, timestamp) + if not response_usage_observed: + local_models.setdefault(current_model, _empty_token_usage()) if delta["totalTokens"] <= 0: continue - _add_token_usage(legacy_usage, delta) + _add_token_usage(counter_total, delta) + if not response_usage_observed: + _add_token_usage(total, delta) + _add_token_usage( + local_models.setdefault(current_model, _empty_token_usage()), delta + ) + + finish_turn() - _add_token_usage(total, response_usage if response_usage is not None else legacy_usage) + if response_epoch_owned and expected_response_tokens > response_tokens: + warnings.add("token_receipts_incomplete") + if model_usage is not None: + for model, usage in finished_models.items(): + _add_token_usage(model_usage.setdefault(model, _empty_token_usage()), usage) if not boundary_reached: warnings.add("thread_ownership_unavailable") elif result.tokens_updated_at is None and not direct_control_session: @@ -771,7 +1263,8 @@ def _read_rollout_usage( warnings.add("token_record_missing") if completed_turn_id is not None and not completed_turn_measured: warnings.add("completion_turn_unavailable") - return result + result.tokens = finished_total + return result, parsed_events def _rate_limit_snapshot(value: object, timestamp: datetime) -> dict[str, Any] | None: @@ -848,6 +1341,7 @@ def _is_owned_task_start( thread_id: str, event: Mapping[str, Any], payload: Mapping[str, Any], + session_started_at: datetime | None, ) -> bool: if event.get("type") != "event_msg" or payload.get("type") != "task_started": return False @@ -858,7 +1352,12 @@ def _is_owned_task_start( thread_order = _uuid7_order(thread_id) turn_order = _uuid7_order(turn_id) if thread_order is None: - return True + task_started_at = payload.get("started_at") + return ( + type(task_started_at) in {int, float} + and session_started_at is not None + and task_started_at >= math.floor(session_started_at.timestamp()) + ) return turn_order is not None and turn_order >= thread_order diff --git a/plugins/codex-security/scripts/workbench_schema.py b/plugins/codex-security/scripts/workbench_schema.py index d2c940079d..7bb30613de 100644 --- a/plugins/codex-security/scripts/workbench_schema.py +++ b/plugins/codex-security/scripts/workbench_schema.py @@ -79,6 +79,16 @@ def apply_migrations( repair_additive_migration(connection, version) elif version == 11: repair_deep_scan_migration(connection) + elif 53 <= version <= 58: + # Recreated Deep tables need the recorded extensions too. + # Do not replay data updates from an already applied migration. + for statement in sql_statements(sql): + if statement.startswith("ALTER TABLE "): + add_migration_column(connection, statement) + elif statement.startswith("CREATE TABLE "): + connection.execute( + statement.replace("CREATE TABLE ", "CREATE TABLE IF NOT EXISTS ", 1) + ) else: if version in (38, 39): connection.create_function("migration_json_extract", 2, migration_json_extract) diff --git a/plugins/codex-security/scripts/workbench_source_coverage.py b/plugins/codex-security/scripts/workbench_source_coverage.py new file mode 100644 index 0000000000..35fe77464c --- /dev/null +++ b/plugins/codex-security/scripts/workbench_source_coverage.py @@ -0,0 +1,104 @@ +"""Project accepted worker coverage into a stopped scan's publication.""" + +from __future__ import annotations + +import argparse +import copy +import sqlite3 +from pathlib import Path +from typing import Any + + +def retain_unmerged_review_coverage( + scan_dir: Path, coverage: dict[str, Any], worker: sqlite3.Row, draft: dict[str, Any] +) -> None: + """Retain review provenance and obligations without promoting its findings.""" + source = draft["coverage"] + provenance = {"workerId": worker["id"], "attempt": worker["attempt"]} + prefix = f"{worker['id']}-attempt-{worker['attempt']}" + artifact_prefix = Path(worker["artifact_dir"]).relative_to(scan_dir).as_posix() + archive_prefix = f"{Path(artifact_prefix).parent.as_posix()}/attempts/" + surfaces = { + item.get("id"): f"{prefix}-surface-{index + 1}" + for index, item in enumerate(source.get("surfaces", [])) + } + + def retain(field: str, item: dict[str, Any]) -> None: + # A committed budget draft can be replayed before the scan is sealed. + # These IDs and provenance identify the same immutable accepted review. + items = coverage.setdefault(field, []) + if "id" in item: + matches = [] + for index, existing in enumerate(items): + existing_provenance = ( + existing.get("provenance") if isinstance(existing, dict) else None + ) + if ( + isinstance(existing_provenance, dict) + and existing.get("id") == item["id"] + and all( + existing_provenance.get(key) == value for key, value in provenance.items() + ) + ): + matches.append(index) + if matches: + # Refresh older projections from the same accepted bytes, even + # if an interrupted writer saved more than one copy. + items[matches[0]] = item + for index in reversed(matches[1:]): + del items[index] + return + if item not in items: + items.append(item) + + for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"): + for index, original in enumerate(source.get(field, [])): + item = copy.deepcopy(original if isinstance(original, dict) else {"question": original}) + source_provenance = item.get("provenance") + if not isinstance(source_provenance, dict): + source_provenance = {} + # Keep source descriptions; the accepted owner supplies identity. + for key in ("workerId", "attempt", "sourceId", "candidateId"): + source_provenance.pop(key, None) + item["provenance"] = { + **source_provenance, + **provenance, + **({"sourceId": item["id"]} if "id" in item else {}), + **({"candidateId": item["candidateId"]} if "candidateId" in item else {}), + } + item["id"] = f"{prefix}-{field}-{index + 1}" + if field == "surfaces": + item["id"] = f"{prefix}-surface-{index + 1}" + item["receiptRefs"] = [ + ref + if ref.startswith((archive_prefix, f"{artifact_prefix}/")) + else f"{artifact_prefix}/{ref}" + for ref in item.get("receiptRefs", []) + ] + if field == "deferred" and "candidateId" in item: + item["candidateId"] = f"{prefix}-candidate-{index + 1}" + if "surfaceIds" in item: + item["surfaceIds"] = [surfaces.get(value, value) for value in item["surfaceIds"]] + retain(field, item) + retain("reviews", provenance) + for index, limitation in enumerate(draft.get("scope", {}).get("limitations", [])): + retain( + "deferred", + { + "id": f"{prefix}-scope-{index + 1}", + "reason": limitation, + "provenance": provenance, + }, + ) + retain( + "deferred", + { + "id": f"{prefix}-unmerged", + "provenance": provenance, + "reason": "This accepted discovery was not merged before the scan reached its cost limit.", + }, + ) + + +if __name__ == "__main__": + argparse.ArgumentParser(description=__doc__).parse_args() diff --git a/plugins/codex-security/scripts/workbench_validation.py b/plugins/codex-security/scripts/workbench_validation.py index 7c52f92ed2..4cabb89f17 100644 --- a/plugins/codex-security/scripts/workbench_validation.py +++ b/plugins/codex-security/scripts/workbench_validation.py @@ -10,7 +10,7 @@ import sys import uuid from datetime import datetime -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import Any # Some plugin hosts launch Python with safe-path isolation enabled. @@ -52,6 +52,76 @@ def sqlite_busy(error: sqlite3.OperationalError) -> bool: path_within_scope = finalizer.path_within_scope +def latest_publication_reducer(db: Any, connection: Any, scan_id: str) -> Any | None: + # Publication follows the durable reducer sequence, not completion timestamps. + def reducer_order(worker: Any) -> tuple[int, str]: + match = re.fullmatch(r"dedup-([0-9]+)", Path(worker["prompt_path"]).parent.name) + return (int(match[1]) if match else 0, worker["id"]) + + return max( + ( + worker + for worker in db.saved_results._saved_workers(connection, scan_id) + if worker["kind"] == "dedup" + and worker["status"] == "succeeded" + and worker["result_manifest_path"] + ), + key=reducer_order, + default=None, + ) + + +def has_verified_selected_result( + db: Any, connection: Any, scan: Any, *, reducer: Any = None +) -> bool: + """Verify persisted Deep publication authority before using the artifact read contract.""" + if scan["mode"] != "deep": + return False + run = db.deep_scan.find_supported_deep_scan_run(connection, scan["id"]) + selection = db.deep_scan.deep_scan_finalization_input(run) if run is not None else None + try: + if selection is not None: + db.saved_results._require_selected_result(scan, selection) + return selection["resultPath"] is not None + if run is None or run["workflow_version"] not in { + "deep-scan-mcp/v1", + "deep-security-scan/v1", + }: + return False + if reducer is None: + reducer = latest_publication_reducer(db, connection, scan["id"]) + accepted = reducer["accepted_result_path"] if reducer is not None else None + digests = db.saved_results._accepted_source_digests(connection, scan["id"]) + if accepted is None or accepted not in digests: + return False + scan_dir = Path(scan["scan_dir"]) + db.saved_results._read_saved_result( + scan_dir, + Path(accepted).relative_to(scan_dir).as_posix(), + scan["id"], + kind="dedup", + accepted_source_digests=digests, + ) + except (finalizer.ContractError, ValueError) as exc: + raise finalizer.RecoverableContractError(str(exc)) from exc + return True + + +def validate_completed_scan_artifacts(db: Any, scan: sqlite3.Row) -> None: + scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"])) + db.require_recorded_manifest_digest(scan, scan_dir) + db.verify_manifest_binding(scan, db.read_json_object(scan_dir / db.ARTIFACTS["manifest"])) + try: + manifest = db._prepare_scan_finalization( + scan_dir, + expected_coverage_mode=db.expected_coverage_mode(scan), + )[2] + except db.ContractError as exc: + raise SystemExit(str(exc)) from exc + db.verify_manifest_binding(scan, manifest) + db.published_manifest_digest(scan_dir, manifest) + + def require_close_note(close_reason: str | None, note: str | None) -> None: if note is None and close_reason == "false_positive": raise SystemExit("Explain why this finding is a false positive.") @@ -137,6 +207,7 @@ def _valid_measured_scan_usage(usage: object) -> bool: "threadCount", "missingThreadCount", "warnings", + "modelUsage", "completion", "byRole", "window", @@ -152,6 +223,22 @@ def _valid_measured_scan_usage(usage: object) -> bool: or usage["cachedInputTokens"] + usage["cacheWriteInputTokens"] > usage["inputTokens"] ): return False + if "modelUsage" in usage: + parts = usage["modelUsage"] + if not isinstance(parts, list) or not parts: + return False + for part in parts: + if not isinstance(part, dict) or set(part) != {"model", *SCAN_USAGE_TOKEN_KEYS}: + return False + if part["model"] is not None and not isinstance(part["model"], str): + return False + if ( + not all(type(part[key]) is int and part[key] >= 0 for key in SCAN_USAGE_TOKEN_KEYS) + or part["cachedInputTokens"] + part["cacheWriteInputTokens"] > part["inputTokens"] + ): + return False + if any(sum(part[key] for part in parts) != usage[key] for key in SCAN_USAGE_TOKEN_KEYS): + return False missing = usage.get("missingThreadCount", 0) if type(missing) is not int or missing < 0: return False @@ -186,7 +273,7 @@ def _valid_measured_scan_usage(usage: object) -> bool: return True -def parse_scan_cost(value: str | None) -> str | None: +def parse_scan_cost(value: str | None, *, allow_lower_bound: bool = False) -> str | None: if value is None: return None if len(value.encode("utf-8")) > 8192: @@ -195,7 +282,10 @@ def parse_scan_cost(value: str | None) -> str | None: cost = json.loads(value, parse_constant=reject_nonstandard_json_number) except (TypeError, UnicodeError, ValueError) as exc: raise SystemExit("Scan cost must be a valid JSON object.") from exc - if isinstance(cost, dict) and "usage" in cost: + if allow_lower_bound and isinstance(cost, dict) and set(cost) == {"lowerBound"}: + if not _valid_legacy_scan_cost(cost["lowerBound"]): + raise SystemExit("Scan cost lower bound must be a valid measured cost.") + elif isinstance(cost, dict) and "usage" in cost: if ( not set(cost).issubset({"usage", "cost"}) or not _valid_measured_scan_usage(cost["usage"]) @@ -229,3 +319,74 @@ def require_occurrence(connection: sqlite3.Connection, occurrence_id: str) -> sq if __name__ == "__main__": argparse.ArgumentParser(description=__doc__).parse_args() + + +def require_target(value: str) -> Path: + expanded = Path(value).expanduser() + if not expanded.is_absolute(): + raise SystemExit("Scan target must be an absolute local directory path.") + target = expanded.resolve() + if not target.is_dir(): + raise SystemExit(f"Scan target is not a readable local directory: {target}") + return target + + +def parse_scan_recipe( + value: str, repository: Path, *, validate_current_target: bool = True +) -> dict[str, Any]: + try: + recipe = json.loads(value, parse_constant=reject_non_finite_json) + except (TypeError, UnicodeError, ValueError) as exc: + raise SystemExit("Scan launch recipe must be a valid JSON object.") from exc + if not isinstance(recipe, dict): + raise SystemExit("Scan launch recipe must be a JSON object.") + requested_repository = recipe.get("repository") + if ( + not isinstance(requested_repository, str) + or ( + require_target(requested_repository) + if validate_current_target + else Path(requested_repository) + ) + != repository + ): + raise SystemExit("Scan launch recipe repository must match the scanned repository.") + if recipe.get("mode") not in {"standard", "deep"}: + raise SystemExit("Scan launch recipe mode must be standard or deep.") + if not isinstance(recipe.get("config"), dict): + raise SystemExit("Scan launch recipe config must be a JSON object.") + target = recipe.get("target") + if not isinstance(target, dict) or target.get("kind") not in { + "repository", + "paths", + "refs", + "working_tree", + }: + raise SystemExit("Scan launch recipe target must identify a supported scan target.") + paths = target.get("paths") + if not isinstance(paths, list) or not all(isinstance(path, str) for path in paths): + raise SystemExit("Scan launch recipe target paths must be an array of strings.") + if target["kind"] == "paths" and not paths: + raise SystemExit("A scoped scan launch recipe must include at least one target path.") + if target["kind"] != "paths" and paths: + raise SystemExit("Only scoped scan launch recipes can include target paths.") + for path in paths: + candidate = PurePosixPath(path) + if ( + not path + or candidate.is_absolute() + or ".." in candidate.parts + or "\\" in path + or ( + validate_current_target + and ( + not (repository / candidate).exists() + or not (repository / candidate).resolve().is_relative_to(repository) + ) + ) + ): + raise SystemExit("Scan launch recipe target paths must exist inside the repository.") + if target["kind"] in {"refs", "working_tree"}: + if not isinstance(target.get("base"), str) or not isinstance(target.get("head"), str): + raise SystemExit("Diff scan launch recipes require resolved base and head revisions.") + return recipe diff --git a/plugins/codex-security/shared/workbench-history-repairs.json b/plugins/codex-security/shared/workbench-history-repairs.json index 16feecea4c..acadb51626 100644 --- a/plugins/codex-security/shared/workbench-history-repairs.json +++ b/plugins/codex-security/shared/workbench-history-repairs.json @@ -1,23 +1,4 @@ [ - [ - "-- Move source-scope previews before the shared history repairs.", - "WITH input AS (SELECT :history AS history)", - "SELECT 'sql' AS operation,", - " 'UPDATE schema_migrations SET version = 50 WHERE version = ' || version AS value", - "FROM schema_migrations", - "WHERE version IN (34, 40, 41, 42, 43, 44, 48)", - " AND name = 'persist authorized source excerpt scopes';" - ], - [ - "WITH history AS (", - " SELECT json_extract(value, '$.version') AS version,", - " json_extract(value, '$.name') AS name FROM json_each(:history)", - ")", - "SELECT 'error' AS operation,", - " 'The Codex Security database has an unsupported source-scope migration history.' AS value", - "FROM schema_migrations JOIN history USING (version)", - "WHERE version IN (43, 50) AND schema_migrations.name != history.name;" - ], [ "-- mirror and early-moves: relocate numbers before checking profiles.", "WITH", @@ -38,9 +19,39 @@ "),", "moves(position, version, destination, name) AS (", " VALUES", - " (0, 33, 40, 'index finding identity and comparison history'),", - " (1, 25, 26, 'persist scan completion warnings'),", - " (2, 12, 20, 'phase-specific scan progress')", + " (0, 57, 58, 'bind original deep scan execution settings'),", + " (1, 56, 57, 'bind original deep scan parent usage turn'),", + " (2, 55, 56, 'freeze stopped scan checkpoint selections'),", + " (3, 54, 55, 'persist selected deep scan finalization input'),", + " (4, 53, 54, 'retain deep scan attempts and exact merge inputs'),", + " (5, 52, 53, 'preserve original deep scan discovery context'),", + " (6, 55, 58, 'bind original deep scan execution settings'),", + " (7, 54, 57, 'bind original deep scan parent usage turn'),", + " (8, 53, 56, 'freeze stopped scan checkpoint selections'),", + " (9, 52, 55, 'persist selected deep scan finalization input'),", + " (10, 51, 54, 'retain deep scan attempts and exact merge inputs'),", + " (11, 50, 53, 'preserve original deep scan discovery context'),", + " (12, 53, 58, 'bind original deep scan execution settings'),", + " (13, 52, 58, 'bind original deep scan execution settings'),", + " (14, 51, 58, 'bind original deep scan execution settings'),", + " (15, 52, 57, 'bind original deep scan parent usage turn'),", + " (16, 51, 57, 'bind original deep scan parent usage turn'),", + " (17, 48, 57, 'bind original deep scan parent usage turn'),", + " (18, 51, 56, 'freeze stopped scan checkpoint selections'),", + " (19, 50, 56, 'freeze stopped scan checkpoint selections'),", + " (20, 47, 56, 'freeze stopped scan checkpoint selections'),", + " (21, 50, 55, 'persist selected deep scan finalization input'),", + " (22, 49, 55, 'persist selected deep scan finalization input'),", + " (23, 46, 55, 'persist selected deep scan finalization input'),", + " (24, 49, 54, 'retain deep scan attempts and exact merge inputs'),", + " (25, 48, 54, 'retain deep scan attempts and exact merge inputs'),", + " (26, 45, 54, 'retain deep scan attempts and exact merge inputs'),", + " (27, 48, 53, 'preserve original deep scan discovery context'),", + " (28, 47, 53, 'preserve original deep scan discovery context'),", + " (29, 44, 53, 'preserve original deep scan discovery context'),", + " (30, 33, 40, 'index finding identity and comparison history'),", + " (31, 25, 26, 'persist scan completion warnings'),", + " (32, 12, 20, 'phase-specific scan progress')", ")", "SELECT 0 AS position, 'error' AS operation,", " 'The Codex Security database has an unsupported mirror migration history.' AS value", @@ -57,6 +68,25 @@ " FROM moves JOIN schema_migrations USING (version, name)", "ORDER BY position;" ], + [ + "-- Move source-scope previews before the shared history repairs.", + "WITH input AS (SELECT :history AS history)", + "SELECT 'sql' AS operation,", + " 'UPDATE schema_migrations SET version = 50 WHERE version = ' || version AS value", + "FROM schema_migrations", + "WHERE version IN (34, 40, 41, 42, 43, 44, 48)", + " AND name = 'persist authorized source excerpt scopes';" + ], + [ + "WITH history AS (", + " SELECT json_extract(value, '$.version') AS version,", + " json_extract(value, '$.name') AS name FROM json_each(:history)", + ")", + "SELECT 'error' AS operation,", + " 'The Codex Security database has an unsupported source-scope migration history.' AS value", + "FROM schema_migrations JOIN history USING (version)", + "WHERE version IN (43, 50) AND schema_migrations.name != history.name;" + ], [ "-- profiles: preserve legacy columns before the later repairs.", "WITH", diff --git a/plugins/codex-security/shared/workbench-migrations.json b/plugins/codex-security/shared/workbench-migrations.json index 1e6a725b0e..9f1ebd4b33 100644 --- a/plugins/codex-security/shared/workbench-migrations.json +++ b/plugins/codex-security/shared/workbench-migrations.json @@ -64,7 +64,9 @@ { "version": 8, "name": "sealed manifest digests", - "statements": ["ALTER TABLE scans\nADD COLUMN seal_manifest_digest TEXT;"] + "statements": [ + "ALTER TABLE scans\nADD COLUMN seal_manifest_digest TEXT;" + ] }, { "version": 9, @@ -77,7 +79,9 @@ { "version": 10, "name": "scan cancellation state", - "statements": ["ALTER TABLE scans\nADD COLUMN canceled_at TEXT;"] + "statements": [ + "ALTER TABLE scans\nADD COLUMN canceled_at TEXT;" + ] }, { "version": 11, @@ -96,7 +100,9 @@ { "version": 12, "name": "scan continuation threads", - "statements": ["ALTER TABLE scans\nADD COLUMN continuation_thread_id TEXT;"] + "statements": [ + "ALTER TABLE scans\nADD COLUMN continuation_thread_id TEXT;" + ] }, { "version": 13, @@ -135,7 +141,9 @@ { "version": 17, "name": "scan target summaries", - "statements": ["ALTER TABLE scans\nADD COLUMN target_summary TEXT;"] + "statements": [ + "ALTER TABLE scans\nADD COLUMN target_summary TEXT;" + ] }, { "version": 18, @@ -190,7 +198,9 @@ { "version": 24, "name": "persist scan cost estimates", - "statements": ["ALTER TABLE scans ADD COLUMN cost_json TEXT;"] + "statements": [ + "ALTER TABLE scans ADD COLUMN cost_json TEXT;" + ] }, { "version": 25, @@ -356,7 +366,9 @@ { "version": 42, "name": "editable scan names", - "statements": ["ALTER TABLE scans ADD COLUMN name TEXT;"] + "statements": [ + "ALTER TABLE scans ADD COLUMN name TEXT;" + ] }, { "version": 43, @@ -369,7 +381,9 @@ { "version": 44, "name": "version local finding embedding inputs", - "statements": ["ALTER TABLE finding_embeddings ADD COLUMN cache_key TEXT;"] + "statements": [ + "ALTER TABLE finding_embeddings ADD COLUMN cache_key TEXT;" + ] }, { "version": 45, @@ -414,7 +428,9 @@ { "version": 50, "name": "persist authorized source excerpt scopes", - "statements": ["ALTER TABLE scans ADD COLUMN source_scopes_json TEXT;"] + "statements": [ + "ALTER TABLE scans ADD COLUMN source_scopes_json TEXT;" + ] }, { "version": 51, @@ -434,5 +450,53 @@ "ALTER TABLE scan_severity_assessments ADD COLUMN classification_runs INTEGER CHECK (classification_runs BETWEEN 1 AND 10);", "ALTER TABLE scan_severity_classifications ADD COLUMN classification_runs INTEGER CHECK (classification_runs BETWEEN 1 AND 10);" ] + }, + { + "version": 53, + "name": "preserve original deep scan discovery context", + "statements": [ + "ALTER TABLE deep_scan_runs ADD COLUMN discovery_user_context TEXT;", + "UPDATE deep_scan_runs\nSET discovery_user_context = (\n SELECT user_context FROM scans WHERE scans.id = deep_scan_runs.scan_id\n)\nWHERE workflow_version IN (\n 'deep-security-scan/v1', 'deep-scan-mcp/v1'\n);" + ] + }, + { + "version": 54, + "name": "retain deep scan attempts and exact merge inputs", + "statements": [ + "CREATE TABLE deep_scan_attempts (\n scan_id TEXT NOT NULL REFERENCES deep_scan_runs(scan_id) ON DELETE CASCADE,\n worker_id TEXT NOT NULL REFERENCES deep_scan_workers(id) ON DELETE CASCADE,\n attempt INTEGER NOT NULL,\n status TEXT NOT NULL,\n started_at TEXT NOT NULL,\n completed_at TEXT,\n end_reason TEXT,\n error_message TEXT,\n accepted_result_path TEXT,\n accepted_result_sha256 TEXT,\n receipt_json TEXT,\n PRIMARY KEY (worker_id, attempt)\n);", + "CREATE TABLE deep_scan_attempt_sessions (\n scan_id TEXT NOT NULL REFERENCES deep_scan_runs(scan_id) ON DELETE CASCADE,\n worker_id TEXT NOT NULL,\n attempt INTEGER NOT NULL,\n sdk_thread_id TEXT NOT NULL,\n observed_at TEXT NOT NULL,\n PRIMARY KEY (worker_id, attempt, sdk_thread_id),\n FOREIGN KEY (worker_id, attempt)\n REFERENCES deep_scan_attempts(worker_id, attempt) ON DELETE CASCADE\n);", + "CREATE TABLE deep_scan_merge_claims (\n worker_id TEXT PRIMARY KEY REFERENCES deep_scan_workers(id) ON DELETE CASCADE,\n scan_id TEXT NOT NULL REFERENCES deep_scan_runs(scan_id) ON DELETE CASCADE,\n previous_worker_id TEXT,\n previous_result_path TEXT,\n previous_result_sha256 TEXT,\n receipt_json TEXT\n);", + "ALTER TABLE deep_scan_dedup_inputs ADD COLUMN result_manifest_path TEXT;", + "ALTER TABLE deep_scan_dedup_inputs ADD COLUMN result_manifest_sha256 TEXT;", + "ALTER TABLE deep_scan_dedup_inputs ADD COLUMN attempt INTEGER;" + ] + }, + { + "version": 55, + "name": "persist selected deep scan finalization input", + "statements": [ + "ALTER TABLE deep_scan_runs ADD COLUMN finalization_input_json TEXT;" + ] + }, + { + "version": 56, + "name": "freeze stopped scan checkpoint selections", + "statements": [ + "ALTER TABLE scans ADD COLUMN retained_checkpoint_heads_json TEXT;" + ] + }, + { + "version": 57, + "name": "bind original deep scan parent usage turn", + "statements": [ + "ALTER TABLE deep_scan_runs ADD COLUMN usage_owner_json TEXT;" + ] + }, + { + "version": 58, + "name": "bind original deep scan execution settings", + "statements": [ + "ALTER TABLE deep_scan_runs ADD COLUMN execution_settings_json TEXT;" + ] } ] diff --git a/plugins/codex-security/tests/test_accepted_publication_references.py b/plugins/codex-security/tests/test_accepted_publication_references.py new file mode 100644 index 0000000000..2f7580525f --- /dev/null +++ b/plugins/codex-security/tests/test_accepted_publication_references.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +import hashlib +import json +from argparse import Namespace + +import pytest +from test_deep_scan_publication_authority import stage_publication +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan + + +def accept_reducer(connection, scan): + result = add_worker(connection, scan) + worker_id = result.parent.name + coverage = { + **scan.coverage, + "completeness": "partial", + "deferred": [{"id": "accepted-follow-up", "reason": "Accepted unresolved review."}], + "reviews": [{"workerId": worker_id, "attempt": 1, "completeness": "partial"}], + } + contents = json.dumps( + { + "scanId": scan.scan_id, + "complete": True, + "findings": scan.findings, + "sourceCoverage": coverage, + } + ).encode() + digest = hashlib.sha256(contents).hexdigest() + result.write_bytes(contents) + accepted = result.parent / "accepted" / f"{digest}.json" + accepted.parent.mkdir() + accepted.write_bytes(contents) + result.unlink() + with connection: + connection.execute( + "UPDATE deep_scan_workers SET kind = 'dedup', merge_state = 'none' WHERE id = ?", + (worker_id,), + ) + connection.execute( + "INSERT INTO deep_scan_attempts (scan_id, worker_id, attempt, status, started_at, " + "completed_at, accepted_result_path, accepted_result_sha256) " + "VALUES (?, ?, 1, 'succeeded', ?, ?, ?, ?)", + (scan.scan_id, worker_id, scan.timestamp, scan.timestamp, str(accepted), digest), + ) + connection.execute( + "UPDATE deep_scan_runs SET coordinator_generation = 3 WHERE scan_id = ?", + (scan.scan_id,), + ) + return result, accepted, coverage + + +@pytest.mark.parametrize("selected", [True, False], ids=["accepted", "replaceable-output"]) +def test_legacy_publication_compares_registered_accepted_reference( + workbench_api, workbench_db, publication_scan, selected +): + scan = publication_scan() + result, accepted, _ = accept_reducer(workbench_db, scan) + staged = stage_publication( + scan, generation=3, result_path=accepted if selected else result, title="Accepted aggregate" + ) + before = {path: path.read_bytes() for path in scan.scan_dir.rglob("*.json")} + + if selected: + workbench_api["saved_results"].write_scan_draft( + workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, staged + ) + else: + with pytest.raises(SystemExit, match="aggregate"): + workbench_api["saved_results"].write_scan_draft( + workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, staged + ) + assert {path: path.read_bytes() for path in scan.scan_dir.rglob("*.json")} == before + + +def test_stopped_recovery_uses_accepted_bytes_after_replaceable_output_disappears( + workbench_api, workbench_db, publication_scan +): + scan = publication_scan() + _, accepted, coverage = accept_reducer(workbench_db, scan) + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + contents = accepted.read_bytes() + + stopped = workbench_api["saved_results"].fail_scan( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."), + )["scan"] + + assert stopped["findingCount"] == 1 + published = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert published["reviews"] == coverage["reviews"] + assert coverage["deferred"][0] in published["deferred"] + manifest = (scan.scan_dir / "scan-manifest.json").read_bytes() + workbench_api["saved_results"].preserve_scan_results( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + Namespace( + scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None + ), + ) + assert (scan.scan_dir / "scan-manifest.json").read_bytes() == manifest + assert accepted.read_bytes() == contents diff --git a/plugins/codex-security/tests/test_budget_coverage_provenance.py b/plugins/codex-security/tests/test_budget_coverage_provenance.py new file mode 100644 index 0000000000..b41398a61c --- /dev/null +++ b/plugins/codex-security/tests/test_budget_coverage_provenance.py @@ -0,0 +1,288 @@ +from __future__ import annotations + +import copy +import hashlib +import json +import sqlite3 +from argparse import Namespace + +import pytest +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan +from test_publication_stop_interleavings import published_bytes +from test_workbench_db import BUDGET_COST + + +@pytest.mark.parametrize("explicit_ids", [True, False], ids=["named-surfaces", "omitted-ids"]) +@pytest.mark.parametrize("replay", ["fresh", "legacy", "duplicated"]) +@pytest.mark.parametrize( + "receipt_origin", ["worker-relative", "archived-scan-relative", "current-scan-relative"] +) +def test_cost_completion_retains_independent_unmerged_surfaces_and_receipts( + workbench_api, + workbench_db, + publication_scan, + explicit_ids, + replay, + receipt_origin, + tmp_path, + monkeypatch, +): + scan = publication_scan() + worker_id = add_worker(workbench_db, scan).parent.name + output = scan.scan_dir / "artifacts/deep_discovery/workers/discovery-0001/output" + output.mkdir(parents=True) + descriptions = { + "source": "source-review", + "worker": "focused-reviewer", + "task": "Check the filesystem boundary.", + } + source_provenance = { + **descriptions, + "workerId": "worker-local-label", + "attempt": 99, + "sourceId": "worker-local-source", + "candidateId": "worker-local-candidate", + } + surfaces = [ + { + "label": "Filesystem boundary", + "disposition": "needs_follow_up", + "notes": "Filesystem race remains untested.", + "reason": "The caller's filesystem policy is unknown.", + }, + { + "label": "Template boundary", + "disposition": "needs_follow_up", + "notes": "Template caller policy remains untested.", + }, + { + "label": "Configuration boundary", + "disposition": "rejected", + "reason": "The input comes only from trusted application configuration.", + }, + ] + receipts = {} + for index, surface in enumerate(surfaces, start=1): + surface["provenance"] = source_provenance + if explicit_ids: + surface["id"] = f"source-surface-{index}" + relative = f"artifacts/review-{index}.md" + surface["receiptRefs"] = [relative] + receipt = output / relative + receipt.parent.mkdir(exist_ok=True) + receipts[receipt] = f"Independent review receipt {index}.\n".encode() + receipt.write_bytes(receipts[receipt]) + if receipt_origin == "archived-scan-relative": + archived = output.parent / "attempts" / "attempt-01" + archived.parent.mkdir() + output.rename(archived) + output.mkdir() + receipts = {archived / path.relative_to(output): value for path, value in receipts.items()} + for surface in surfaces: + surface["receiptRefs"] = [ + (archived / ref).relative_to(scan.scan_dir).as_posix() + for ref in surface["receiptRefs"] + ] + elif receipt_origin == "current-scan-relative": + for surface in surfaces: + surface["receiptRefs"] = [ + (output / ref).relative_to(scan.scan_dir).as_posix() + for ref in surface["receiptRefs"] + ] + deferred = { + "reason": "Both independent validation tasks remain unfinished.", + "provenance": source_provenance, + } + if explicit_ids: + deferred["surfaceIds"] = [surface["id"] for surface in surfaces] + contents = json.dumps( + { + "scanId": scan.scan_id, + "complete": True, + "findings": scan.findings, + "coverage": { + **scan.coverage, + "completeness": "partial", + "surfaces": surfaces, + "deferred": [deferred], + "explicitExclusions": [ + { + "pattern": "vendor/", + "reason": "External dependencies were excluded.", + "provenance": source_provenance, + } + ], + "openQuestions": [ + { + "question": "Which deployment controls apply?", + "provenance": source_provenance, + } + ], + }, + } + ).encode() + digest = hashlib.sha256(contents).hexdigest() + accepted = output / "checkpoints" / f"{digest}.json" + accepted.parent.mkdir() + accepted.write_bytes(contents) + # The mutable result may advance after acceptance. Publication must read the + # accepted checkpoint, while receipt paths remain relative to its owner. + result = output / "result.json" + result.write_text("A later, unaccepted result.") + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_workers SET artifact_dir = ?, result_manifest_path = ?, " + "prompt_path = ?, merge_state = 'buffered' WHERE id = ?", + (str(output), str(result), str(output.parent / "prompt.md"), worker_id), + ) + workbench_db.execute( + "INSERT INTO deep_scan_attempts (scan_id, worker_id, attempt, status, started_at, " + "completed_at, accepted_result_path, accepted_result_sha256) " + "VALUES (?, ?, 1, 'succeeded', ?, ?, ?, ?)", + (scan.scan_id, worker_id, scan.timestamp, scan.timestamp, str(accepted), digest), + ) + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', phase = 'discovery', " + "workflow_version = 'deep-security-scan/v2', manifest_path = NULL, " + "terminal_reason = NULL, completed_at = NULL, max_discovery_runs = 100" + ) + counter_query = ( + "SELECT discovery_runs_dispatched, completion_sequence, consecutive_no_new " + "FROM deep_scan_runs" + ) + counters = tuple(workbench_db.execute(counter_query).fetchone()) + for name in ("scan-manifest.json", "findings.json", "coverage.json"): + (scan.scan_dir / name).unlink() + + budget = workbench_api["complete_budget_exhausted_scan"] + args = Namespace( + scan_id=scan.scan_id, + cost_json=json.dumps(BUDGET_COST), + message="Scan reached its original cost limit.", + ) + if replay != "fresh": + + def legacy_projection(coverage): + # Older writers kept host identity but omitted source descriptions. + for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"): + items = coverage.get(field, []) + for item in list(items): + if item.get("provenance", {}).get("workerId") == worker_id: + if replay == "duplicated" and any( + key in item["provenance"] for key in descriptions + ): + # A second interrupted writer appended the described + # projection beside the old item before sealing. + items.append(copy.deepcopy(item)) + for key in descriptions: + item["provenance"].pop(key, None) + + def interrupt_before_seal(connection, *_args, **_kwargs): + coverage_path = scan.scan_dir / "coverage.json" + coverage = json.loads(coverage_path.read_text()) + legacy_projection(coverage) + # The old writer appended the budget marker after projecting worker rows. + coverage["deferred"].sort(key=lambda item: item.get("id") == "scan-cost-limit") + coverage_path.write_text( + json.dumps(coverage, allow_nan=False, indent=2, sort_keys=True) + "\n" + ) + selection = json.loads( + connection.execute( + "SELECT finalization_input_json FROM deep_scan_runs WHERE scan_id = ?", + (scan.scan_id,), + ).fetchone()[0] + ) + selection.pop("publicationSha256", None) + with connection: + connection.execute( + "UPDATE deep_scan_runs SET finalization_input_json = ? WHERE scan_id = ?", + (json.dumps(selection), scan.scan_id), + ) + raise RuntimeError("Publication interrupted after the budget draft committed.") + + database = tmp_path / "interrupted-budget.sqlite3" + with sqlite3.connect(database) as connection: + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + workbench_db.backup(connection) + with monkeypatch.context() as patch: + patch.setitem(budget.__globals__, "complete_scan_locked", interrupt_before_seal) + with pytest.raises(RuntimeError, match="budget draft committed"): + budget(connection, args) + # Reconnect to the committed old draft and run the real public completion. + with sqlite3.connect(database) as connection: + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + assert ( + connection.execute("SELECT status FROM deep_scan_runs").fetchone()[0] == "succeeded" + ) + assert connection.execute("SELECT status FROM scans").fetchone()[0] == "running" + assert ( + connection.execute("SELECT seal_manifest_digest FROM scans").fetchone()[0] is None + ) + budget(connection, args) + connection.backup(workbench_db) + else: + budget(workbench_db, args) + + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "complete" + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert "completeness" not in coverage + actual = coverage["surfaces"] + assert [(item["label"], item.get("notes"), item.get("reason")) for item in actual] == [ + (item["label"], item.get("notes"), item.get("reason")) for item in surfaces + ] + assert len({item["id"] for item in actual}) == len(surfaces) + for original, retained in zip(surfaces, actual, strict=True): + assert retained["provenance"] == { + **descriptions, + "workerId": worker_id, + "attempt": 1, + **({"sourceId": original["id"]} if explicit_ids else {}), + } + assert retained["receiptRefs"] == [ + (output / ref).relative_to(scan.scan_dir).as_posix() + if receipt_origin == "worker-relative" + else ref + for ref in original["receiptRefs"] + ] + for path, receipt_contents in receipts.items(): + assert path.read_bytes() == receipt_contents + obligation = next(item for item in coverage["deferred"] if item["reason"] == deferred["reason"]) + assert obligation["provenance"] == {**descriptions, "workerId": worker_id, "attempt": 1} + assert sum(item["reason"] == deferred["reason"] for item in coverage["deferred"]) == 1 + for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"): + assert len({item["id"] for item in coverage[field]}) == len(coverage[field]) + for field in ("explicitExclusions", "openQuestions"): + assert len(coverage[field]) == 1 + item = coverage[field][0] + assert item["id"] == f"{worker_id}-attempt-1-{field}-1" + assert item["provenance"] == {**descriptions, "workerId": worker_id, "attempt": 1} + if explicit_ids: + assert obligation["surfaceIds"] == [item["id"] for item in actual] + assert {"workerId": worker_id, "attempt": 1} in coverage["reviews"] + report = (scan.scan_dir / "report.md").read_text() + for surface in surfaces: + for field in ("notes", "reason"): + if field in surface: + assert surface[field] in report + assert deferred["reason"] in report + assert json.loads((scan.scan_dir / "findings.json").read_text())["findings"] == [] + assert tuple(workbench_db.execute(counter_query).fetchone()) == counters + attempt = workbench_db.execute( + "SELECT accepted_result_path, accepted_result_sha256 FROM deep_scan_attempts " + "WHERE worker_id = ?", + (worker_id,), + ).fetchone() + assert tuple(attempt) == (str(accepted), digest) + assert accepted.read_bytes() == contents + before = published_bytes(scan) + workbench_api["complete_scan"]( + workbench_db, Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + ) + assert published_bytes(scan) == before + assert accepted.read_bytes() == contents diff --git a/plugins/codex-security/tests/test_budget_finding_identity.py b/plugins/codex-security/tests/test_budget_finding_identity.py new file mode 100644 index 0000000000..44bcaec826 --- /dev/null +++ b/plugins/codex-security/tests/test_budget_finding_identity.py @@ -0,0 +1,251 @@ +from __future__ import annotations + +import copy +import hashlib +import json +import sqlite3 +from argparse import Namespace + +import pytest +from test_accepted_publication_references import accept_reducer +from test_deep_scan_successful_publication import publication_scan as publication_scan +from test_publication_stop_interleavings import published_bytes +from test_workbench_db import BUDGET_COST +from workbench_test_support import run_workbench + + +def prepare_budget_publication(connection, scan): + _, accepted, _ = accept_reducer(connection, scan) + for name in ("scan-manifest.json", "findings.json", "coverage.json"): + (scan.scan_dir / name).unlink() + with connection: + recipe = json.loads(connection.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + connection.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + connection.execute( + "UPDATE deep_scan_runs SET status = 'running', phase = 'discovery', " + "workflow_version = 'deep-security-scan/v2', manifest_path = NULL, " + "terminal_reason = NULL, completed_at = NULL" + ) + return accepted, Namespace( + scan_id=scan.scan_id, cost_json=json.dumps(BUDGET_COST), message=None + ) + + +@pytest.mark.parametrize("identity_kind", ["omitted", "candidate", "authored", "anchor-only"]) +def test_budget_publication_identifies_accepted_semantic_findings( + workbench_api, workbench_db, publication_scan, identity_kind +): + scan = publication_scan() + finding = scan.findings[0] + finding["title"] = "Archive extraction crosses output boundary" + finding.pop("identity", None) + finding.pop("extensions", None) + expected = {"anchor": "archive-extraction-crosses-output-boundary"} + if identity_kind == "candidate": + finding["extensions"] = {"candidateId": "archive-candidate"} + expected = {"anchor": "archive-candidate"} + elif identity_kind == "authored": + finding["identity"] = {"anchor": "authored-anchor", "instance": "first-route"} + expected = finding["identity"].copy() + elif identity_kind == "anchor-only": + finding["identity"] = expected = {"anchor": "authored-anchor"} + accepted, args = prepare_budget_publication(workbench_db, scan) + original = accepted.read_bytes() + workbench_api["complete_budget_exhausted_scan"](workbench_db, args) + published = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert len(published) == 1 + assert published[0]["identity"] == expected + assert published[0]["codeEvidence"] == finding["codeEvidence"] + assert published[0]["remediation"] == finding["remediation"] + assert accepted.read_bytes() == original + selection = json.loads( + workbench_db.execute("SELECT finalization_input_json FROM deep_scan_runs").fetchone()[0] + ) + assert selection["resultSha256"] == hashlib.sha256(original).hexdigest() + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "complete" + + +def add_sibling_repairs(scan, count=2): + first = scan.findings[0] + first["title"] = "Shared repair" + first.pop("identity", None) + first.pop("extensions", None) + scan.findings.extend(copy.deepcopy(first) for _ in range(count - 1)) + for index, finding in enumerate(scan.findings, 1): + finding["locations"] = [{"path": "subdir/extract.py", "startLine": index}] + finding["provenance"]["sourceFindingIds"] = [f"review-{index}:candidate-{index}"] + finding["provenance"]["sourceFindings"][0]["id"] = f"review-{index}:candidate-{index}" + finding["codeEvidence"][0].update( + startLine=index, endLine=index, code=f"repair_point_{index}()" + ) + finding["remediation"] = f"Apply distinct repair {index} at this location." + finding["remediationTests"] = [f"Check distinct repair {index}."] + + +@pytest.mark.parametrize( + "case", + [ + "generated", + "candidate", + "authored-first", + "authored-last", + "reserved-instance", + "authored-siblings", + "different-rules", + "different-titles", + ], +) +def test_budget_publication_keeps_generated_sibling_repairs( + workbench_api, workbench_db, publication_scan, case +): + scan = publication_scan() + add_sibling_repairs(scan, 4 if case == "reserved-instance" else 2) + if case == "candidate": + for finding in scan.findings: + finding["extensions"] = {"candidateId": "shared-candidate"} + elif case in {"authored-first", "authored-last"}: + scan.findings[0 if case == "authored-first" else 1]["identity"] = { + "anchor": "shared-repair" + } + elif case == "reserved-instance": + scan.findings[2]["identity"] = {"anchor": "shared-repair"} + scan.findings[3]["identity"] = {"anchor": "shared-repair", "instance": "saved-2"} + elif case == "authored-siblings": + for index, finding in enumerate(scan.findings): + finding["identity"] = {"anchor": "shared-repair", "instance": f"route-{index}"} + elif case == "different-rules": + scan.findings[1]["ruleId"] = "distinct-rule" + elif case == "different-titles": + scan.findings[1]["title"] = "Distinct repair" + originals = copy.deepcopy(scan.findings) + accepted, args = prepare_budget_publication(workbench_db, scan) + original = accepted.read_bytes() + + workbench_api["complete_budget_exhausted_scan"](workbench_db, args) + + published = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert len(published) == len(scan.findings) + assert len({finding["occurrenceId"] for finding in published}) == len(scan.findings) + for expected, actual in zip(scan.findings, published, strict=True): + for key in expected.keys() - {"findingId", "occurrenceId", "fingerprints"}: + assert actual[key] == expected[key] + if "identity" not in expected: + anchor = ( + "shared-candidate" + if case == "candidate" + else expected["title"].lower().replace(" ", "-") + ) + assert actual["identity"]["anchor"] == anchor + assert expected["remediation"] in (scan.scan_dir / "report.md").read_text() + if case in {"different-rules", "different-titles"}: + assert "instance" not in actual["identity"] + selection = workbench_db.execute( + "SELECT finalization_input_json FROM deep_scan_runs" + ).fetchone()[0] + assert json.loads(selection)["resultSha256"] == hashlib.sha256(original).hexdigest() + before = published_bytes(scan) + workbench_api["complete_scan"]( + workbench_db, Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + ) + assert published_bytes(scan) == before + assert ( + workbench_db.execute("SELECT finalization_input_json FROM deep_scan_runs").fetchone()[0] + == selection + ) + assert accepted.read_bytes() == original + assert scan.findings == originals + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "complete" + + +@pytest.mark.parametrize("identity", [{}, {"instance": "first-route"}, None]) +def test_budget_publication_does_not_repair_authored_invalid_identity( + workbench_api, workbench_db, publication_scan, identity +): + scan = publication_scan() + scan.findings[0]["identity"] = identity + accepted, args = prepare_budget_publication(workbench_db, scan) + original = accepted.read_bytes() + with pytest.raises(SystemExit, match="identity"): + workbench_api["complete_budget_exhausted_scan"](workbench_db, args) + assert accepted.read_bytes() == original + assert scan.findings[0]["identity"] == identity + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] != "complete" + + +def test_budget_publication_rejects_duplicate_authored_identities( + workbench_api, workbench_db, publication_scan +): + scan = publication_scan() + add_sibling_repairs(scan) + for finding in scan.findings: + finding["identity"] = {"anchor": "shared-repair", "instance": "authored-route"} + accepted, args = prepare_budget_publication(workbench_db, scan) + original = accepted.read_bytes() + with pytest.raises(SystemExit, match="duplicate occurrence identity"): + workbench_api["complete_budget_exhausted_scan"](workbench_db, args) + assert accepted.read_bytes() == original + + +@pytest.mark.parametrize("boundary", ["before-selection-commit", "after-selection-commit"]) +def test_budget_sibling_selection_replays_in_a_fresh_process( + workbench_api, workbench_db, publication_scan, tmp_path, monkeypatch, boundary +): + scan = publication_scan() + add_sibling_repairs(scan) + accepted, args = prepare_budget_publication(workbench_db, scan) + original = accepted.read_bytes() + budget = workbench_api["complete_budget_exhausted_scan"] + drafts = [] + + def interrupt(*args, **kwargs): + drafts.append(json.loads((scan.scan_dir / "findings.json").read_text())["findings"]) + raise RuntimeError("Synthetic publication interruption.") + + state = tmp_path / "state" + state.mkdir(exist_ok=True) + with sqlite3.connect(state / "workbench.sqlite3") as connection: + workbench_db.backup(connection) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + with monkeypatch.context() as patch: + if boundary == "before-selection-commit": + patch.setattr(workbench_api["deep_scan"], "cancel_active_workers", interrupt) + else: + patch.setitem(budget.__globals__, "complete_scan_locked", interrupt) + with pytest.raises(RuntimeError, match="Synthetic publication interruption"): + budget(connection, args) + selection = connection.execute( + "SELECT finalization_input_json FROM deep_scan_runs" + ).fetchone()[0] + if boundary == "before-selection-commit": + assert selection is None + assert not (scan.scan_dir / "findings.json").exists() + else: + assert json.loads(selection)["resultSha256"] == hashlib.sha256(original).hexdigest() + run_workbench( + state, + "complete-budget-exhausted-scan", + "--scan-id", + scan.scan_id, + "--cost-json", + args.cost_json, + ) + published = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert [finding["identity"] for finding in published] == [ + finding["identity"] for finding in drafts[0] + ] + with sqlite3.connect(state / "workbench.sqlite3") as connection: + replayed_selection = connection.execute( + "SELECT finalization_input_json FROM deep_scan_runs" + ).fetchone()[0] + if selection is not None: + assert replayed_selection == selection + assert ( + json.loads(replayed_selection)["resultSha256"] == hashlib.sha256(original).hexdigest() + ) + assert connection.execute("SELECT status FROM scans").fetchone()[0] == "complete" + before = published_bytes(scan) + run_workbench(state, "complete-scan", "--scan-id", scan.scan_id) + assert published_bytes(scan) == before + assert accepted.read_bytes() == original diff --git a/plugins/codex-security/tests/test_budget_report_evidence.py b/plugins/codex-security/tests/test_budget_report_evidence.py new file mode 100644 index 0000000000..cd563c8041 --- /dev/null +++ b/plugins/codex-security/tests/test_budget_report_evidence.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import json +from argparse import Namespace + +import pytest +from test_budget_selection_publication import accept_unmerged +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan +from test_workbench_db import BUDGET_COST + + +@pytest.mark.parametrize("accepted", [False, True], ids=["unfinished-audit", "accepted-unmerged"]) +def test_empty_budget_report_does_not_deny_saved_validation( + workbench_api, workbench_db, publication_scan, accepted +): + scan = publication_scan() + if accepted: + evidence = accept_unmerged(workbench_db, scan) + else: + result = add_worker(workbench_db, scan, status="running") + checkpoints = result.parent / "checkpoints" + checkpoints.mkdir() + evidence = checkpoints / ("a" * 64 + ".json") + evidence.write_text( + json.dumps( + { + "scanId": scan.scan_id, + "complete": False, + "findings": scan.findings, + "coverage": {**scan.coverage, "completeness": "partial"}, + } + ) + ) + (result.parent / "checkpoint-head.json").write_text( + json.dumps({"checkpoint": evidence.name}) + ) + original = evidence.read_bytes() + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', phase = 'discovery', " + "workflow_version = 'deep-security-scan/v2', manifest_path = NULL, " + "terminal_reason = NULL, completed_at = NULL" + ) + for name in ("scan-manifest.json", "findings.json", "coverage.json"): + (scan.scan_dir / name).unlink() + + workbench_api["complete_budget_exhausted_scan"]( + workbench_db, + Namespace(scan_id=scan.scan_id, cost_json=json.dumps(BUDGET_COST), message=None), + ) + assert evidence.read_bytes() == original + assert json.loads(original)["findings"] + assert json.loads((scan.scan_dir / "findings.json").read_text())["findings"] == [] + assert "completeness" not in json.loads((scan.scan_dir / "coverage.json").read_text()) + report = (scan.scan_dir / "report.md").read_text() + assert "No findings were validated" not in report + assert "No findings were reported." in report + assert "Open Questions And Follow Up" in report diff --git a/plugins/codex-security/tests/test_budget_selection_publication.py b/plugins/codex-security/tests/test_budget_selection_publication.py new file mode 100644 index 0000000000..8176053243 --- /dev/null +++ b/plugins/codex-security/tests/test_budget_selection_publication.py @@ -0,0 +1,937 @@ +from __future__ import annotations + +import hashlib +import json +import shutil +import sqlite3 +import subprocess +import sys +from argparse import Namespace +from pathlib import Path + +import pytest +from test_accepted_publication_references import accept_reducer +from test_deep_scan_publication_authority import stage_publication +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan +from test_publication_stop_interleavings import published_bytes, saved_selection +from test_workbench_db import BUDGET_COST + + +def accept_unmerged(connection, scan): + result = add_worker(connection, scan) + worker_id = result.parent.name + result.write_text( + json.dumps( + { + "scanId": scan.scan_id, + "complete": True, + "findings": scan.findings, + "coverage": { + **scan.coverage, + "completeness": "partial", + "deferred": [ + { + "id": "shared-gap", + "reason": "Independent unresolved review.", + "paths": ["subdir/extract.py"], + } + ], + }, + } + ) + ) + digest = hashlib.sha256(result.read_bytes()).hexdigest() + accepted = result.parent / "accepted" / f"{digest}.json" + accepted.parent.mkdir() + accepted.write_bytes(result.read_bytes()) + with connection: + connection.execute( + "UPDATE deep_scan_workers SET merge_state = 'buffered' WHERE id = ?", (worker_id,) + ) + connection.execute( + "INSERT INTO deep_scan_attempts (scan_id, worker_id, attempt, status, started_at, " + "completed_at, accepted_result_path, accepted_result_sha256) VALUES (?, ?, 1, 'succeeded', ?, ?, ?, ?)", + (scan.scan_id, worker_id, scan.timestamp, scan.timestamp, str(accepted), digest), + ) + return accepted + + +@pytest.mark.parametrize("defect", ["changed-bytes", "incomplete-result"]) +def test_cost_before_selection_does_not_replace_invalid_accepted_evidence( + workbench_api, workbench_db, publication_scan, defect +): + scan = publication_scan() + _, accepted, _ = accept_reducer(workbench_db, scan) + document = json.loads(accepted.read_bytes()) + document["complete"] = False + accepted.write_text(json.dumps(document)) + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', phase = 'discovery', " + "workflow_version = 'deep-security-scan/v2', finalization_input_json = NULL" + ) + if defect == "incomplete-result": + workbench_db.execute( + "UPDATE deep_scan_attempts SET accepted_result_sha256 = ?", + (hashlib.sha256(accepted.read_bytes()).hexdigest(),), + ) + before = published_bytes(scan) + with pytest.raises(SystemExit): + workbench_api["complete_budget_exhausted_scan"]( + workbench_db, + Namespace(scan_id=scan.scan_id, cost_json=json.dumps(BUDGET_COST), message=None), + ) + assert published_bytes(scan) == before + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "running" + assert ( + workbench_db.execute("SELECT finalization_input_json FROM deep_scan_runs").fetchone()[0] + is None + ) + + +@pytest.mark.parametrize("accepted_kind", ["none", "unmerged", "reducer"]) +@pytest.mark.parametrize("cancel_first", [False, True]) +@pytest.mark.parametrize("lower_bound", [False, True]) +def test_cost_before_selection_retains_only_merged_findings( + workbench_api, workbench_db, publication_scan, accepted_kind, cancel_first, lower_bound +): + scan = publication_scan() + accepted = None + if accepted_kind == "reducer": + _, accepted, _ = accept_reducer(workbench_db, scan) + elif accepted_kind == "unmerged": + accepted = accept_unmerged(workbench_db, scan) + active = add_worker(workbench_db, scan, status="running") + active_id = active.parent.name + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_workers SET completed_at = NULL WHERE id = ?", (active_id,) + ) + workbench_db.execute( + "INSERT INTO deep_scan_attempts (scan_id, worker_id, attempt, status, started_at) " + "VALUES (?, ?, 1, 'running', ?)", + (scan.scan_id, active_id, scan.timestamp), + ) + counters = tuple( + workbench_db.execute( + "SELECT discovery_runs_dispatched, consecutive_no_new, completion_sequence FROM deep_scan_runs" + ).fetchone() + ) + original = accepted.read_bytes() if accepted is not None else None + for name in ("scan-manifest.json", "findings.json", "coverage.json"): + (scan.scan_dir / name).unlink() + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', phase = 'discovery', " + "workflow_version = 'deep-security-scan/v2', manifest_path = NULL, " + "terminal_reason = NULL, completed_at = NULL, max_discovery_runs = 100" + ) + args = Namespace( + scan_id=scan.scan_id, + cost_json=json.dumps({"lowerBound": BUDGET_COST} if lower_bound else BUDGET_COST), + message="Scan reached its original cost limit.", + ) + if cancel_first: + workbench_api["saved_results"].cancel_scan( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + Namespace(scan_id=scan.scan_id, thread_id=None), + ) + before = published_bytes(scan) + with pytest.raises(SystemExit, match="running"): + workbench_api["complete_budget_exhausted_scan"](workbench_db, args) + assert published_bytes(scan) == before + else: + workbench_api["complete_budget_exhausted_scan"](workbench_db, args) + row = workbench_db.execute("SELECT * FROM scans").fetchone() + run = workbench_db.execute("SELECT * FROM deep_scan_runs").fetchone() + assert row["status"] == "complete" + stored = json.loads(row["cost_json"]) + if lower_bound: + assert "cost" not in stored and "estimatedUsd" not in stored + assert stored["usage"]["coverage"] == "unavailable" + else: + assert stored == { + "cost": BUDGET_COST, + "usage": { + "coverage": "unavailable", + "source": "codex_rollout", + "threadCount": 0, + "warnings": ["codex_state_unavailable"], + }, + } + assert run["terminal_reason"] == "capped" + assert run["error_message"] == args.message + assert args.message in json.loads(row["completion_warnings_json"]) + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert len(findings) == (1 if accepted_kind == "reducer" else 0) + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert "completeness" not in coverage + assert any(item["id"] == "scan-cost-limit" for item in coverage["deferred"]) + if accepted_kind == "reducer": + selection = json.loads(run["finalization_input_json"]) + assert selection["resultSha256"] == hashlib.sha256(original).hexdigest() + assert any(item["id"] == "accepted-follow-up" for item in coverage["deferred"]) + assert ( + tuple( + workbench_db.execute( + "SELECT discovery_runs_dispatched, consecutive_no_new, completion_sequence FROM deep_scan_runs" + ).fetchone() + ) + == counters + ) + interrupted = workbench_db.execute( + "SELECT * FROM deep_scan_attempts WHERE worker_id = ?", (active_id,) + ).fetchone() + assert interrupted["status"] == "canceled" + assert interrupted["end_reason"] == "scan_stopped" + if accepted_kind == "unmerged": + assert any( + item.get("provenance", {}).get("sourceId") == "shared-gap" + for item in coverage["deferred"] + ) + before = published_bytes(scan) + workbench_api["complete_scan"]( + workbench_db, Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + ) + assert published_bytes(scan) == before + if accepted is not None: + assert accepted.read_bytes() == original + + +@pytest.mark.parametrize( + "bound", [None, {**BUDGET_COST, "estimatedUsd": -1}, {**BUDGET_COST, "estimatedUsd": 0.005}] +) +def test_lower_bound_budget_rejects_invalid_or_unexceeded_cost( + workbench_api, workbench_db, publication_scan, bound +): + scan = publication_scan() + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + before = published_bytes(scan) + with pytest.raises(SystemExit): + workbench_api["complete_budget_exhausted_scan"]( + workbench_db, + Namespace( + scan_id=scan.scan_id, cost_json=json.dumps({"lowerBound": bound}), message=None + ), + ) + assert published_bytes(scan) == before + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "running" + + +def test_lower_bound_is_not_an_ordinary_completion_cost( + workbench_api, workbench_db, publication_scan +): + scan = publication_scan() + before = published_bytes(scan) + with pytest.raises(SystemExit): + workbench_api["complete_scan"]( + workbench_db, + Namespace( + scan_id=scan.scan_id, + claim_token=None, + cost_json=json.dumps({"lowerBound": BUDGET_COST}), + ), + ) + assert published_bytes(scan) == before + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "running" + + +@pytest.mark.parametrize("reason", ["saturated", "capped"]) +def test_budget_keeps_selection_committed_before_transaction_and_unmerged_obligations( + workbench_api, workbench_db, publication_scan, reason +): + scan = publication_scan() + _, accepted, original_coverage = accept_reducer(workbench_db, scan) + unmerged = [accept_unmerged(workbench_db, scan) for _ in range(2)] + originals = {path: path.read_bytes() for path in [accepted, *unmerged]} + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', workflow_version = 'deep-security-scan/v2'" + ) + # The selected publication may commit before cost recovery reaches its + # transaction. Budget completion must use that exact input and cause. + committed = saved_selection(workbench_db, scan, accepted, reason=reason) + workbench_api["complete_budget_exhausted_scan"]( + workbench_db, + Namespace( + scan_id=scan.scan_id, cost_json=json.dumps(BUDGET_COST), message="Original cost stop." + ), + ) + run = workbench_db.execute("SELECT * FROM deep_scan_runs").fetchone() + recorded = json.loads(run["finalization_input_json"]) + assert {key: recorded[key] for key in committed} == committed + assert len(recorded["publicationSha256"]) == 64 + assert run["terminal_reason"] == reason + assert all(path.read_bytes() == contents for path, contents in originals.items()) + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert len(findings) == 1 + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert original_coverage["deferred"][0] in coverage["deferred"] + gaps = [ + item + for item in coverage["deferred"] + if item.get("provenance", {}).get("sourceId") == "shared-gap" + ] + assert len(gaps) == 2 + assert len({item["id"] for item in gaps}) == 2 + assert {item["provenance"]["workerId"] for item in gaps} == { + path.parent.parent.name for path in unmerged + } + assert all(item["provenance"]["attempt"] == 1 for item in gaps) + + +def test_budget_accepts_complete_reducer_without_optional_complete_flag( + workbench_api, workbench_db, publication_scan +): + scan = publication_scan() + _, accepted, _ = accept_reducer(workbench_db, scan) + document = json.loads(accepted.read_bytes()) + del document["complete"] + accepted.write_text(json.dumps(document)) + digest = hashlib.sha256(accepted.read_bytes()).hexdigest() + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute("UPDATE deep_scan_attempts SET accepted_result_sha256 = ?", (digest,)) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', workflow_version = 'deep-security-scan/v2'" + ) + workbench_api["complete_budget_exhausted_scan"]( + workbench_db, + Namespace( + scan_id=scan.scan_id, cost_json=json.dumps(BUDGET_COST), message="Original cost stop." + ), + ) + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "complete" + assert hashlib.sha256(accepted.read_bytes()).hexdigest() == digest + assert len(json.loads((scan.scan_dir / "findings.json").read_text())["findings"]) == 1 + + +def test_budget_preserves_accepted_scope_and_unmerged_scope_limits( + workbench_api, workbench_db, publication_scan +): + scan = publication_scan() + _, accepted, _ = accept_reducer(workbench_db, scan) + unmerged = accept_unmerged(workbench_db, scan) + scope = { + "summary": "Synthetic upload boundaries.", + "artifactsReviewed": ["subdir/extract.py"], + "runtimeStatus": "Synthetic runtime unavailable.", + "validationMode": "Static validation of the accepted findings.", + "context": "External deployment behavior remains unresolved.", + "limitations": ["The accepted review did not execute the deployment integration."], + } + unmerged_limit = "A separate unmerged review could not inspect the deployment credentials." + for path, value in [(accepted, scope), (unmerged, {"limitations": [unmerged_limit]})]: + document = json.loads(path.read_bytes()) + document["scope"] = value + path.write_text(json.dumps(document)) + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_attempts SET accepted_result_sha256 = ? WHERE accepted_result_path = ?", + (hashlib.sha256(path.read_bytes()).hexdigest(), str(path)), + ) + originals = {p: p.read_bytes() for p in [accepted, unmerged]} + for name in ("scan-manifest.json", "findings.json", "coverage.json"): + (scan.scan_dir / name).unlink() + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', workflow_version = 'deep-security-scan/v2'" + ) + warning = "Scan stopped at its original cost limit before further validation." + workbench_api["complete_budget_exhausted_scan"]( + workbench_db, + Namespace(scan_id=scan.scan_id, cost_json=json.dumps(BUDGET_COST), message=warning), + ) + manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text()) + saved_scope = manifest["scan"]["scope"] + for key, value in scope.items(): + if key == "limitations": + assert all(item in saved_scope[key] for item in value) + else: + assert saved_scope[key] == value + assert warning in saved_scope["limitations"] + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert "completeness" not in coverage + assert any( + item["reason"] == unmerged_limit + and item["provenance"]["workerId"] == unmerged.parent.parent.name + for item in coverage["deferred"] + ) + assert all(p.read_bytes() == contents for p, contents in originals.items()) + assert len(json.loads((scan.scan_dir / "findings.json").read_text())["findings"]) == 1 + + +@pytest.mark.parametrize("has_reducer", [False, True]) +def test_budget_replay_after_draft_commit_keeps_unmerged_coverage_once( + workbench_api, workbench_db, publication_scan, tmp_path, monkeypatch, has_reducer +): + scan = publication_scan() + if has_reducer: + accept_reducer(workbench_db, scan) + unmerged = accept_unmerged(workbench_db, scan) + original = unmerged.read_bytes() + for name in ("scan-manifest.json", "findings.json", "coverage.json"): + (scan.scan_dir / name).unlink() + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', workflow_version = 'deep-security-scan/v2'" + ) + budget = workbench_api["complete_budget_exhausted_scan"] + original_complete = budget.__globals__["complete_scan_locked"] + + def fail_before_seal(*args, **kwargs): + raise RuntimeError("Synthetic process loss after budget draft commit.") + + monkeypatch.setitem(budget.__globals__, "complete_scan_locked", fail_before_seal) + args = Namespace( + scan_id=scan.scan_id, + cost_json=json.dumps(BUDGET_COST), + message="Original cost interruption.", + ) + path = tmp_path / "budget-gap.sqlite3" + with sqlite3.connect(path) as connection: + workbench_db.backup(connection) + connection.row_factory = sqlite3.Row + with pytest.raises(RuntimeError, match="after budget draft commit"): + budget(connection, args) + run = connection.execute("SELECT * FROM deep_scan_runs").fetchone() + assert run["status"] == "succeeded" + selection = run["finalization_input_json"] + assert connection.execute("SELECT status FROM scans").fetchone()[0] == "running" + before = json.loads((scan.scan_dir / "coverage.json").read_text()) + monkeypatch.setitem(budget.__globals__, "complete_scan_locked", original_complete) + with sqlite3.connect(path) as connection: + connection.row_factory = sqlite3.Row + budget(connection, args) + assert connection.execute("SELECT status FROM scans").fetchone()[0] == "complete" + assert ( + connection.execute("SELECT finalization_input_json FROM deep_scan_runs").fetchone()[0] + == selection + ) + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + for key in ("surfaces", "explicitExclusions", "deferred", "reviews"): + assert coverage.get(key, []) == before.get(key, []) + assert len({row["id"] for row in coverage["deferred"]}) == len(coverage["deferred"]) + assert unmerged.read_bytes() == original + + +@pytest.mark.parametrize("selected", [False, True], ids=["legacy-v1", "selected-v2"]) +@pytest.mark.parametrize("reason", ["saturated", "capped"]) +@pytest.mark.parametrize("cancel_first", [False, True], ids=["budget-first", "cancel-first"]) +def test_budget_completion_and_cancel_keep_the_committed_outcome( + workbench_api, workbench_db, publication_scan, tmp_path, selected, reason, cancel_first +): + scan = publication_scan() + _, accepted, coverage = accept_reducer(workbench_db, scan) + selection = saved_selection(workbench_db, scan, accepted, reason=reason) if selected else None + scan.coverage = coverage + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute("UPDATE deep_scan_runs SET terminal_reason = ?", (reason,)) + staged = stage_publication( + scan, generation=3, result_path=accepted, title="Selected accepted aggregate" + ) + database_path = tmp_path / "budget-publication.sqlite3" + with sqlite3.connect(database_path) as connection: + workbench_db.backup(connection) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + workbench_api["saved_results"].write_scan_draft( + workbench_api["_WORKBENCH_DB_CONTEXT"], connection, staged + ) + accepted_bytes = accepted.read_bytes() + warning = "Scan stopped after reaching its configured cost limit." + budget_args = Namespace( + scan_id=scan.scan_id, cost_json=json.dumps(BUDGET_COST), message=warning + ) + cancel_args = Namespace(scan_id=scan.scan_id, thread_id=None) + if cancel_first: + workbench_api["saved_results"].cancel_scan( + workbench_api["_WORKBENCH_DB_CONTEXT"], connection, cancel_args + ) + frozen = published_bytes(scan) + with pytest.raises(SystemExit, match="running"): + workbench_api["complete_budget_exhausted_scan"](connection, budget_args) + else: + workbench_api["complete_budget_exhausted_scan"](connection, budget_args) + frozen = published_bytes(scan) + with pytest.raises(SystemExit, match="running"): + workbench_api["saved_results"].cancel_scan( + workbench_api["_WORKBENCH_DB_CONTEXT"], connection, cancel_args + ) + assert published_bytes(scan) == frozen + + with sqlite3.connect(database_path) as connection: + connection.row_factory = sqlite3.Row + row = connection.execute("SELECT * FROM scans").fetchone() + run = connection.execute("SELECT * FROM deep_scan_runs").fetchone() + assert row["status"] == ("failed" if cancel_first else "complete") + assert bool(row["canceled_at"]) == cancel_first + assert run["terminal_reason"] == reason + recorded = json.loads(run["finalization_input_json"] or "null") + if selection is None: + assert recorded is None + else: + assert {key: recorded[key] for key in selection} == selection + assert len(recorded["publicationSha256"]) == 64 + assert accepted.read_bytes() == accepted_bytes + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert "completeness" not in coverage + assert any(item["id"] == "accepted-follow-up" for item in coverage["deferred"]) + if not cancel_first: + assert any(item["id"] == "scan-cost-limit" for item in coverage["deferred"]) + assert warning in json.loads(row["completion_warnings_json"]) + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert len(findings) == 1 + with pytest.raises(SystemExit, match="stopped"): + workbench_api["saved_results"].write_scan_draft( + workbench_api["_WORKBENCH_DB_CONTEXT"], connection, staged + ) + assert published_bytes(scan) == frozen + + +@pytest.mark.parametrize( + "cut", ["after-findings", "after-coverage", "after-draft-commit", "after-seal"] +) +@pytest.mark.parametrize("has_reducer", [False, True]) +def test_budget_process_death_replays_exact_evidence( + workbench_api, workbench_db, publication_scan, tmp_path, cut, has_reducer +): + scan = publication_scan() + thread_id = "a23e657b-c14c-4da7-bd20-baa9e7579390" + workbench_api["set_scan_thread"]( + workbench_db, Namespace(scan_id=scan.scan_id, thread_id=thread_id) + ) + accepted = None + if has_reducer: + _, accepted, _ = accept_reducer(workbench_db, scan) + unmerged = accept_unmerged(workbench_db, scan) + originals = {str(p): p.read_bytes() for p in [unmerged, *([accepted] if accepted else [])]} + for name in ("scan-manifest.json", "findings.json", "coverage.json"): + (scan.scan_dir / name).unlink() + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute( + "UPDATE deep_scan_runs SET status='running', workflow_version='deep-security-scan/v2'" + ) + database = tmp_path / "budget-process-cut.sqlite3" + with sqlite3.connect(database) as db: + workbench_db.backup(db) + budget = workbench_api["complete_budget_exhausted_scan"] + args = Namespace( + scan_id=scan.scan_id, + cost_json=json.dumps(BUDGET_COST), + message="Original synthetic cost stop.", + ) + marker = tmp_path / "crash-marker.json" + child_source = """ +from argparse import Namespace +from pathlib import Path +import json +import os +import runpy +import sqlite3 +import sys + +script, database, scan_id, cost_json, warning, cut, marker = sys.argv[1:] +api = runpy.run_path(str(script), run_name='fault_review_budget_child') +budget = api['complete_budget_exhausted_scan'] +deep = budget.__globals__['deep_scan'] +from types import SimpleNamespace +deep.configure(SimpleNamespace(**{**api, "preserve_stopped_results": api["preserve_stopped_results_after_transition"]})) + + +def terminate(stage): + Path(marker).write_text(json.dumps({'stage': stage, 'pid': os.getpid()})) + os._exit(86) + + +if cut == 'after-draft-commit': + budget.__globals__['complete_scan_locked'] = lambda *a, **kw: terminate(cut) +elif cut == 'after-seal': + original = budget.__globals__['_write_prepared_scan_finalization'] + + def seal_then_die(prepared, **kwargs): + original(prepared, **kwargs) + terminate(cut) + + budget.__globals__['_write_prepared_scan_finalization'] = seal_then_die +else: + saved = budget.__globals__['saved_results'] + original = saved.write_scan_local_bytes + + def write_then_die(scan_dir, name, contents): + result = original(scan_dir, name, contents) + if name == ('findings.json' if cut == 'after-findings' else 'coverage.json'): + terminate(cut) + return result + + saved.write_scan_local_bytes = write_then_die + +with sqlite3.connect(database) as connection: + connection.row_factory = sqlite3.Row + connection.execute('PRAGMA foreign_keys=ON') + budget(connection, Namespace(scan_id=scan_id, cost_json=cost_json, message=warning)) +os._exit(87) +""" + child = subprocess.run( + [ + sys.executable, + "-c", + child_source, + workbench_api["__file__"], + str(database), + scan.scan_id, + args.cost_json, + args.message, + cut, + str(marker), + ], + capture_output=True, + text=True, + ) + assert child.returncode == 86, (child.returncode, child.stdout, child.stderr) + assert json.loads(marker.read_text())["stage"] == cut + before = { + name: (scan.scan_dir / name).exists() + for name in ("findings.json", "coverage.json", "scan-manifest.json") + } + sealed_bytes = published_bytes(scan) if cut == "after-seal" else None + if sealed_bytes is not None: + assert json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"]["sealedAt"] + with sqlite3.connect(database) as db: + db.row_factory = sqlite3.Row + db.execute("PRAGMA foreign_keys=ON") + run = dict(db.execute("SELECT * FROM deep_scan_runs").fetchone()) + snapshot = { + "cut": cut, + "hasReducer": has_reducer, + "database": str(database), + "scanDir": str(scan.scan_dir), + "beforeFiles": before, + "beforeStatus": run["status"], + "beforeSelection": run["finalization_input_json"], + } + saved_scan = workbench_api["require_scan"](db, scan.scan_id) + resumed = workbench_api["scan_history"].cli_scan_resume( + workbench_api["_WORKBENCH_DB_CONTEXT"], + db, + saved_scan, + workbench_api["require_workspace"](db, saved_scan["workspace_id"]), + ) + assert resumed["scanId"] == scan.scan_id + assert resumed["threadId"] == thread_id + try: + budget(db, args) + except BaseException as error: + snapshot["replayError"] = str(error) + (tmp_path / "crash-result.json").write_text(json.dumps(snapshot, indent=2)) + raise + snapshot["afterStatus"] = db.execute("SELECT status FROM scans").fetchone()[0] + snapshot["afterSelection"] = db.execute( + "SELECT finalization_input_json FROM deep_scan_runs" + ).fetchone()[0] + (tmp_path / "crash-result.json").write_text(json.dumps(snapshot, indent=2)) + assert snapshot["afterStatus"] == "complete" + if sealed_bytes is not None: + assert published_bytes(scan) == sealed_bytes + if run["finalization_input_json"] is not None: + assert snapshot["afterSelection"] == run["finalization_input_json"] + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert len(findings) == int(has_reducer) + assert "completeness" not in coverage + assert len({x["id"] for x in coverage["deferred"]}) == len(coverage["deferred"]) + assert all(Path(path).read_bytes() == data for path, data in originals.items()) + + +@pytest.mark.parametrize("files", ["findings", "coverage", "manifest", "findings-and-coverage"]) +def test_budget_rejects_unrelated_incomplete_drafts( + workbench_api, workbench_db, publication_scan, files +): + scan = publication_scan() + accept_unmerged(workbench_db, scan) + keep = { + "findings": {"findings.json"}, + "coverage": {"coverage.json"}, + "manifest": {"scan-manifest.json"}, + "findings-and-coverage": {"findings.json", "coverage.json"}, + }[files] + for name in ("scan-manifest.json", "findings.json", "coverage.json"): + if name not in keep: + (scan.scan_dir / name).unlink() + before = {name: (scan.scan_dir / name).read_bytes() for name in keep} + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', workflow_version = 'deep-security-scan/v2'" + ) + with pytest.raises(SystemExit, match="incomplete canonical scan draft"): + workbench_api["complete_budget_exhausted_scan"]( + workbench_db, + Namespace( + scan_id=scan.scan_id, + cost_json=json.dumps(BUDGET_COST), + message="Original cost stop.", + ), + ) + assert {name: (scan.scan_dir / name).read_bytes() for name in keep} == before + assert all( + not (scan.scan_dir / name).exists() + for name in ("scan-manifest.json", "findings.json", "coverage.json") + if name not in keep + ) + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "running" + + +@pytest.mark.parametrize( + "state", + [ + "changed-findings", + "late-findings", + "wrong-scan", + "running-discovery", + "canceled", + "other-owner", + "complete", + ], +) +def test_budget_sealed_replay_keeps_integrity_and_ownership_guards( + workbench_api, workbench_db, publication_scan, monkeypatch, state +): + scan = publication_scan() + accept_reducer(workbench_db, scan) + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', workflow_version = 'deep-security-scan/v2'" + ) + budget = workbench_api["complete_budget_exhausted_scan"] + complete = budget.__globals__["complete_scan_locked"] + args = Namespace( + scan_id=scan.scan_id, cost_json=json.dumps(BUDGET_COST), message="Original cost stop." + ) + with monkeypatch.context() as patch: + patch.setitem( + budget.__globals__, + "complete_scan_locked", + lambda *a, **kw: complete(*a, **kw, prepare_only=True), + ) + budget(workbench_db, args) + manifest = scan.scan_dir / "scan-manifest.json" + assert json.loads(manifest.read_text())["scan"]["sealedAt"] + if state in {"changed-findings", "wrong-scan"}: + path = scan.scan_dir / ( + "findings.json" if state == "changed-findings" else "scan-manifest.json" + ) + document = json.loads(path.read_text()) + if state == "changed-findings": + document["findings"] = [] + else: + document["scan"]["id"] = "69078890-d24c-4416-a6fa-c286825bef88" + path.write_text(json.dumps(document)) + elif state == "late-findings": + + def change_before_completion(*a, **kw): + path = scan.scan_dir / "findings.json" + document = json.loads(path.read_text()) + document["findings"] = [] + path.write_text(json.dumps(document)) + before_files["findings.json"] = path.read_bytes() + return complete(*a, **kw) + + monkeypatch.setitem(budget.__globals__, "complete_scan_locked", change_before_completion) + elif state == "running-discovery": + with workbench_db: + workbench_db.execute("UPDATE deep_scan_runs SET status = 'running'") + elif state == "canceled": + workbench_api["saved_results"].cancel_scan( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + Namespace(scan_id=scan.scan_id, thread_id=None), + ) + elif state == "other-owner": + with workbench_db: + workbench_db.execute( + "UPDATE scans SET handoff_claim_token = 'a3292ae4-9b47-430f-8ed6-73ff73db575c'" + ) + else: + complete(workbench_db, scan.scan_id, None, args.cost_json) + before_files = published_bytes(scan) + before_scan = dict(workbench_db.execute("SELECT * FROM scans").fetchone()) + before_run = dict(workbench_db.execute("SELECT * FROM deep_scan_runs").fetchone()) + with pytest.raises(SystemExit): + budget(workbench_db, args) + assert published_bytes(scan) == before_files + assert dict(workbench_db.execute("SELECT * FROM scans").fetchone()) == before_scan + assert dict(workbench_db.execute("SELECT * FROM deep_scan_runs").fetchone()) == before_run + + +@pytest.mark.parametrize( + "state", ["parent-canceled", "stopping", "failed", "canceled", "unselected"] +) +def test_budget_resume_keeps_explicit_stop_and_unselected_guards( + workbench_api, workbench_db, publication_scan, state +): + scan = publication_scan() + workbench_api["set_scan_thread"]( + workbench_db, + Namespace(scan_id=scan.scan_id, thread_id="a23e657b-c14c-4da7-bd20-baa9e7579390"), + ) + with workbench_db: + if state == "parent-canceled": + workbench_db.execute("UPDATE scans SET canceled_at = ?", (scan.timestamp,)) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = ?, cancel_requested = 1, finalization_input_json = NULL", + ( + "running" + if state == "stopping" + else state + if state in {"failed", "canceled"} + else "succeeded", + ), + ) + saved_scan = workbench_api["require_scan"](workbench_db, scan.scan_id) + with pytest.raises(SystemExit, match="cannot resume"): + workbench_api["scan_history"].cli_scan_resume( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + saved_scan, + workbench_api["require_workspace"](workbench_db, saved_scan["workspace_id"]), + ) + + +@pytest.mark.parametrize("changed", [False, True], ids=["unchanged", "changed-publication"]) +def test_budget_selection_binds_publication_in_its_commit( + workbench_api, workbench_db, publication_scan, tmp_path, changed +): + scan = publication_scan() + accept_reducer(workbench_db, scan) + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', phase = 'discovery', " + "workflow_version = 'deep-security-scan/v2', finalization_input_json = NULL" + ) + database = tmp_path / "budget-commit.sqlite3" + with sqlite3.connect(database) as connection: + workbench_db.backup(connection) + child_program = """ +import json, os, runpy, sqlite3, sys +from argparse import Namespace +from types import SimpleNamespace +api = runpy.run_path(sys.argv[1], run_name="budget_publication_commit_loss") +api["deep_scan"].configure(SimpleNamespace(**{**api, "preserve_stopped_results": api["preserve_stopped_results_after_transition"]})) +class CrashConnection(sqlite3.Connection): + def commit(self): + super().commit() + os._exit(73) +connection = sqlite3.connect(sys.argv[2], factory=CrashConnection) +connection.row_factory = sqlite3.Row +connection.execute("PRAGMA foreign_keys = ON") +api["complete_budget_exhausted_scan"](connection, Namespace(scan_id=sys.argv[3], cost_json=sys.argv[4], message=None)) +raise AssertionError("budget selection did not commit") +""" + child = subprocess.run( + [ + sys.executable, + "-c", + child_program, + workbench_api["__file__"], + str(database), + scan.scan_id, + json.dumps(BUDGET_COST), + ], + capture_output=True, + text=True, + ) + assert child.returncode == 73, child.stderr + findings_path = scan.scan_dir / "findings.json" + findings = json.loads(findings_path.read_text()) + assert findings["findings"] + if changed: + findings["findings"][0]["title"] = "Synthetic substituted publication finding" + findings_path.write_text(json.dumps(findings)) + with sqlite3.connect(database) as connection: + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + args = Namespace(scan_id=scan.scan_id, cost_json=json.dumps(BUDGET_COST), message=None) + if changed: + with pytest.raises(SystemExit, match="selected Deep Scan publication"): + workbench_api["complete_budget_exhausted_scan"](connection, args) + assert connection.execute("SELECT status FROM scans").fetchone()[0] == "running" + else: + completed = workbench_api["complete_budget_exhausted_scan"](connection, args) + assert completed["scan"]["scanId"] == scan.scan_id + assert connection.execute("SELECT status FROM scans").fetchone()[0] == "complete" + + +@pytest.mark.parametrize("selected", [False, True]) +@pytest.mark.parametrize("drift", ["edited", "moved", "deleted"]) +def test_selected_resume_retains_saved_scope_when_checkout_changes( + workbench_api, workbench_db, publication_scan, selected, drift +): + scan = publication_scan(scope="subdir") + thread_id = "a23e657b-c14c-4da7-bd20-baa9e7579390" + workbench_api["set_scan_thread"]( + workbench_db, Namespace(scan_id=scan.scan_id, thread_id=thread_id) + ) + if selected: + _, accepted, _ = accept_reducer(workbench_db, scan) + saved_selection(workbench_db, scan, accepted) + saved_scan = workbench_api["require_scan"](workbench_db, scan.scan_id) + repository = Path(saved_scan["target_path"]) + if drift == "edited": + (repository / "subdir" / "extract.py").write_text("# Changed after selection\n") + elif drift == "moved": + repository.rename(repository.with_name("moved")) + else: + shutil.rmtree(repository) + args = ( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + saved_scan, + workbench_api["require_workspace"](workbench_db, saved_scan["workspace_id"]), + ) + if not selected: + with pytest.raises(SystemExit, match="Cannot resume"): + workbench_api["scan_history"].cli_scan_resume(*args) + return + resumed = workbench_api["scan_history"].cli_scan_resume(*args) + assert resumed["selectedFinalization"] is True + assert resumed["threadId"] == thread_id + assert resumed["recipe"]["repository"] == str(repository) + assert resumed["recipe"]["target"] == {"kind": "paths", "paths": ["subdir"]} + assert resumed["targetRevision"] == saved_scan["target_revision"] diff --git a/plugins/codex-security/tests/test_checkpoint_publication_authority.py b/plugins/codex-security/tests/test_checkpoint_publication_authority.py new file mode 100644 index 0000000000..6f10ff36fe --- /dev/null +++ b/plugins/codex-security/tests/test_checkpoint_publication_authority.py @@ -0,0 +1,427 @@ +from __future__ import annotations + +import copy +import json +import os +import uuid +from argparse import Namespace + +import pytest +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan +from test_workbench_standard_deep_results import deep_scan_fixture, worker_paths +from workbench_test_support import run_workbench, write_checkpoint, write_completed_contract + + +def test_public_stop_retains_accepted_partial_evidence_and_newer_rejection(tmp_path): + state, home, target, scan_dir, scan_id = deep_scan_fixture(tmp_path, workers=2) + environment = {"CODEX_HOME": str(home)} + contract = tmp_path / "contract" + contract.mkdir() + write_completed_contract(contract, scan_id, target, relative_path="app.py") + finding = json.loads((contract / "findings.json").read_text())["findings"][0] + deferred = { + "candidateId": "pending-query", + "reason": "Validation is pending.", + "paths": ["app.py"], + } + coverage = { + "completeness": "partial", + "surfaces": [], + "explicitExclusions": [], + "deferred": [deferred], + } + workers = [] + for accepted in (True, False): + name = "accepted" if accepted else "interrupted" + prompt, output, result = worker_paths(scan_dir, name) + worker_id = str(uuid.uuid4()) + worker_args = ( + "upsert-deep-scan-worker", + "--scan-id", + scan_id, + "--worker-id", + worker_id, + "--kind", + "discovery", + "--prompt-path", + str(prompt), + "--artifact-dir", + str(output), + "--attempt", + "1", + ) + run_workbench(state, *worker_args, "--status", "running", environment=environment) + current = copy.deepcopy(finding) + current["identity"]["anchor"] = name + current["provenance"]["candidateId"] = name + current["extensions"] = {"candidateId": name} + draft = {"scanId": scan_id, "complete": True, "findings": [current], "coverage": coverage} + result.write_text(json.dumps(draft)) + write_checkpoint(output / "checkpoints", draft) + if accepted: + run_workbench( + state, + *worker_args, + "--status", + "succeeded", + "--result-manifest-path", + str(result), + environment=environment, + ) + else: + rejected = { + **draft, + "complete": False, + "findings": [], + "coverage": { + **coverage, + "surfaces": [ + { + "candidateId": name, + "label": "Reviewed candidate", + "disposition": "rejected", + "receiptRefs": [], + } + ], + }, + } + head = write_checkpoint(output / "checkpoints", rejected) + (output / "checkpoint-head.json").write_text(json.dumps({"checkpoint": head.name})) + workers.append(worker_id) + + stopped = run_workbench( + state, + "fail-deep-scan", + "--scan-id", + scan_id, + "--message", + "Original worker failure.", + "--deep-status", + "interrupted", + environment=environment, + )["deepScan"] + assert stopped["status"] == "interrupted" + scan = run_workbench(state, "get-scan", "--scan-id", scan_id)["scan"] + assert scan["findingCount"] == 1 + findings = json.loads((scan_dir / "findings.json").read_text())["findings"] + assert [item["identity"]["anchor"] for item in findings] == ["accepted"] + retained_coverage = json.loads((scan_dir / "coverage.json").read_text()) + assert "completeness" not in retained_coverage + assert any(item.get("candidateId") == "pending-query" for item in retained_coverage["deferred"]) + assert any( + item.get("candidateId") == "interrupted" and item["disposition"] == "rejected" + for item in retained_coverage["surfaces"] + ) + assert scan["failureMessage"] == "Original worker failure." + assert {worker["id"]: worker["status"] for worker in stopped["workers"]} == { + workers[0]: "succeeded", + workers[1]: "canceled", + } + + +@pytest.mark.parametrize("archived", [False, True], ids=["current", "archived"]) +@pytest.mark.parametrize("has_head", [True, False], ids=["committed-head", "legacy"]) +@pytest.mark.parametrize("complete", [False, True], ids=["checkpoint", "complete"]) +def test_recovery_honors_rejection_committed_before_result_replacement( + workbench_api, workbench_db, publication_scan, archived, has_head, complete +): + scan = publication_scan() + provisional = copy.deepcopy(scan.findings[0]) + provisional["extensions"] = {"candidateId": "candidate-rejected"} + retained = copy.deepcopy(scan.findings[0]) + retained["identity"]["anchor"] = "independent-finding" + retained["extensions"] = {"candidateId": "candidate-retained"} + retained["locations"][0]["startLine"] = 20 + retained["locations"][0]["endLine"] = 21 + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + result_path = add_worker(workbench_db, scan, status="canceled") + if archived: + result_path = result_path.parent / "attempts" / "attempt-1" / "result.json" + result_path.parent.mkdir(parents=True) + previous = { + "scanId": scan.scan_id, + "complete": True, + "findings": [provisional, retained], + "coverage": scan.coverage, + } + result_path.write_text(json.dumps(previous)) + old_checkpoint = write_checkpoint(result_path.parent / "checkpoints", previous) + rejected = { + **previous, + "complete": complete, + "findings": [retained], + "coverage": { + **scan.coverage, + "surfaces": [ + { + "candidateId": "candidate-rejected", + "label": "Validated candidate disposition", + "disposition": "rejected", + "receiptRefs": [], + } + ], + }, + } + checkpoint = write_checkpoint(result_path.parent / "checkpoints", rejected) + if has_head: + head = result_path.parent / "checkpoint-head.json" + head.write_text(json.dumps({"checkpoint": checkpoint.name})) + observed = result_path.stat().st_mtime_ns + 1_000_000_000 + os.utime(head, ns=(observed, observed)) + assert head.stat().st_mtime_ns > result_path.stat().st_mtime_ns + saved_bytes = {path: path.read_bytes() for path in (result_path, old_checkpoint, checkpoint)} + + stopped = workbench_api["saved_results"].fail_scan( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."), + )["scan"] + + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert stopped["findingCount"] == len(findings) == (1 if has_head else 2) + assert any(finding["identity"]["anchor"] == "independent-finding" for finding in findings) + assert all(path.read_bytes() == contents for path, contents in saved_bytes.items()) + if has_head: + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert any( + surface.get("candidateId") == "candidate-rejected" + and surface.get("disposition") == "rejected" + for surface in coverage["surfaces"] + ) + + +def save_disposition(scan, directory, disposition): + directory.mkdir(parents=True, exist_ok=True) + finding = copy.deepcopy(scan.findings[0]) + finding["extensions"] = {"candidateId": "candidate-disposition"} + draft = { + "scanId": scan.scan_id, + "complete": True, + "findings": [finding] if disposition == "reported" else [], + "coverage": { + **scan.coverage, + "surfaces": [ + { + "candidateId": "candidate-disposition", + "label": "Validated candidate disposition", + "disposition": disposition, + "receiptRefs": [], + } + ], + }, + } + checkpoint = write_checkpoint(directory / "checkpoints", draft) + (directory / "checkpoint-head.json").write_text(json.dumps({"checkpoint": checkpoint.name})) + return draft + + +@pytest.mark.parametrize("archived", [False, True], ids=["current-head", "newer-archive"]) +@pytest.mark.parametrize("disposition", ["reported", "rejected"]) +def test_newer_checkpoint_disposition_precedes_older_archived_head( + workbench_api, workbench_db, publication_scan, archived, disposition +): + scan = publication_scan() + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + result = add_worker(workbench_db, scan, status="canceled") + old = result.parent / "attempts" / "attempt-2" + save_disposition(scan, old, "rejected" if disposition == "reported" else "reported") + current = result.parent / "attempts" / "attempt-10" if archived else result.parent + if not archived: + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_workers SET attempt = 3 WHERE scan_id = ?", (scan.scan_id,) + ) + draft = save_disposition(scan, current, disposition) + (current / "result.json").write_text(json.dumps(draft)) + + stopped = workbench_api["saved_results"].fail_scan( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."), + )["scan"] + + assert stopped["findingCount"] == (1 if disposition == "reported" else 0) + + +@pytest.mark.parametrize("head_change", ["replaced", "removed", "missing-checkpoint"]) +def test_frozen_stopped_replay_ignores_later_worker_head_changes( + workbench_api, workbench_db, publication_scan, monkeypatch, head_change +): + import finalize_scan_contract + + scan = publication_scan() + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + result = add_worker(workbench_db, scan, status="canceled") + previous = save_disposition(scan, result.parent, "reported") + result.write_text(json.dumps(previous)) + save_disposition(scan, result.parent, "rejected") + # The rejected head was committed after the result, before its replacement failed. + head = result.parent / "checkpoint-head.json" + observed = result.stat().st_mtime_ns + 1_000_000_000 + os.utime(head, ns=(observed, observed)) + assert head.stat().st_mtime_ns > result.stat().st_mtime_ns + checkpoint_name = json.loads(head.read_text())["checkpoint"] + directory = result.parent.relative_to(scan.scan_dir).as_posix() + expected_heads = {directory: f"{directory}/checkpoints/{checkpoint_name}"} + original_outputs = { + name: (scan.scan_dir / name).read_bytes() + for name in ("findings.json", "coverage.json", "scan-manifest.json") + } + write_bytes = finalize_scan_contract.write_scan_local_bytes + failed_writes = [] + + def fail_coverage_write(directory, relative, payload, **kwargs): + if relative != "coverage.json" or failed_writes: + return write_bytes(directory, relative, payload, **kwargs) + # Exercise the real writer after findings have reached disk. Remove the + # temporary obstruction before the publisher restores its old outputs. + failed_writes.append(json.loads((directory / "findings.json").read_text())) + path = directory / relative + previous_bytes = path.read_bytes() + path.unlink() + path.mkdir() + try: + return write_bytes(directory, relative, payload, **kwargs) + finally: + path.rmdir() + path.write_bytes(previous_bytes) + + with monkeypatch.context() as patch: + patch.setattr(finalize_scan_contract, "write_scan_local_bytes", fail_coverage_write) + workbench_api["saved_results"].fail_scan( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + Namespace( + scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped." + ), + ) + row = workbench_db.execute("SELECT * FROM scans WHERE id = ?", (scan.scan_id,)).fetchone() + assert len(failed_writes) == 1 + assert "scanId" in failed_writes[0] + assert row["status"] == "failed" + assert row["failure_message"] == "Audit stopped." + assert row["retained_source_digests_json"] + assert row["seal_manifest_digest"] is None + assert all( + (scan.scan_dir / name).read_bytes() == contents + for name, contents in original_outputs.items() + ) + original_sources = row["retained_source_digests_json"] + original_run = dict( + workbench_db.execute( + "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan.scan_id,) + ).fetchone() + ) + head = result.parent / "checkpoint-head.json" + if head_change == "replaced": + save_disposition(scan, result.parent, "reported") + elif head_change == "removed": + head.unlink() + else: + head.write_text(json.dumps({"checkpoint": "a" * 64 + ".json"})) + + replayed = workbench_api["saved_results"].preserve_scan_results( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + Namespace( + scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None + ), + )["scan"] + + assert replayed["findingCount"] == 0 + assert json.loads((scan.scan_dir / "findings.json").read_text())["findings"] == [] + assert json.loads(result.read_text()) == previous + row = workbench_db.execute("SELECT * FROM scans WHERE id = ?", (scan.scan_id,)).fetchone() + assert row["failure_message"] == "Audit stopped." + assert row["retained_source_digests_json"] == original_sources + assert json.loads(row["retained_checkpoint_heads_json"]) == expected_heads + assert row["seal_manifest_digest"] + manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text()) + assert manifest["scan"]["preservedCheckpointHeads"] == expected_heads + assert ( + dict( + workbench_db.execute( + "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan.scan_id,) + ).fetchone() + ) + == original_run + ) + + +def test_explicit_recovery_observes_head_change_between_existing_checkpoints( + workbench_api, workbench_db, publication_scan +): + scan = publication_scan() + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + result = add_worker(workbench_db, scan, status="canceled") + previous = save_disposition(scan, result.parent, "reported") + result.write_text(json.dumps(previous)) + save_disposition(scan, result.parent, "rejected") + head = result.parent / "checkpoint-head.json" + rejected_at = result.stat().st_mtime_ns + 1_000_000_000 + os.utime(head, ns=(rejected_at, rejected_at)) + assert head.stat().st_mtime_ns > result.stat().st_mtime_ns + stopped = workbench_api["saved_results"].fail_scan( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."), + )["scan"] + assert stopped["findingCount"] == 0 + + save_disposition(scan, result.parent, "reported") + reported_at = rejected_at + 1_000_000_000 + os.utime(head, ns=(reported_at, reported_at)) + assert head.stat().st_mtime_ns > rejected_at + + context = workbench_api["scan_context"](workbench_db, scan.scan_id)["scan"] + assert context["resultsRecoveryNeeded"] is True + recovered = workbench_api["saved_results"].recover_scan_results( + workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id) + )["scan"] + assert recovered["findingCount"] == 1 + assert recovered["resultsRecoveryNeeded"] is False + + +def test_legacy_frozen_publication_keeps_result_fallback_without_saved_heads( + workbench_api, workbench_db, publication_scan, monkeypatch +): + scan = publication_scan() + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + result = add_worker(workbench_db, scan, status="canceled") + previous = save_disposition(scan, result.parent, "reported") + result.write_text(json.dumps(previous)) + save_disposition(scan, result.parent, "rejected") + (result.parent / "checkpoint-head.json").unlink() + + def fail_before_publication(*args, **kwargs): + raise OSError("Synthetic publication interruption") + + with monkeypatch.context() as patch: + patch.setattr( + workbench_api["saved_results"], + "_write_prepared_scan_finalization", + fail_before_publication, + ) + workbench_api["saved_results"].fail_scan( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + Namespace( + scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped." + ), + ) + with workbench_db: + workbench_db.execute( + "UPDATE scans SET retained_checkpoint_heads_json = NULL WHERE id = ?", (scan.scan_id,) + ) + save_disposition(scan, result.parent, "rejected") + + replayed = workbench_api["saved_results"].preserve_scan_results( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + Namespace( + scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None + ), + )["scan"] + + assert replayed["findingCount"] == 1 diff --git a/plugins/codex-security/tests/test_checkpoint_schema.py b/plugins/codex-security/tests/test_checkpoint_schema.py new file mode 100644 index 0000000000..be44a6426c --- /dev/null +++ b/plugins/codex-security/tests/test_checkpoint_schema.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import json +import sqlite3 +from contextlib import closing + +import pytest + + +@pytest.mark.parametrize("upgrade", [False, True], ids=["fresh", "upgrade"]) +def test_frozen_checkpoint_head_migration_preserves_scan_state(workbench_api, upgrade): + migrations = workbench_api["MIGRATIONS"] + checkpoint_version = next( + version + for version, name, _ in migrations + if name == "freeze stopped scan checkpoint selections" + ) + timestamp = "2026-07-01T00:00:00Z" + + def migrate(connection, selected): + workbench_api["apply_schema_migrations"]( + connection, selected, lambda: timestamp, workbench_api["backfill_security_targets"] + ) + + with closing(sqlite3.connect(":memory:")) as connection: + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + migrate( + connection, + tuple(item for item in migrations if item[0] != checkpoint_version) + if upgrade + else migrations, + ) + connection.execute( + "INSERT INTO workspaces (id, created_at, updated_at) VALUES (?, ?, ?)", + ("workspace", timestamp, timestamp), + ) + for status in ("running", "failed"): + connection.execute( + "INSERT INTO scans (id, workspace_id, target_path, target_revision, scope, " + "mode, scan_dir, status, phase, started_at, created_at, updated_at, " + "failure_message, retained_source_digests_json) " + "VALUES (?, 'workspace', 'target', 'revision', '.', 'deep', ?, ?, " + "'discovery', ?, ?, ?, ?, ?)", + ( + status, + f"scans/{status}", + status, + timestamp, + timestamp, + timestamp, + "Original stop reason." if status == "failed" else None, + json.dumps({"workers/review/result.json": "b" * 64}) + if status == "failed" + else None, + ), + ) + before = [dict(row) for row in connection.execute("SELECT * FROM scans ORDER BY id")] + migrate(connection, migrations) + after = [dict(row) for row in connection.execute("SELECT * FROM scans ORDER BY id")] + for original, updated in zip(before, after, strict=True): + assert updated.pop("retained_checkpoint_heads_json") is None + original.pop("retained_checkpoint_heads_json", None) + assert updated == original + + heads = json.dumps({"workers/review": "workers/review/checkpoints/" + "a" * 64 + ".json"}) + connection.execute( + "UPDATE scans SET retained_checkpoint_heads_json = ? WHERE id = 'failed'", (heads,) + ) + migrate(connection, migrations) + assert ( + connection.execute( + "SELECT retained_checkpoint_heads_json FROM scans WHERE id = 'failed'" + ).fetchone()[0] + == heads + ) + assert ( + connection.execute( + "SELECT retained_checkpoint_heads_json FROM scans WHERE id = 'running'" + ).fetchone()[0] + is None + ) + assert [ + tuple(row) + for row in connection.execute( + "SELECT version, name FROM schema_migrations WHERE version = ?", + (checkpoint_version,), + ) + ] == [(checkpoint_version, "freeze stopped scan checkpoint selections")] + assert connection.execute("PRAGMA foreign_key_check").fetchall() == [] diff --git a/plugins/codex-security/tests/test_deep_scan_compatibility.py b/plugins/codex-security/tests/test_deep_scan_compatibility.py new file mode 100644 index 0000000000..b9b46f7266 --- /dev/null +++ b/plugins/codex-security/tests/test_deep_scan_compatibility.py @@ -0,0 +1,388 @@ +"""Compatibility checks use real persisted scans and preserve unsupported state.""" + +from __future__ import annotations + +import json +import os +import sqlite3 +import subprocess +import sys +import uuid +from pathlib import Path + +import pytest +from workbench_test_support import SCRIPT, run_workbench + + +def snapshot(state_dir: Path) -> str: + with sqlite3.connect(state_dir / "workbench.sqlite3") as connection: + return "\n".join(connection.iterdump()) + + +def claim_requiring_original_settings( + state: Path, scan_id: str +) -> subprocess.CompletedProcess[str]: + # Exercise the private MCP requirement against both the parent and fixed + # workbench, without adding a public CLI argument. + return subprocess.run( + [ + sys.executable, + "-c", + "\n".join( + [ + "import runpy, sys", + "script = sys.argv.pop(1)", + "main = runpy.run_path(script)['main']", + "namespace = main.__globals__", + "parse = namespace['parse_args']", + "def parse_with_requirement(*args, **kwargs):", + " result = parse(*args, **kwargs)", + " result.require_execution_settings = True", + " return result", + "namespace['parse_args'] = parse_with_requirement", + "main()", + ] + ), + str(SCRIPT), + "claim-deep-scan-coordinator", + "--scan-id", + scan_id, + "--thread-id", + "fixture-thread", + ], + env={**os.environ, "CODEX_SECURITY_STATE_DIR": str(state)}, + capture_output=True, + text=True, + timeout=30, + ) + + +@pytest.mark.parametrize( + "workflow,settings_version", + [ + ("deep-security-scan/v1", 99), + ("deep-scan-mcp/v1", 99), + ("deep-security-scan/v2", 99), + ("deep-security-scan/v2", None), + ], +) +def test_missing_or_unsupported_settings_reject_before_takeover( + tmp_path: Path, workflow: str, settings_version: int | None +) -> None: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + run = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "fixture-thread", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + "--workflow-version", + workflow, + )["deepScan"] + with sqlite3.connect(state / "workbench.sqlite3") as connection: + connection.execute( + "UPDATE deep_scan_runs SET coordinator_generation = 2, " + "phase = 'discovery', updated_at = '2000-01-01T00:00:00Z'" + ) + settings_path = Path(run["scanDir"]) / "artifacts/deep_discovery/execution-settings.json" + saved = None + if settings_version is not None: + settings_path.parent.mkdir(parents=True, exist_ok=True) + saved = json.dumps( + { + "version": settings_version, + "settings": {"codexPath": "/fixture/codex", "codexHome": "/fixture/home"}, + } + ).encode() + settings_path.write_bytes(saved) + with sqlite3.connect(state / "workbench.sqlite3") as connection: + connection.execute( + "UPDATE deep_scan_runs SET execution_settings_json = ? WHERE scan_id = ?", + (saved.decode(), run["scanId"]), + ) + before = snapshot(state) + result = claim_requiring_original_settings(state, run["scanId"]) + assert snapshot(state) == before, ( + "settings rejection must precede ownership and worker recovery" + ) + assert result.returncode != 0 + assert "execution settings" in result.stderr + assert (settings_path.read_bytes() if settings_path.exists() else None) == saved + + +@pytest.mark.parametrize("workflow", ["deep-security-scan/v1", "deep-scan-mcp/v1"]) +def test_legacy_takeover_does_not_require_or_create_a_new_snapshot( + tmp_path: Path, workflow: str +) -> None: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + run = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "fixture-thread", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + "--workflow-version", + workflow, + )["deepScan"] + result = claim_requiring_original_settings(state, run["scanId"]) + assert result.returncode == 0, result.stderr + observed = json.loads(result.stdout)["deepScan"] + assert observed["workflowVersion"] == workflow + assert observed["config"] == run["config"] + assert not (Path(run["scanDir"]) / "artifacts/deep_discovery/execution-settings.json").exists() + + +@pytest.mark.parametrize("completion_only", [False, True]) +def test_observation_and_selected_completion_do_not_require_worker_settings( + tmp_path: Path, completion_only: bool +) -> None: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + run = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "fixture-thread", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + )["deepScan"] + claim = run_workbench( + state, + "claim-deep-scan-coordinator", + "--scan-id", + run["scanId"], + "--thread-id", + "fixture-thread", + ) + if completion_only: + selection = { + "version": 1, + "resultPath": None, + "resultSha256": None, + "terminalReason": "capped", + "omittedWorkerIds": [], + "selectedAt": run["createdAt"], + } + with sqlite3.connect(state / "workbench.sqlite3") as connection: + connection.execute( + "UPDATE deep_scan_runs SET finalization_input_json = ?, " + "updated_at = '2000-01-01T00:00:00Z'", + (json.dumps(selection),), + ) + before = snapshot(state) + result = claim_requiring_original_settings(state, run["scanId"]) + assert result.returncode == 0, result.stderr + observed = json.loads(result.stdout) + if completion_only: + assert observed["coordinatorDisposition"] == "adopted" + assert observed["deepScan"]["finalizationInput"] == selection + else: + assert observed["coordinatorDisposition"] == "observing" + assert ( + observed["deepScan"]["coordinatorGeneration"] + == claim["deepScan"]["coordinatorGeneration"] + ) + assert snapshot(state) == before + + +@pytest.mark.parametrize( + "version", ["deep-security-scan/v1", "deep-scan-mcp/v1", "deep-security-scan/v2"] +) +def test_supported_workflows_keep_their_identity(tmp_path: Path, version: str) -> None: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + begun = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "fixture-thread", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + "--workflow-version", + version, + )["deepScan"] + claimed = run_workbench( + state, + "claim-deep-scan-coordinator", + "--scan-id", + str(begun["scanId"]), + "--thread-id", + "fixture-thread", + )["deepScan"] + assert claimed["workflowVersion"] == version + assert claimed["schemaVersion"] == 1 + assert claimed["noNewStreak"] == begun["noNewStreak"] + assert claimed["config"] == begun["config"] + + +@pytest.mark.parametrize("field,value", [("workflow_version", "future/v99")]) +@pytest.mark.parametrize("operation", ["begin", "claim", "handoff"]) +def test_unsupported_execution_does_not_mutate( + tmp_path: Path, + field: str, + value: str | int, + operation: str, +) -> None: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + begun = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "fixture-thread", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + )["deepScan"] + scan_id = str(begun["scanId"]) + if operation == "claim": + artifact_dir = Path(str(begun["scanDir"])) / "artifacts" / "deep_discovery" / "worker" + artifact_dir.mkdir(parents=True) + prompt = artifact_dir / "prompt.md" + prompt.write_text("Original discovery input") + run_workbench( + state, + "upsert-deep-scan-worker", + "--scan-id", + scan_id, + "--worker-id", + str(uuid.uuid4()), + "--kind", + "discovery", + "--status", + "running", + "--prompt-path", + str(prompt), + "--artifact-dir", + str(artifact_dir), + "--attempt", + "1", + ) + with sqlite3.connect(state / "workbench.sqlite3") as connection: + connection.execute( + f"UPDATE deep_scan_runs SET {field} = ?, updated_at = ?", + (value, "2000-01-01T00:00:00Z"), + ) + if operation == "handoff": + connection.execute( + "UPDATE scans SET deep_scan_owner_thread_id = NULL, recipe_json = '{}'" + ) + connection.execute("UPDATE workspaces SET thread_id = NULL") + before = snapshot(state) + command = "claim-deep-scan-coordinator" if operation == "claim" else "begin-deep-scan" + result = run_workbench( + state, + command, + "--scan-id", + scan_id, + "--thread-id", + "fixture-thread", + *(["--model", "observer-model"] if operation != "claim" else []), + check=False, + ) + assert result["returncode"] != 0 + assert "unsupported" in str(result["stderr"]).lower() + assert snapshot(state) == before + if operation != "handoff": + observed = run_workbench( + state, "get-deep-scan", "--scan-id", scan_id, "--thread-id", "fixture-thread" + )["deepScan"] + assert ( + observed["workflowVersion" if field == "workflow_version" else "schemaVersion"] == value + ) + assert snapshot(state) == before + + +@pytest.mark.parametrize("original", [None, "Original discovery context"]) +def test_reader_honors_original_context_when_present(tmp_path: Path, original: str | None) -> None: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + begun = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "fixture-thread", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + )["deepScan"] + with sqlite3.connect(state / "workbench.sqlite3") as connection: + columns = {row[1] for row in connection.execute("PRAGMA table_info(deep_scan_runs)")} + if "discovery_user_context" not in columns: + connection.execute("ALTER TABLE deep_scan_runs ADD COLUMN discovery_user_context TEXT") + connection.execute( + "UPDATE deep_scan_runs SET discovery_user_context = ?, discovery_user_context_json = NULL", + (original,), + ) + connection.execute("UPDATE scans SET user_context = 'Later discussion'") + observed = run_workbench( + state, + "get-deep-scan", + "--scan-id", + str(begun["scanId"]), + "--thread-id", + "fixture-thread", + )["deepScan"] + assert observed["userContext"] == original + + +def test_unsupported_new_workflow_does_not_claim_registered_scan(tmp_path: Path) -> None: + state = tmp_path / "state" + target = tmp_path / "target" + target.mkdir() + scan_dir = tmp_path / "scan" + scan_dir.mkdir(mode=0o700) + registered = run_workbench( + state, + "register-cli-scan", + "--scan-dir", + str(scan_dir), + "--repository", + str(target), + "--registration-json-stdin", + input_text=json.dumps( + { + "recipe": { + "config": {}, + "mode": "deep", + "repository": str(target), + "target": {"kind": "repository", "paths": []}, + } + } + ), + ) + before = snapshot(state) + rejected = run_workbench( + state, + "begin-deep-scan", + "--scan-id", + str(registered["scanId"]), + "--thread-id", + "fixture-thread", + "--workflow-version", + "future/v99", + check=False, + ) + assert rejected["returncode"] != 0 + assert "unsupported" in str(rejected["stderr"]).lower() + assert snapshot(state) == before diff --git a/plugins/codex-security/tests/test_deep_scan_finalization_compatibility.py b/plugins/codex-security/tests/test_deep_scan_finalization_compatibility.py new file mode 100644 index 0000000000..6e1cd3a026 --- /dev/null +++ b/plugins/codex-security/tests/test_deep_scan_finalization_compatibility.py @@ -0,0 +1,123 @@ +"""Selected finalization survives ownership recovery without restarting discovery.""" + +from __future__ import annotations + +import json +import sqlite3 +from pathlib import Path + +import pytest +from workbench_test_support import run_workbench + + +@pytest.mark.parametrize("legacy_version", [None, "deep-security-scan/v1", "deep-scan-mcp/v1"]) +def test_new_workflow_default_preserves_existing_run_version( + tmp_path: Path, legacy_version: str | None +) -> None: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + created = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "fixture-thread", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + *(["--workflow-version", legacy_version] if legacy_version else []), + )["deepScan"] + expected_version = legacy_version or "deep-security-scan/v2" + assert created["workflowVersion"] == expected_version + resumed = run_workbench( + state, + "begin-deep-scan", + "--scan-id", + created["scanId"], + "--thread-id", + "fixture-thread", + )["deepScan"] + assert resumed["workflowVersion"] == expected_version + assert resumed["createdAt"] == created["createdAt"] + + +def selected_scan(tmp_path: Path, version: int) -> tuple[Path, str, dict[str, object]]: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + run = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "fixture-thread", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + )["deepScan"] + selected = { + "version": version, + "resultPath": None, + "resultSha256": None, + "terminalReason": "capped", + "omittedWorkerIds": [], + "selectedAt": "2000-01-01T00:00:00Z", + } + with sqlite3.connect(state / "workbench.sqlite3") as connection: + columns = {row[1] for row in connection.execute("PRAGMA table_info(deep_scan_runs)")} + if "finalization_input_json" not in columns: + connection.execute("ALTER TABLE deep_scan_runs ADD COLUMN finalization_input_json TEXT") + connection.execute( + "UPDATE deep_scan_runs SET workflow_version = 'deep-security-scan/v2', " + "finalization_input_json = ?, phase = 'reducing', " + "created_at = '2000-01-01T00:00:00Z', updated_at = '2000-01-01T00:00:00Z'", + (json.dumps(selected),), + ) + return state, str(run["scanId"]), selected + + +def test_claim_preserves_selected_finalization_without_discovery_recovery(tmp_path: Path) -> None: + state, scan_id, selected = selected_scan(tmp_path, 1) + claimed = run_workbench( + state, + "claim-deep-scan-coordinator", + "--scan-id", + scan_id, + "--thread-id", + "fixture-thread", + )["deepScan"] + assert claimed["finalizationInput"] == selected + assert claimed["phase"] == "reducing" + assert claimed["coordinatorGeneration"] == 2 + assert claimed["dispatchedCount"] == 0 + assert claimed["createdAt"] == "2000-01-01T00:00:00Z" + + +@pytest.mark.parametrize("command", ["begin-deep-scan", "claim-deep-scan-coordinator"]) +def test_unsupported_selection_rejects_without_mutation(tmp_path: Path, command: str) -> None: + state, scan_id, selected = selected_scan(tmp_path, 99) + with sqlite3.connect(state / "workbench.sqlite3") as connection: + before = "\n".join(connection.iterdump()) + rejected = run_workbench( + state, + command, + "--scan-id", + scan_id, + "--thread-id", + "fixture-thread", + check=False, + ) + assert rejected["returncode"] != 0 + assert "unsupported" in str(rejected["stderr"]).lower() + with sqlite3.connect(state / "workbench.sqlite3") as connection: + assert "\n".join(connection.iterdump()) == before + observed = run_workbench( + state, + "get-deep-scan", + "--scan-id", + scan_id, + "--thread-id", + "fixture-thread", + )["deepScan"] + assert observed["finalizationInput"] == selected diff --git a/plugins/codex-security/tests/test_deep_scan_persistence.py b/plugins/codex-security/tests/test_deep_scan_persistence.py new file mode 100644 index 0000000000..c2c8584276 --- /dev/null +++ b/plugins/codex-security/tests/test_deep_scan_persistence.py @@ -0,0 +1,614 @@ +from __future__ import annotations + +import json +import sqlite3 +import subprocess +import sys +import uuid +from pathlib import Path +from types import SimpleNamespace + +import pytest +from test_workbench_deep_scan import ( + begin_target_scan, + commit_reducer, + dispatch_discovery_worker, + upsert_worker, + worker_paths, +) +from workbench_test_support import run_workbench + + +def test_state_snapshot_does_not_mix_concurrent_acceptance( + tmp_path: Path, workbench_api, monkeypatch: pytest.MonkeyPatch +) -> None: + state_dir, codex_home, target = tmp_path / "state", tmp_path / "codex", tmp_path / "target" + target.mkdir() + initial = begin_target_scan(state_dir, codex_home, target, tmp_path / "scans")["deepScan"] + scan_id = initial["scanId"] + worker_id, _, _, _ = dispatch_discovery_worker( + state_dir, + codex_home, + scan_id=scan_id, + scan_dir=Path(initial["scanDir"]), + name="discovery-1", + succeed=False, + ) + database = state_dir / "workbench.sqlite3" + deep_scan = sys.modules["deep_scan_workbench"] + monkeypatch.setattr( + deep_scan, + "_dependencies", + SimpleNamespace( + **workbench_api, + preserve_stopped_results=workbench_api["preserve_stopped_results_after_transition"], + ), + ) + original = deep_scan.require_deep_scan_run + + def accept_after_read(connection, requested_scan_id): + run = original(connection, requested_scan_id) + with sqlite3.connect(database) as writer: + writer.execute( + "UPDATE deep_scan_runs SET completion_sequence = 1 WHERE scan_id = ?", (scan_id,) + ) + writer.execute( + "UPDATE deep_scan_workers SET status = 'succeeded', completion_sequence = 1 " + "WHERE id = ?", + (worker_id,), + ) + return run + + monkeypatch.setattr(deep_scan, "require_deep_scan_run", accept_after_read) + with sqlite3.connect(database) as reader: + reader.row_factory = sqlite3.Row + snapshot = deep_scan.deep_scan_state(reader, scan_id) + assert snapshot["completionSequence"] == 0 + assert snapshot["workers"][0]["status"] == "running" + assert not reader.in_transaction + + +def test_state_snapshot_preserves_its_callers_transaction( + tmp_path: Path, workbench_api, monkeypatch: pytest.MonkeyPatch +) -> None: + target = tmp_path / "target" + target.mkdir() + initial = begin_target_scan(tmp_path / "state", tmp_path / "codex", target, tmp_path / "scans") + scan_id = initial["deepScan"]["scanId"] + deep_scan = sys.modules["deep_scan_workbench"] + monkeypatch.setattr( + deep_scan, + "_dependencies", + SimpleNamespace( + **workbench_api, + preserve_stopped_results=workbench_api["preserve_stopped_results_after_transition"], + ), + ) + with sqlite3.connect(tmp_path / "state" / "workbench.sqlite3") as connection: + connection.row_factory = sqlite3.Row + connection.execute("BEGIN IMMEDIATE") + connection.execute( + "UPDATE deep_scan_runs SET consecutive_errors = 2 WHERE scan_id = ?", (scan_id,) + ) + snapshot = deep_scan.deep_scan_state(connection, scan_id) + assert snapshot["consecutiveErrors"] == 2 + assert connection.in_transaction + connection.rollback() + assert deep_scan.deep_scan_state(connection, scan_id)["consecutiveErrors"] == 0 + + +def test_replaced_attempts_retain_observed_sessions_and_accepted_result(tmp_path: Path) -> None: + state, home, target = tmp_path / "state", tmp_path / "codex", tmp_path / "target" + target.mkdir() + run = begin_target_scan(state, home, target, tmp_path / "scans")["deepScan"] + worker_id = str(uuid.uuid4()) + prompt, artifacts, result = worker_paths(Path(run["scanDir"]), "discovery-1") + mutation = dict( + scan_id=run["scanId"], + worker_id=worker_id, + kind="discovery", + prompt_path=prompt, + artifact_dir=artifacts, + ) + upsert_worker(state, home, **mutation, status="running", attempt=1, thread_id="old-session") + upsert_worker( + state, + home, + **mutation, + status="running", + attempt=1, + thread_id="old-session", + error="Artifact validation failed", + ) + upsert_worker(state, home, **mutation, status="running", attempt=2, thread_id="new-session") + result.write_text('{"findings": []}\n') + accepted = upsert_worker( + state, + home, + **mutation, + status="succeeded", + attempt=2, + thread_id="new-session", + result_path=result, + )["deepScan"] + attempts = accepted["attempts"] + assert [(item["attempt"], item["status"]) for item in attempts] == [ + (1, "failed"), + (2, "succeeded"), + ] + assert [item["sdkThreadId"] for item in accepted["attemptSessions"]] == [ + "old-session", + "new-session", + ] + assert attempts[0]["error"] == "Artifact validation failed" + assert attempts[0]["completedAt"] is not None + assert attempts[1]["acceptedResultSha256"] + assert Path(attempts[1]["acceptedResultPath"]).read_text() == result.read_text() + result.unlink() + replayed = upsert_worker( + state, + home, + **mutation, + status="succeeded", + attempt=2, + thread_id="new-session", + result_path=result, + )["deepScan"] + assert replayed == accepted + + +def test_merge_replay_returns_original_operation_after_later_work(tmp_path: Path) -> None: + state, home, target = tmp_path / "state", tmp_path / "codex", tmp_path / "target" + target.mkdir() + run = begin_target_scan(state, home, target, tmp_path / "scans")["deepScan"] + scan_id, scan_dir = run["scanId"], Path(run["scanDir"]) + inputs = [ + dispatch_discovery_worker( + state, + home, + scan_id=scan_id, + scan_dir=scan_dir, + name=f"discovery-{index}", + )[0] + for index in range(2) + ] + committed = commit_reducer( + state, + home, + scan_id=scan_id, + scan_dir=scan_dir, + name="dedup-1", + input_worker_ids=inputs, + new_findings_count=0, + ) + reducer = next(worker for worker in committed["workers"] if worker["kind"] == "dedup") + frozen = committed["committedMerge"]["resultManifestPath"] + assert Path(frozen).is_file() + Path(reducer["resultManifestPath"]).unlink() + assert [item["discoveryWorkerId"] for item in committed["dedupInputs"]] == inputs + assert all(item["attempt"] == 1 for item in committed["dedupInputs"]) + assert all("/checkpoints/" in item["resultManifestPath"] for item in committed["dedupInputs"]) + later = dispatch_discovery_worker( + state, + home, + scan_id=scan_id, + scan_dir=scan_dir, + name="discovery-2", + )[0] + second_commit = commit_reducer( + state, + home, + scan_id=scan_id, + scan_dir=scan_dir, + name="dedup-2", + input_worker_ids=[later], + new_findings_count=1, + ) + second_claim = second_commit["mergeClaims"][-1] + assert second_claim["previousWorkerId"] == reducer["id"] + assert second_claim["previousResultPath"] == frozen + assert ( + second_claim["previousResultSha256"] == committed["committedMerge"]["resultManifestSha256"] + ) + replay = run_workbench( + state, + "commit-deep-scan-dedup", + "--scan-id", + scan_id, + "--worker-id", + reducer["id"], + "--result-manifest-path", + str(scan_dir / "different.json"), + "--new-findings-count", + "99", + environment={"CODEX_HOME": str(home)}, + )["deepScan"] + assert replay["committedMerge"] == committed["committedMerge"] + assert replay["completionSequence"] == second_commit["completionSequence"] + assert replay["noNewStreak"] == second_commit["noNewStreak"] + with sqlite3.connect(state / "workbench.sqlite3") as connection: + receipt = json.loads( + connection.execute( + "SELECT receipt_json FROM deep_scan_merge_claims WHERE worker_id = ?", + (reducer["id"],), + ).fetchone()[0] + ) + assert receipt == committed["committedMerge"] + + +def test_native_usage_keeps_replaced_failed_canceled_attempts_and_descendants( + tmp_path: Path, workbench_api, monkeypatch: pytest.MonkeyPatch +) -> None: + from datetime import datetime, timedelta + + from test_workbench_scan_usage import _counts, _event, _rollout, _state_graph, _token_event + + state, home, target = tmp_path / "state", tmp_path / "codex", tmp_path / "target" + target.mkdir() + run = begin_target_scan(state, home, target, tmp_path / "scans")["deepScan"] + worker_id = str(uuid.uuid4()) + prompt, artifacts, _ = worker_paths(Path(run["scanDir"]), "discovery-1") + mutation = dict( + scan_id=run["scanId"], + worker_id=worker_id, + kind="discovery", + prompt_path=prompt, + artifact_dir=artifacts, + ) + upsert_worker(state, home, **mutation, status="running", attempt=1, thread_id="old") + upsert_worker(state, home, **mutation, status="running", attempt=2, thread_id="old") + upsert_worker( + state, + home, + **mutation, + status="running", + attempt=2, + thread_id="old", + error="fixture failure", + ) + upsert_worker(state, home, **mutation, status="running", attempt=3, thread_id="replacement") + terminal = upsert_worker( + state, home, **mutation, status="canceled", attempt=3, thread_id="replacement" + )["deepScan"] + assert [item["status"] for item in terminal["attempts"]] == ["replaced", "failed", "canceled"] + environment = { + "CODEX_HOME": str(home), + "CODEX_SQLITE_HOME": str(tmp_path / "native"), + "CODEX_STATE_DB": "", + } + for key, value in environment.items(): + monkeypatch.setenv(key, value) + with sqlite3.connect(state / "workbench.sqlite3") as connection: + connection.row_factory = sqlite3.Row + scan = connection.execute("SELECT * FROM scans WHERE id = ?", (run["scanId"],)).fetchone() + timestamp = datetime.fromisoformat(scan["started_at"].replace("Z", "+00:00")) + timedelta( + microseconds=1 + ) + context = _event( + timestamp, "turn_context", {"turn_id": "fixture-turn", "model": "gpt-5.6-sol"} + ) + old = _rollout(tmp_path, "old", [context]) + _state_graph( + environment, + { + "thread-deep-scan": _rollout(tmp_path, "thread-deep-scan", []), + "old": old, + "replacement": _rollout( + tmp_path, "replacement", [context, _token_event(timestamp, 30, 0)] + ), + "child": _rollout( + tmp_path, + "child", + [context, _token_event(timestamp, 5, 0)], + parent_thread_id="old", + ), + "unrelated": _rollout( + tmp_path, + "unrelated", + [context, _token_event(timestamp, 900, 0)], + parent_thread_id="thread-deep-scan", + ), + }, + [("old", "child"), ("thread-deep-scan", "unrelated")], + ) + reader = sys.modules["workbench_scan_usage"] + pending = reader.collect_scan_usage(connection, scan) + assert pending["inputTokens"] == 35 + assert pending["missingThreadCount"] == 2 + old.write_text(old.read_text() + json.dumps(_token_event(timestamp, 20, 0)) + "\n") + measured = reader.collect_scan_usage(connection, scan) + assert measured["inputTokens"] == 55 + assert measured["threadCount"] == 3 + assert measured["missingThreadCount"] == 1 + assert measured["coverage"] == "partial" # Original shared parent turn was unavailable. + assert measured["modelUsage"] == [{"model": "gpt-5.6-sol", **_counts(55, 0, 0)}] + + +def test_claim_replay_preserves_original_inputs_after_concurrent_discovery(tmp_path: Path) -> None: + state, home, target = tmp_path / "state", tmp_path / "codex", tmp_path / "target" + target.mkdir() + run = begin_target_scan(state, home, target, tmp_path / "scans")["deepScan"] + scan_id, scan_dir = run["scanId"], Path(run["scanDir"]) + inputs = [ + dispatch_discovery_worker( + state, home, scan_id=scan_id, scan_dir=scan_dir, name=f"discovery-{index}" + )[0] + for index in range(2) + ] + prompt, artifacts, _ = worker_paths(scan_dir, "reducer") + args = [ + "claim-deep-scan-dedup", + "--scan-id", + scan_id, + "--worker-id", + str(uuid.uuid4()), + "--prompt-path", + str(prompt), + "--artifact-dir", + str(artifacts), + ] + for worker in inputs: + args.extend(["--input-worker-id", worker]) + claimed = run_workbench(state, *args, environment={"CODEX_HOME": str(home)}) + dispatch_discovery_worker( + state, home, scan_id=scan_id, scan_dir=scan_dir, name="concurrent-discovery" + ) + replayed = run_workbench(state, *args, environment={"CODEX_HOME": str(home)}) + assert replayed["deepScan"]["mergeClaims"] == claimed["deepScan"]["mergeClaims"] + assert replayed["deepScan"]["dedupInputs"] == claimed["deepScan"]["dedupInputs"] + assert ( + replayed["deepScan"]["completionSequence"] == claimed["deepScan"]["completionSequence"] + 1 + ) + + +def test_acceptance_reuses_authoritative_checkpoint_without_rewriting(tmp_path: Path) -> None: + import hashlib + + state, home, target = tmp_path / "state", tmp_path / "codex", tmp_path / "target" + target.mkdir() + run = begin_target_scan(state, home, target, tmp_path / "scans")["deepScan"] + worker_id = str(uuid.uuid4()) + prompt, artifacts, result = worker_paths(Path(run["scanDir"]), "discovery") + mutation = dict( + scan_id=run["scanId"], + worker_id=worker_id, + kind="discovery", + prompt_path=prompt, + artifact_dir=artifacts, + attempt=1, + ) + upsert_worker(state, home, **mutation, status="running") + draft = {"scanId": run["scanId"], "findings": [], "coverage": {}} + content = json.dumps(draft, indent=2).encode() + b"\n" + checkpoint = artifacts / "checkpoints" / f"{hashlib.sha256(content).hexdigest()}.json" + checkpoint.parent.mkdir() + checkpoint.write_bytes(content) + (artifacts / "checkpoint-head.json").write_text(json.dumps({"checkpoint": checkpoint.name})) + result.write_text(json.dumps({**draft, "handoffClaimToken": "synthetic-claim"})) + accepted = upsert_worker(state, home, **mutation, status="succeeded", result_path=result)[ + "deepScan" + ] + assert accepted["attempts"][0]["acceptedResultPath"] == str(checkpoint) + assert accepted["attempts"][0]["acceptedResultSha256"] == hashlib.sha256(content).hexdigest() + result.unlink() + upsert_worker(state, home, **mutation, status="succeeded", result_path=result) + assert list(checkpoint.parent.iterdir()) == [checkpoint] + assert checkpoint.read_bytes() == content + assert not (artifacts / "accepted").exists() + + +def test_acceptance_rejects_mutable_result_behind_checkpoint_head(tmp_path: Path) -> None: + import hashlib + + state, home, target = tmp_path / "state", tmp_path / "codex", tmp_path / "target" + target.mkdir() + run = begin_target_scan(state, home, target, tmp_path / "scans")["deepScan"] + worker_id = str(uuid.uuid4()) + prompt, artifacts, result = worker_paths(Path(run["scanDir"]), "discovery") + mutation = dict( + scan_id=run["scanId"], + worker_id=worker_id, + kind="discovery", + prompt_path=prompt, + artifact_dir=artifacts, + attempt=1, + ) + upsert_worker(state, home, **mutation, status="running") + draft = {"scanId": run["scanId"], "findings": [], "coverage": {"deferred": ["unresolved"]}} + content = json.dumps(draft).encode() + checkpoint = artifacts / "checkpoints" / f"{hashlib.sha256(content).hexdigest()}.json" + checkpoint.parent.mkdir() + checkpoint.write_bytes(content) + (artifacts / "checkpoint-head.json").write_text(json.dumps({"checkpoint": checkpoint.name})) + result.write_text(json.dumps({**draft, "coverage": {}})) + with pytest.raises(subprocess.CalledProcessError) as failure: + upsert_worker(state, home, **mutation, status="succeeded", result_path=result) + assert "does not match its current checkpoint head" in failure.value.stderr + assert checkpoint.read_bytes() == content + + +@pytest.mark.parametrize("replacement", ["result", "checkpoint", "ordinary"]) +def test_accepted_result_does_not_reopen_replaced_path( + tmp_path, workbench_api, monkeypatch, replacement +): + from finalize_scan_contract import ContractError + + state, home, target = tmp_path / "state", tmp_path / "codex", tmp_path / "target" + target.mkdir() + run = begin_target_scan(state, home, target, tmp_path / "scans")["deepScan"] + scan_id, scan_dir = run["scanId"], Path(run["scanDir"]) + worker_id, prompt, artifacts, result = dispatch_discovery_worker( + state, home, scan_id=scan_id, scan_dir=scan_dir, name="discovery-1", succeed=False + ) + payload = { + "scanId": scan_id, + "complete": False, + "findings": [], + "coverage": { + "completeness": "complete", + "surfaces": [], + "explicitExclusions": [], + "deferred": [], + }, + } + original_bytes = json.dumps(payload).encode() + result.write_bytes(original_bytes) + checkpoint = artifacts / "checkpoints" / "accepted.json" + if replacement == "checkpoint": + checkpoint.parent.mkdir(parents=True) + checkpoint.write_bytes(original_bytes) + external = tmp_path / "outside-scan.json" + external.write_text(json.dumps(payload, indent=2) + "\n") + deep = sys.modules["deep_scan_workbench"] + monkeypatch.setattr( + deep, + "_dependencies", + SimpleNamespace( + **workbench_api, + preserve_stopped_results=workbench_api["preserve_stopped_results_after_transition"], + ), + ) + validate = deep.deep_scan_path + swapped = [] + + def validate_then_replace(scan, value, label, *, kind): + checked = validate(scan, value, label, kind=kind) + selected = ( + "Accepted worker result" if replacement == "result" else "Accepted worker checkpoint" + ) + if replacement != "ordinary" and label == selected: + path = Path(checked) + path.unlink() + path.symlink_to(external) + swapped.append(label) + return checked + + monkeypatch.setattr(deep, "deep_scan_path", validate_then_replace) + args = SimpleNamespace( + scan_id=scan_id, + worker_id=worker_id, + kind="discovery", + status="succeeded", + prompt_path=str(prompt), + artifact_dir=str(artifacts), + result_manifest_path=str(result), + attempt=1, + sdk_thread_id="synthetic-worker", + error_message=None, + replaceable_failure_kind=None, + coordinator_generation=None, + ) + with sqlite3.connect(state / "workbench.sqlite3") as connection: + connection.row_factory = sqlite3.Row + if replacement == "ordinary": + accepted = deep.upsert_deep_scan_worker(connection, args)["deepScan"] + attempt = accepted["attempts"][-1] + assert Path(attempt["acceptedResultPath"]).read_bytes() == original_bytes + assert ( + attempt["acceptedResultSha256"] + == __import__("hashlib").sha256(original_bytes).hexdigest() + ) + else: + with pytest.raises(ContractError, match="inside the scan directory"): + deep.upsert_deep_scan_worker(connection, args) + assert swapped + assert ( + connection.execute( + "SELECT status FROM deep_scan_workers WHERE id = ?", (worker_id,) + ).fetchone()[0] + == "running" + ) + assert ( + connection.execute( + "SELECT accepted_result_path FROM deep_scan_attempts WHERE worker_id = ?", + (worker_id,), + ).fetchone()[0] + is None + ) + + +@pytest.mark.parametrize("heartbeat", ["absent", "ordinary", "released", "other-generation"]) +def test_artifact_release_does_not_override_current_persisted_lease(tmp_path, heartbeat): + from test_workbench_deep_scan import claim_deep_scan_coordinator + + state, home, target = tmp_path / "state", tmp_path / "codex", tmp_path / "target" + target.mkdir() + initial = begin_target_scan(state, home, target, tmp_path / "scans")["deepScan"] + claimed = claim_deep_scan_coordinator(state, home, initial["scanId"])["deepScan"] + assert claimed["coordinatorGeneration"] == 2 + if heartbeat != "absent": + record = { + "coordinatorGeneration": 3 if heartbeat == "other-generation" else 2, + "updatedAt": claimed["updatedAt"], + } + if heartbeat in {"released", "other-generation"}: + record["released"] = True + path = ( + Path(claimed["scanDir"]) + / "artifacts" + / "deep_discovery" + / "coordinator-heartbeat-2.json" + ) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(record)) + rejoined = claim_deep_scan_coordinator(state, home, initial["scanId"]) + assert rejoined["coordinatorDisposition"] == "observing" + assert rejoined["deepScan"]["coordinatorGeneration"] == 2 + + +@pytest.mark.parametrize("release", ["normal", "repeat", "wrong-owner", "stale-generation"]) +def test_host_release_preserves_lease_owner_and_generation(tmp_path, release): + import os + + from test_workbench_deep_scan import claim_deep_scan_coordinator + from workbench_test_support import SCRIPT + + state, home, target = tmp_path / "state", tmp_path / "codex", tmp_path / "target" + target.mkdir() + initial = begin_target_scan(state, home, target, tmp_path / "scans")["deepScan"] + scan_id = initial["scanId"] + claimed = claim_deep_scan_coordinator(state, home, scan_id)["deepScan"] + assert claimed["coordinatorGeneration"] == 2 + + def host_release(thread="thread-deep-scan", generation=2): + return subprocess.run( + [ + sys.executable, + "-c", + "import runpy,sys; script=sys.argv.pop(1); runpy.run_path(script)['main'](release_coordinator=True)", + str(SCRIPT), + "claim-deep-scan-coordinator", + "--scan-id", + scan_id, + "--thread-id", + thread, + "--coordinator-generation", + str(generation), + ], + capture_output=True, + text=True, + env={**os.environ, "CODEX_SECURITY_STATE_DIR": str(state), "CODEX_HOME": str(home)}, + ) + + if release == "wrong-owner": + rejected = host_release("different-synthetic-owner") + assert rejected.returncode != 0 + observed = claim_deep_scan_coordinator(state, home, scan_id) + assert observed["coordinatorDisposition"] == "observing" + assert observed["deepScan"]["coordinatorGeneration"] == 2 + return + accepted = host_release() + assert accepted.returncode == 0, accepted.stderr + if release == "repeat": + repeated = host_release() + assert repeated.returncode == 0, repeated.stderr + adopted = claim_deep_scan_coordinator(state, home, scan_id) + assert adopted["coordinatorDisposition"] == "adopted" + assert adopted["deepScan"]["coordinatorGeneration"] == 3 + if release == "stale-generation": + rejected = host_release() + assert rejected.returncode != 0 + assert "newer generation" in rejected.stderr + observed = claim_deep_scan_coordinator(state, home, scan_id) + assert observed["coordinatorDisposition"] == "observing" + assert observed["deepScan"]["coordinatorGeneration"] == 3 diff --git a/plugins/codex-security/tests/test_deep_scan_publication_replay.py b/plugins/codex-security/tests/test_deep_scan_publication_replay.py index 4060d1e2a5..039ca00de3 100644 --- a/plugins/codex-security/tests/test_deep_scan_publication_replay.py +++ b/plugins/codex-security/tests/test_deep_scan_publication_replay.py @@ -8,9 +8,11 @@ from pathlib import Path import pytest +from test_accepted_publication_references import accept_reducer from test_deep_scan_publication_authority import stage_publication from test_deep_scan_successful_publication import add_worker from test_deep_scan_successful_publication import publication_scan as publication_scan +from test_publication_stop_interleavings import saved_selection _CRASH_PUBLICATION = """ import json, os, runpy, sqlite3, sys @@ -56,11 +58,21 @@ def crash_after_write(root, relative, contents): "boundary", ["findings.json", "coverage.json", "scan-manifest.json", "sqlite-before", "sqlite-after"], ) +@pytest.mark.parametrize("selection_reason", [None, "saturated", "capped"]) def test_publication_crash_replays_selected_input_without_stale_overwrite( - workbench_api, workbench_db, publication_scan, tmp_path, boundary + workbench_api, workbench_db, publication_scan, tmp_path, boundary, selection_reason ): scan = publication_scan() - result_path = add_worker(workbench_db, scan) + if selection_reason is None: + result_path = add_worker(workbench_db, scan) + else: + _, result_path, _ = accept_reducer(workbench_db, scan) + saved_selection(workbench_db, scan, result_path, reason=selection_reason) + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_runs SET terminal_reason = ? WHERE scan_id = ?", + (selection_reason, scan.scan_id), + ) with workbench_db: workbench_db.execute( "UPDATE deep_scan_runs SET coordinator_generation = 3 WHERE scan_id = ?", @@ -133,7 +145,18 @@ def test_publication_crash_replays_selected_input_without_stale_overwrite( )["scan"] assert completed["progress"]["status"] == "complete" assert completed["findingCount"] == 1 - assert dict(connection.execute("SELECT * FROM deep_scan_runs").fetchone()) == run_before + run_after = dict(connection.execute("SELECT * FROM deep_scan_runs").fetchone()) + if run_before["finalization_input_json"] is not None: + before_selection = json.loads(run_before["finalization_input_json"]) + after_selection = json.loads(run_after["finalization_input_json"]) + digest = after_selection.pop("publicationSha256") + assert len(digest) == 64 and all(char in "0123456789abcdef" for char in digest) + if "publicationSha256" in before_selection: + before_digest = before_selection.pop("publicationSha256") + assert digest == before_digest + assert after_selection == before_selection + run_after["finalization_input_json"] = run_before["finalization_input_json"] + assert run_after == run_before assert [ dict(row) for row in connection.execute("SELECT * FROM deep_scan_workers") ] == workers_before diff --git a/plugins/codex-security/tests/test_deep_scan_recovery_settings.py b/plugins/codex-security/tests/test_deep_scan_recovery_settings.py new file mode 100644 index 0000000000..74949000a7 --- /dev/null +++ b/plugins/codex-security/tests/test_deep_scan_recovery_settings.py @@ -0,0 +1,135 @@ +"""Original discovery input and observation remain stable across reconstruction.""" + +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +import pytest +from workbench_test_support import run_workbench + + +@pytest.mark.parametrize("original_context", [None, "Audit the original parser"]) +def test_reconstruction_preserves_discovery_input_settings_and_deadline( + tmp_path: Path, + original_context: str | None, +) -> None: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + codex_home = tmp_path / "home" + config = codex_home / "codex-security" / "config.toml" + config.parent.mkdir(parents=True) + config.write_text("[deep_scan]\nworkers = 2\nmax_time_hours = 2.5\n") + environment = {"CODEX_HOME": str(codex_home)} + begun = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "fixture-thread", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + "--model", + "original-model", + "--reasoning-effort", + "high", + *(["--user-context-stdin"] if original_context is not None else []), + input_text=original_context, + environment=environment, + )["deepScan"] + scan_id = str(begun["scanId"]) + with sqlite3.connect(state / "workbench.sqlite3") as connection: + connection.execute("UPDATE scans SET user_context = 'Later discussion'") + connection.execute("UPDATE deep_scan_runs SET updated_at = '2000-01-01T00:00:00Z'") + before = "\n".join(connection.iterdump()) + config.write_text("[deep_scan]\nworkers = 8\nmax_time_hours = 12\n") + joined = run_workbench( + state, + "begin-deep-scan", + "--scan-id", + scan_id, + "--thread-id", + "fixture-thread", + "--model", + "observer-model", + "--reasoning-effort", + "low", + environment=environment, + )["deepScan"] + with sqlite3.connect(state / "workbench.sqlite3") as connection: + assert "\n".join(connection.iterdump()) == before + assert connection.execute("SELECT model, reasoning_effort FROM scans").fetchone() == ( + "original-model", + "high", + ) + recovered = run_workbench( + state, + "claim-deep-scan-coordinator", + "--scan-id", + scan_id, + "--thread-id", + "fixture-thread", + environment=environment, + )["deepScan"] + for run in (joined, recovered): + assert run["model"] == "original-model" + assert run["reasoningEffort"] == "high" + assert run["userContext"] == original_context + assert run["createdAt"] == begun["createdAt"] + assert run["config"] == begun["config"] + assert run["workflowVersion"] == begun["workflowVersion"] + + +@pytest.mark.parametrize("workflow_version", ["deep-security-scan/v1", "deep-scan-mcp/v1"]) +def test_supported_old_run_snapshots_context_on_upgrade( + tmp_path: Path, workflow_version: str +) -> None: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + begun = run_workbench( + state, + "begin-deep-scan", + "--workflow-version", + workflow_version, + "--thread-id", + "fixture-thread", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + "--user-context", + "Legacy context", + )["deepScan"] + with sqlite3.connect(state / "workbench.sqlite3") as connection: + connection.execute("ALTER TABLE deep_scan_runs DROP COLUMN discovery_user_context") + connection.execute("UPDATE deep_scan_runs SET discovery_user_context_json = NULL") + connection.execute( + "DELETE FROM schema_migrations WHERE name = ?", + ("preserve original deep scan discovery context",), + ) + upgraded = run_workbench( + state, + "get-deep-scan", + "--scan-id", + str(begun["scanId"]), + "--thread-id", + "fixture-thread", + )["deepScan"] + assert upgraded["workflowVersion"] == workflow_version + assert upgraded["userContext"] == "Legacy context" + assert upgraded["config"] == begun["config"] + assert upgraded["createdAt"] == begun["createdAt"] + with sqlite3.connect(state / "workbench.sqlite3") as connection: + connection.execute("UPDATE scans SET user_context = 'Later discussion'") + observed = run_workbench( + state, + "get-deep-scan", + "--scan-id", + str(begun["scanId"]), + "--thread-id", + "fixture-thread", + )["deepScan"] + assert observed["userContext"] == "Legacy context" diff --git a/plugins/codex-security/tests/test_deep_scan_settings_creation.py b/plugins/codex-security/tests/test_deep_scan_settings_creation.py new file mode 100644 index 0000000000..f23212ee7a --- /dev/null +++ b/plugins/codex-security/tests/test_deep_scan_settings_creation.py @@ -0,0 +1,375 @@ +"""Original settings survive real creation/claim process death and competing starts.""" + +from __future__ import annotations + +import json +import os +import sqlite3 +import subprocess +import sys +from pathlib import Path + +import pytest +from workbench_test_support import ( + SCRIPT, + create_saved_workspace, + run_workbench, + start_delivered_scan, +) + +# Run the real workbench, pausing only at a requested transaction boundary. +# The parse hook also runs against the parent source, where the settings input +# was ignored, so its regression fails on missing durable bytes. +BOUNDARY_PROCESS = """ +import json, runpy, sys +from pathlib import Path +boundary = sys.argv.pop(1) +script = sys.argv.pop(1) +main = runpy.run_path(script)['main'] +namespace = main.__globals__ +deep = namespace['deep_scan'] +parse = namespace['parse_args'] +def parse_with_settings(*args, **kwargs): + result = parse(*args, **kwargs) + payload = json.loads(sys.stdin.readline()) + result.execution_settings = payload['executionSettings'] + result.user_context = payload.get('userContext') + result.user_context_stdin = False + return result +namespace['parse_args'] = parse_with_settings +def pause(scan): + print(json.dumps({'scanId': scan['id'], 'scanDir': scan['scan_dir']}), flush=True) + sys.stdin.readline() +write = deep.write_scan_local_bytes +def write_settings(scan_dir, relative_path, payload, **kwargs): + if boundary == 'before-write' and relative_path.endswith('/execution-settings.json'): + pause({'id': None, 'scan_dir': str(scan_dir)}) + return write(scan_dir, relative_path, payload, **kwargs) +deep.write_scan_local_bytes = write_settings +ensure = deep.ensure_deep_scan_run +def ensure_run(connection, scan, *args): + result = ensure(connection, scan, *args) + if boundary == 'before-commit': + pause(scan) + return result +deep.ensure_deep_scan_run = ensure_run +begin = deep.begin_deep_scan +def begin_run(connection, args): + result = begin(connection, args) + if boundary == 'after-commit': + pause({'id': result['deepScan']['scanId'], 'scan_dir': result['deepScan']['scanDir']}) + return result +deep.begin_deep_scan = begin_run +claim = deep.claim_deep_scan_coordinator +def claim_run(connection, args): + result = claim(connection, args) + if boundary == 'after-claim': + pause({'id': result['deepScan']['scanId'], 'scan_dir': result['deepScan']['scanDir']}) + return result +deep.claim_deep_scan_coordinator = claim_run +main() +""" + + +def settings(root: Path, name: str) -> dict[str, object]: + return { + "codexPath": str(root / name / "codex"), + "codexHome": str(root / name / "home"), + "model": f"{name}-model", + "reasoningEffort": "high", + "reasoningSummary": "concise", + "parentSandbox": {"filesystemDenies": [str(root / name / "denied")]}, + } + + +def start_process( + root: Path, + selected: dict[str, object] | None, + *args: str, + boundary: str = "none", +) -> subprocess.Popen[str]: + process = subprocess.Popen( + [sys.executable, "-c", BOUNDARY_PROCESS, boundary, str(SCRIPT), *args], + env={ + **os.environ, + "CODEX_SECURITY_STATE_DIR": str(root / "state"), + "CODEX_HOME": str(root / "current-home"), + }, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert process.stdin is not None + process.stdin.write(json.dumps({"executionSettings": selected}) + "\n") + process.stdin.flush() + return process + + +def finish(process: subprocess.Popen[str]) -> dict[str, object]: + stdout, stderr = process.communicate(timeout=30) + assert process.returncode == 0, stderr + return json.loads(stdout) + + +def begin_args(root: Path, *, name: str = "target") -> list[str]: + (root / name).mkdir(exist_ok=True) + return [ + "begin-deep-scan", + "--thread-id", + "original-thread", + "--target-path", + str(root / name), + "--scan-root", + str(root / "scans"), + ] + + +def snapshot(scan_dir: str) -> Path: + return Path(scan_dir) / "artifacts" / "deep_discovery" / "execution-settings.json" + + +@pytest.mark.parametrize( + "boundary", ["before-write", "before-commit", "after-commit", "after-claim"] +) +def test_process_death_keeps_original_settings_or_no_recoverable_run( + tmp_path: Path, boundary: str +) -> None: + original = settings(tmp_path, "original") + args = begin_args(tmp_path) + if boundary == "after-claim": + begun = finish(start_process(tmp_path, original, *args))["deepScan"] + args = [ + "claim-deep-scan-coordinator", + "--scan-id", + begun["scanId"], + "--thread-id", + "original-thread", + ] + process = start_process(tmp_path, original, *args, boundary=boundary) + try: + assert process.stdout is not None + paused = json.loads(process.stdout.readline()) + path = snapshot(paused["scanDir"]) + with sqlite3.connect(tmp_path / "state" / "workbench.sqlite3") as connection: + rows = connection.execute("SELECT scan_id FROM deep_scan_runs").fetchall() + if boundary in {"before-write", "before-commit"}: + assert rows == [], "uncommitted settings must not leave a recoverable run" + else: + assert rows == [(paused["scanId"],)] + if boundary != "before-write": + assert json.loads(path.read_bytes())["settings"] == original + finally: + process.kill() + process.communicate(timeout=30) + with sqlite3.connect(tmp_path / "state" / "workbench.sqlite3") as connection: + remaining = connection.execute("SELECT scan_id FROM deep_scan_runs").fetchall() + if boundary in {"before-write", "before-commit"}: + assert remaining == [] + return + before = path.read_bytes() + with sqlite3.connect(tmp_path / "state" / "workbench.sqlite3") as connection: + connection.execute("UPDATE deep_scan_runs SET updated_at = '2000-01-01T00:00:00Z'") + run_workbench( + tmp_path / "state", + "claim-deep-scan-coordinator", + "--scan-id", + paused["scanId"], + "--thread-id", + "original-thread", + ) + joined = finish(start_process(tmp_path, settings(tmp_path, "later"), *begin_args(tmp_path))) + assert joined["startDisposition"] == "joined" + assert path.read_bytes() == before + + +def test_competing_creation_and_observer_cannot_replace_original_settings(tmp_path: Path) -> None: + first = start_process( + tmp_path, settings(tmp_path, "original"), *begin_args(tmp_path), boundary="before-commit" + ) + second = None + try: + assert first.stdout is not None + paused = json.loads(first.stdout.readline()) + path = snapshot(paused["scanDir"]) + before = path.read_bytes() + second = start_process(tmp_path, settings(tmp_path, "later"), *begin_args(tmp_path)) + # Release the creation lock with the original process still alive. The + # contender must observe the committed run, not rewrite the settings. + created = finish(first) + joined = finish(second) + assert created["startDisposition"] == "created" + assert joined["startDisposition"] == "joined" + assert joined["deepScan"]["scanId"] == paused["scanId"] + assert path.read_bytes() == before + observer = finish(start_process(tmp_path, None, *begin_args(tmp_path))) + assert observer["startDisposition"] == "joined" + assert path.read_bytes() == before + other = finish( + start_process( + tmp_path, settings(tmp_path, "other"), *begin_args(tmp_path, name="other-target") + ) + ) + assert json.loads(snapshot(other["deepScan"]["scanDir"]).read_bytes())[ + "settings" + ] == settings(tmp_path, "other") + finally: + for process in (first, second): + if process is not None and process.poll() is None: + process.kill() + process.communicate(timeout=30) + + +def test_internal_settings_input_preserves_public_user_context_and_shape(tmp_path: Path) -> None: + context = "Review the parser.\nKeep this second line." + process = subprocess.run( + [ + sys.executable, + "-c", + ( + "import runpy, sys; script = sys.argv.pop(1); " + "runpy.run_path(script)['main'](with_execution_settings=True)" + ), + str(SCRIPT), + *begin_args(tmp_path), + ], + env={**os.environ, "CODEX_SECURITY_STATE_DIR": str(tmp_path / "state")}, + input=json.dumps( + {"executionSettings": settings(tmp_path, "original"), "userContext": context} + ), + capture_output=True, + text=True, + timeout=30, + ) + assert process.returncode == 0, process.stderr + result = json.loads(process.stdout) + assert result["deepScan"]["userContext"] == context + assert result["deepScan"]["executionSettings"] == { + "version": 1, + "settings": settings(tmp_path, "original"), + } + public = run_workbench( + tmp_path / "state", + "get-deep-scan", + "--scan-id", + result["deepScan"]["scanId"], + "--thread-id", + "original-thread", + ) + assert "executionSettings" not in public["deepScan"] + assert json.loads(snapshot(result["deepScan"]["scanDir"]).read_bytes())["settings"] == settings( + tmp_path, "original" + ) + + +def test_managed_creation_retry_preserves_saved_settings_before_commit(tmp_path: Path) -> None: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + workspace = create_saved_workspace(state, target, thread_id="original-thread", mode="deep") + started = start_delivered_scan( + state, + "--workspace-id", + workspace["id"], + "--scan-root", + str(tmp_path / "scans"), + "--model", + "original-model", + "--reasoning-effort", + "high", + ) + scan = started["results"] + args = ["begin-deep-scan", "--scan-id", scan["scanId"], "--thread-id", "original-thread"] + original = settings(tmp_path, "original") + process = start_process(tmp_path, original, *args, boundary="before-commit") + try: + assert process.stdout is not None + paused = json.loads(process.stdout.readline()) + path = snapshot(paused["scanDir"]) + before = path.read_bytes() + finally: + process.kill() + process.communicate(timeout=30) + with sqlite3.connect(state / "workbench.sqlite3") as connection: + assert connection.execute("SELECT COUNT(*) FROM deep_scan_runs").fetchone() == (0,) + recovered = finish( + start_process( + tmp_path, + original, + *args, + ) + ) + assert recovered["startDisposition"] == "created" + assert path.read_bytes() == before + assert json.loads(before)["settings"] == original + assert recovered["deepScan"]["model"] == "original-model" + assert recovered["deepScan"]["reasoningEffort"] == "high" + with sqlite3.connect(state / "workbench.sqlite3") as connection: + bound = json.loads( + connection.execute( + "SELECT execution_settings_json FROM deep_scan_runs WHERE scan_id = ?", + (scan["scanId"],), + ).fetchone()[0] + ) + assert bound == {"version": 1, "settings": original} + + +def test_preexisting_artifact_does_not_choose_original_execution_settings(tmp_path: Path) -> None: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + workspace = create_saved_workspace(state, target, thread_id="original-thread", mode="deep") + scan = start_delivered_scan( + state, "--workspace-id", workspace["id"], "--scan-root", str(tmp_path / "scans") + )["results"] + path = snapshot(scan["scanDir"]) + path.parent.mkdir(parents=True) + path.write_text(json.dumps({"version": 1, "settings": settings(tmp_path, "substituted")})) + original = settings(tmp_path, "original") + begun = finish( + start_process( + tmp_path, + original, + "begin-deep-scan", + "--scan-id", + scan["scanId"], + "--thread-id", + "original-thread", + ) + ) + assert begun["startDisposition"] == "created" + assert json.loads(path.read_text()) == {"version": 1, "settings": original} + with sqlite3.connect(state / "workbench.sqlite3") as connection: + bound = json.loads( + connection.execute( + "SELECT execution_settings_json FROM deep_scan_runs WHERE scan_id = ?", + (scan["scanId"],), + ).fetchone()[0] + ) + assert bound == {"version": 1, "settings": original} + path.write_text(json.dumps({"version": 1, "settings": settings(tmp_path, "substituted")})) + joined = finish( + start_process( + tmp_path, + settings(tmp_path, "observer"), + "begin-deep-scan", + "--scan-id", + scan["scanId"], + "--thread-id", + "original-thread", + ) + ) + assert joined["startDisposition"] == "joined" + with sqlite3.connect(state / "workbench.sqlite3") as connection: + repeated = json.loads( + connection.execute( + "SELECT execution_settings_json FROM deep_scan_runs WHERE scan_id = ?", + (scan["scanId"],), + ).fetchone()[0] + ) + assert repeated == bound + public = run_workbench(state, "get-scan", "--scan-id", scan["scanId"]) + assert public["scan"]["executionAttribution"]["workerCodexHome"] == original["codexHome"] + path.unlink() + public = run_workbench(state, "get-scan", "--scan-id", scan["scanId"]) + assert public["scan"]["executionAttribution"]["workerCodexHome"] == original["codexHome"] diff --git a/plugins/codex-security/tests/test_deep_scan_successful_publication.py b/plugins/codex-security/tests/test_deep_scan_successful_publication.py index 63fe19e42e..955629ab64 100644 --- a/plugins/codex-security/tests/test_deep_scan_successful_publication.py +++ b/plugins/codex-security/tests/test_deep_scan_successful_publication.py @@ -350,6 +350,87 @@ def test_deep_prepare_and_complete_preserve_the_same_aggregate( assert_published_aggregate(scan) +@pytest.mark.parametrize("prepare_only", [False, True], ids=["complete", "prepare"]) +@pytest.mark.parametrize("pending_artifact", ["writeup", "receipt"]) +def test_incomplete_deep_draft_remains_publishable_with_pending_artifacts( + workbench_api, workbench_db, publication_scan, prepare_only, pending_artifact +): + scan = publication_scan() + # Discovery has handed the unfinished draft back to the parent; no terminal + # canonical publication has been selected yet. + discovery_manifest = scan.scan_dir / "artifacts" / "deep_discovery" / "manifest.json" + discovery_manifest.parent.mkdir(parents=True, exist_ok=True) + discovery_manifest.write_bytes((scan.scan_dir / "scan-manifest.json").read_bytes()) + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_runs SET manifest_path = ? WHERE scan_id = ?", + (str(discovery_manifest), scan.scan_id), + ) + manifest_path = scan.scan_dir / "scan-manifest.json" + manifest = json.loads(manifest_path.read_text()) + manifest["scan"]["complete"] = False + if pending_artifact == "writeup": + relative_path = "findings/pending/pending.md" + scan.findings[0]["writeup"] = {"reportPath": relative_path} + else: + relative_path = "artifacts/receipts/pending.md" + scan.coverage["surfaces"] = [ + { + "id": "source-review", + "label": "Source review", + "disposition": "no_issue_found", + "receiptRefs": [relative_path], + } + ] + draft_dir = scan.scan_dir / "drafts" + draft_dir.mkdir() + draft_path = draft_dir / f"{uuid.uuid4()}.json" + documents = { + "manifest": manifest, + "findings": {"findings": scan.findings}, + "coverage": scan.coverage, + } + args = Namespace( + scan_id=scan.scan_id, + claim_token=None, + draft_path=str(draft_path), + checkpoint_path=None, + expected_draft_digest=None, + ) + saved_results = workbench_api["saved_results"] + context = workbench_api["_WORKBENCH_DB_CONTEXT"] + draft_path.write_text(json.dumps(documents)) + assert saved_results.write_scan_draft(context, workbench_db, args)["status"] == "draft_written" + names = ( + "scan-manifest.json", + "findings.json", + "coverage.json", + "checkpoint-head.json", + "report.md", + ) + before = {name: (scan.scan_dir / name).read_bytes() for name in names} + + with pytest.raises(SystemExit, match="The latest saved scan draft is incomplete"): + complete(workbench_api, workbench_db, scan, prepare_only=prepare_only) + + assert ( + workbench_db.execute("SELECT status FROM scans WHERE id = ?", (scan.scan_id,)).fetchone()[0] + == "running" + ) + assert {name: (scan.scan_dir / name).read_bytes() for name in names} == before + assert not (scan.scan_dir / relative_path).exists() + + artifact = scan.scan_dir / relative_path + artifact.parent.mkdir(parents=True, exist_ok=True) + artifact.write_text("# Completed source review\n\nSynthetic review evidence.\n") + manifest["scan"]["complete"] = True + draft_path.write_text(json.dumps(documents)) + assert saved_results.write_scan_draft(context, workbench_db, args)["status"] == "draft_written" + completed = complete(workbench_api, workbench_db, scan) + assert completed["progress"]["status"] == "complete" + assert_published_aggregate(scan) + + @pytest.mark.cross_platform @pytest.mark.parametrize("scope", [".", "subdir"]) @pytest.mark.parametrize("missing_parent", [False, True]) @@ -625,6 +706,7 @@ def test_standard_publication_preserves_deferred_work( ): scan = publication_scan(mode="standard") scan.coverage["deferred"] = [{"id": "remaining-review", "reason": "Another surface remains."}] + scan.coverage["reviews"] = [] (scan.scan_dir / "coverage.json").write_text(json.dumps(scan.coverage)) complete(workbench_api, workbench_db, scan) @@ -871,6 +953,155 @@ def test_stopped_deep_projection_filters_the_canonical_parent( assert json.loads((scan.scan_dir / "findings.json").read_text())["findings"] == [] +def test_deep_publication_renders_each_source_remediation( + workbench_api, workbench_db, publication_scan +): + scan = publication_scan() + finding = scan.findings[0] + first = copy.deepcopy(finding) + first.pop("provenance") + first["remediation"] = "Check the destination before writing the archive entry." + first["remediationTests"] = ["Reject an archive entry outside the destination."] + second = copy.deepcopy(first) + second["remediation"] = "Reject symbolic links before opening the destination." + second["remediationTests"] = ["Reject a symbolic link inside the destination."] + second["preventiveControls"] = ["Use a directory-relative file handle."] + finding["remediation"] = first["remediation"] + finding["remediationTests"] = first["remediationTests"] + finding["provenance"]["sourceFindings"] = [ + {"id": "review-1:0", "finding": first}, + {"id": "review-2:0", "finding": second}, + ] + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": scan.findings})) + + complete(workbench_api, workbench_db, scan) + + assert_published_aggregate(scan) + report = (scan.scan_dir / "report.md").read_text() + for source in (first, second): + assert report.count(source["remediation"]) == 1 + for test in source["remediationTests"]: + assert report.count(test) == 1 + assert "Use a directory-relative file handle." in report + + +@pytest.mark.parametrize("operation", ["fail", "cancel"]) +def test_stopped_legacy_reducers_keep_latest_completed_result( + workbench_api, workbench_db, publication_scan, operation +): + scan = publication_scan() + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', phase = 'reducing', " + "terminal_reason = NULL, completed_at = NULL WHERE scan_id = ?", + (scan.scan_id,), + ) + evidence = {} + for index, worker_id in enumerate( + ("ffffffff-ffff-4fff-8fff-ffffffffffff", "00000000-0000-4000-8000-000000000001") + ): + result = add_worker(workbench_db, scan) + finding = copy.deepcopy(scan.findings[0]) + finding["identity"]["anchor"] = f"legacy-reducer-{index}" + finding["summary"] = f"Synthetic reducer result {index}." + result.write_text( + json.dumps({"scanId": scan.scan_id, "complete": True, "findings": [finding]}) + ) + evidence[result] = result.read_bytes() + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_workers SET id = ?, kind = 'dedup', merge_state = 'none', " + "prompt_path = ?, completed_at = ? WHERE result_manifest_path = ?", + ( + worker_id, + str(scan.scan_dir / f"legacy-review-{index}" / "prompt.md"), + f"2026-01-0{index + 1}T00:00:00Z", + str(result), + ), + ) + for name in ("scan-manifest.json", "findings.json", "coverage.json"): + (scan.scan_dir / name).unlink() + saved_results = workbench_api["saved_results"] + args = Namespace( + scan_id=scan.scan_id, + claim_token=None, + thread_id=None, + cost_json=None, + message="Synthetic stop.", + ) + getattr(saved_results, f"{operation}_scan")( + workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, args + ) + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert [finding["summary"] for finding in findings] == ["Synthetic reducer result 1."] + published = (scan.scan_dir / "findings.json").read_bytes() + getattr(saved_results, f"{operation}_scan")( + workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, args + ) + assert (scan.scan_dir / "findings.json").read_bytes() == published + assert {path: path.read_bytes() for path in evidence} == evidence + + +@pytest.mark.parametrize("existing_results", [False, True]) +def test_unsupported_partial_draft_rejects_before_publishing( + workbench_api, workbench_db, publication_scan, existing_results +): + scan = publication_scan() + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_runs SET workflow_version = 'future/v99', " + "status = 'running', phase = 'discovery', manifest_path = NULL " + "WHERE scan_id = ?", + (scan.scan_id,), + ) + documents = { + key: json.loads((scan.scan_dir / filename).read_text()) + for key, filename in ( + ("manifest", "scan-manifest.json"), + ("findings", "findings.json"), + ("coverage", "coverage.json"), + ) + } + documents["manifest"]["scan"].update( + complete=False, threatModel={"summary": "Uncommitted setup model"} + ) + drafts = scan.scan_dir / "drafts" + drafts.mkdir(exist_ok=True) + staged = drafts / f"{uuid.uuid4()}.json" + staged.write_text(json.dumps(documents)) + if existing_results: + (scan.scan_dir / "checkpoint-head.json").write_text( + json.dumps({"checkpoint": "existing.json"}) + ) + else: + for filename in ("scan-manifest.json", "findings.json", "coverage.json"): + (scan.scan_dir / filename).unlink() + + def saved_files(): + return { + path.relative_to(scan.scan_dir): path.read_bytes() + for path in scan.scan_dir.rglob("*") + if path.is_file() + } + + before = saved_files() + database_before = "\n".join(workbench_db.iterdump()) + with pytest.raises(SystemExit, match="unsupported workflow or schema version"): + workbench_api["saved_results"].write_scan_draft( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + Namespace( + scan_id=scan.scan_id, + claim_token=None, + draft_path=str(staged), + checkpoint_path=None, + expected_draft_digest=None, + ), + ) + assert saved_files() == before + assert "\n".join(workbench_db.iterdump()) == database_before + + def test_already_sealed_legacy_capped_scan_remains_immutable( workbench_api, workbench_db, publication_scan ): diff --git a/plugins/codex-security/tests/test_deep_scan_usage_owner.py b/plugins/codex-security/tests/test_deep_scan_usage_owner.py new file mode 100644 index 0000000000..9d714ace8f --- /dev/null +++ b/plugins/codex-security/tests/test_deep_scan_usage_owner.py @@ -0,0 +1,209 @@ +from __future__ import annotations + +import json +import sqlite3 +import uuid +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import pytest +from test_workbench_scan_usage import _event, _rollout, _state_graph, _token_event +from workbench_test_support import run_workbench + + +def test_original_usage_turn_survives_join_and_coordinator_recovery(tmp_path: Path) -> None: + state = tmp_path / "state" + environment = { + "CODEX_HOME": str(tmp_path / "codex"), + "CODEX_SQLITE_HOME": str(tmp_path / "native"), + "CODEX_STATE_DB": "", + } + timestamp = datetime.now(timezone.utc) + rollout = _rollout( + tmp_path, + "shared-parent", + [ + _event(timestamp, "turn_context", {"turn_id": "original-turn", "model": "gpt-5.6-sol"}), + ], + ) + _state_graph(environment, {"shared-parent": rollout}, []) + target = tmp_path / "target" + target.mkdir() + begun = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "shared-parent", + "--target-path", + str(target), + "--scope", + ".", + "--scan-root", + str(tmp_path / "scans"), + environment=environment, + )["deepScan"] + owner = begun["usageOwner"] + assert owner["threadId"] == "shared-parent" + assert owner["turnId"] == "original-turn" + assert owner["dedicated"] is False + rollout.write_text( + rollout.read_text() + + json.dumps( + _event(timestamp, "turn_context", {"turn_id": "later-turn", "model": "gpt-6-astra"}) + ) + + "\n" + ) + joined = run_workbench( + state, + "begin-deep-scan", + "--scan-id", + begun["scanId"], + "--thread-id", + "shared-parent", + environment=environment, + )["deepScan"] + assert joined["usageOwner"] == owner + claim_args = [ + "claim-deep-scan-coordinator", + "--scan-id", + begun["scanId"], + "--thread-id", + "shared-parent", + ] + claimed = run_workbench(state, *claim_args, environment=environment)["deepScan"] + with sqlite3.connect(state / "workbench.sqlite3") as connection: + connection.execute( + "UPDATE deep_scan_runs SET updated_at = '2000-01-01T00:00:00+00:00' WHERE scan_id = ?", + (begun["scanId"],), + ) + recovered = run_workbench(state, *claim_args, environment=environment)["deepScan"] + assert recovered["coordinatorGeneration"] == claimed["coordinatorGeneration"] + 1 + assert recovered["usageOwner"] == owner + other_target = tmp_path / "other-target" + other_target.mkdir() + other = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "shared-parent", + "--target-path", + str(other_target), + "--scope", + ".", + "--scan-root", + str(tmp_path / "scans"), + environment=environment, + )["deepScan"] + assert other["usageOwner"]["turnId"] == "later-turn" + original = run_workbench( + state, "get-scan", "--scan-id", begun["scanId"], environment=environment + )["scan"] + assert original["executionAttribution"]["owner"] == owner + + +@pytest.mark.parametrize("migrated", [False, True]) +@pytest.mark.parametrize("attempt", [False, True]) +def test_migrated_owner_accounting_survives_first_worker_attempt( + tmp_path: Path, migrated: bool, attempt: bool, workbench_api, monkeypatch +) -> None: + from workbench_scan_usage import collect_scan_usage, scan_execution_attribution + + state = tmp_path / "state" + target = tmp_path / "target" + target.mkdir() + environment = { + "CODEX_HOME": str(tmp_path / "codex"), + "CODEX_SQLITE_HOME": str(tmp_path / "native"), + "CODEX_STATE_DB": "", + } + timestamp = datetime.now(timezone.utc) + parent = _rollout( + tmp_path, + "legacy-owner", + [ + _event(timestamp, "turn_context", {"turn_id": "original-turn", "model": "gpt-5.6-sol"}), + ], + ) + _state_graph(environment, {"legacy-owner": parent}, []) + begun = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "legacy-owner", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + environment=environment, + )["deepScan"] + database = state / "workbench.sqlite3" + if migrated: + # The usage-owner migration adds a nullable owner without an original turn binding. + with sqlite3.connect(database) as connection: + connection.execute("ALTER TABLE deep_scan_runs DROP COLUMN usage_owner_json") + connection.execute( + "DELETE FROM schema_migrations WHERE name = ?", + ("bind original deep scan parent usage turn",), + ) + run_workbench( + state, + "get-deep-scan", + "--scan-id", + begun["scanId"], + "--thread-id", + "legacy-owner", + environment=environment, + ) + if attempt: + artifact = Path(begun["scanDir"]) / "artifacts" / "first-worker" + artifact.mkdir(parents=True) + prompt = artifact / "prompt.md" + prompt.write_text("Review the synthetic target.\n") + run_workbench( + state, + "upsert-deep-scan-worker", + "--scan-id", + begun["scanId"], + "--worker-id", + str(uuid.uuid4()), + "--kind", + "discovery", + "--status", + "running", + "--prompt-path", + str(prompt), + "--artifact-dir", + str(artifact), + environment=environment, + ) + counted = datetime.fromisoformat(begun["createdAt"].replace("Z", "+00:00")) + timedelta( + microseconds=1 + ) + _rollout( + tmp_path, + "legacy-owner", + [ + _event( + counted, + "event_msg", + { + "type": "task_started", + "turn_id": "original-turn", + "started_at": int(counted.timestamp()), + }, + ), + _event(counted, "turn_context", {"turn_id": "original-turn", "model": "gpt-5.6-sol"}), + _token_event(counted, 100, 10), + ], + task_started_at=counted, + ) + for name, value in environment.items(): + monkeypatch.setenv(name, value) + with sqlite3.connect(database) as connection: + connection.row_factory = sqlite3.Row + scan = workbench_api["require_scan"](connection, begun["scanId"]) + attribution = scan_execution_attribution(connection, scan) + assert bool(attribution.get("legacy")) is migrated + usage = collect_scan_usage(connection, scan) + assert usage["inputTokens"] == 100 + assert usage["outputTokens"] == 10 diff --git a/plugins/codex-security/tests/test_finalization_selection_order.py b/plugins/codex-security/tests/test_finalization_selection_order.py new file mode 100644 index 0000000000..2c3a8989f9 --- /dev/null +++ b/plugins/codex-security/tests/test_finalization_selection_order.py @@ -0,0 +1,259 @@ +from __future__ import annotations + +import json +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +@pytest.mark.parametrize( + ("timing", "successor"), + [ + ("tied", "succeeded"), + ("earlier", "succeeded"), + ("later", "succeeded"), + ("later", "running"), + ("later", "failed"), + ("legacy", "succeeded"), + ("legacy-tied", "succeeded"), + ("legacy-earlier", "succeeded"), + ], +) +def test_final_selection_follows_committed_reducer_chain( + workbench_api, workbench_db, tmp_path, monkeypatch, timing, successor +): + target, scan_dir, home = (tmp_path / name for name in ("target", "scan", "home")) + target.mkdir() + scan_dir.mkdir(mode=0o700) + (target / "app.py").write_text("value = 1\n") + (home / "codex-security").mkdir(parents=True) + (home / "codex-security/config.toml").write_text( + "[deep_scan]\nworkers=1\nsubagents=0\nmax_discovery_runs=4\nstop_after_no_new=5\n" + ) + monkeypatch.setenv("CODEX_HOME", str(home)) + monkeypatch.setenv("CODEX_SECURITY_STATE_DIR", str(tmp_path / "state")) + clock = ["2030-01-01T00:00:01Z"] + deep = workbench_api["deep_scan"] + monkeypatch.setattr( + deep, + "_dependencies", + SimpleNamespace( + **{ + **workbench_api, + "now": lambda: clock[0], + "preserve_stopped_results": workbench_api[ + "preserve_stopped_results_after_transition" + ], + } + ), + ) + + def args(command, *words, **overrides): + monkeypatch.setattr(sys, "argv", ["workbench_db", command, *map(str, words)]) + parsed = workbench_api["parse_args"]("test") + vars(parsed).update(overrides) + return parsed + + registered = workbench_api["register_cli_scan"]( + workbench_db, + args( + "register-cli-scan", + "--scan-dir", + scan_dir, + "--repository", + target, + "--recipe-json", + json.dumps( + { + "config": {}, + "mode": "deep", + "repository": str(target), + "target": {"kind": "repository", "paths": []}, + } + ), + ), + ) + scan_id = registered["scanId"] + deep.begin_deep_scan( + workbench_db, + args( + "begin-deep-scan", + "--scan-id", + scan_id, + "--thread-id", + "synthetic-owner", + execution_settings={ + "codexPath": sys.executable, + "codexHome": str(home), + "parentSandbox": {"filesystemDenies": []}, + }, + ), + ) + coverage = { + "completeness": "complete", + "surfaces": [], + "explicitExclusions": [], + "deferred": [], + } + + def worker(number, inputs=None, status="succeeded"): + kind = "discovery" if inputs is None else "dedup" + prefix = "10000000" if inputs is None else "ffffffff" if number == 1 else "00000000" + worker_id = f"{prefix}-0000-4000-8000-{number:012d}" + directory = scan_dir / "workers" / f"{kind}-{number}" + directory.mkdir(parents=True) + prompt, result = directory / "prompt.md", directory / "result.json" + prompt.write_text("Synthetic review") + common = [ + "--scan-id", + scan_id, + "--worker-id", + worker_id, + "--prompt-path", + prompt, + "--artifact-dir", + directory, + ] + if inputs is not None: + deep.claim_deep_scan_dedup( + workbench_db, + args( + "claim-deep-scan-dedup", + *common, + *(value for item in inputs for value in ("--input-worker-id", item)), + ), + ) + common += ["--kind", kind, "--attempt", "1"] + deep.upsert_deep_scan_worker( + workbench_db, args("upsert-deep-scan-worker", *common, "--status", "running") + ) + result.write_text( + json.dumps( + { + "scanId": scan_id, + "complete": True, + "findings": [], + "coverage" if inputs is None else "sourceCoverage": { + **coverage, + "completeness": "partial", + "deferred": [{"reason": f"Review gap from {kind} {number}."}], + }, + } + ) + ) + if inputs is not None and status == "succeeded": + deep.commit_deep_scan_dedup( + workbench_db, + args( + "commit-deep-scan-dedup", + "--scan-id", + scan_id, + "--worker-id", + worker_id, + "--result-manifest-path", + result, + "--new-findings-count", + "1", + ), + ) + elif status != "running": + deep.upsert_deep_scan_worker( + workbench_db, + args( + "upsert-deep-scan-worker", + *common, + "--status", + status, + "--result-manifest-path", + result, + ), + ) + return worker_id + + first = worker(1, [worker(1), worker(2)]) + clock[0] = { + "earlier": "2030-01-01T00:00:00Z", + "tied": clock[0], + "legacy-tied": clock[0], + }.get(timing, "2030-01-01T00:00:02Z") + second = worker(2, [worker(3)], successor) + assert ( + workbench_db.execute( + "SELECT previous_worker_id FROM deep_scan_merge_claims WHERE worker_id = ?", (second,) + ).fetchone()[0] + == first + ) + if timing.startswith("legacy"): + # Exercise the legacy no-claim fallback with controlled metadata. + with workbench_db: + workbench_db.execute("DELETE FROM deep_scan_merge_claims") + expected_worker = first + if successor == "succeeded": + second_reference = workbench_db.execute( + "SELECT accepted_result_path, accepted_result_sha256 FROM deep_scan_attempts " + "WHERE worker_id = ?", + (second,), + ).fetchone() + if timing == "later": + clock[0] = "2030-01-01T00:00:03Z" + elif timing == "legacy-earlier": + clock[0] = "2030-01-01T00:00:00Z" + expected_worker = worker(3, [worker(4)]) + claim = workbench_db.execute( + "SELECT * FROM deep_scan_merge_claims WHERE worker_id = ?", (expected_worker,) + ).fetchone() + assert claim["previous_worker_id"] == second + assert claim["previous_result_path"] == second_reference["accepted_result_path"] + assert claim["previous_result_sha256"] == second_reference["accepted_result_sha256"] + accepted = workbench_db.execute( + "SELECT accepted_result_path, accepted_result_sha256 FROM deep_scan_attempts WHERE worker_id = ?", + (expected_worker,), + ).fetchone() + run, scan = deep.require_running_deep_scan(workbench_db, scan_id) + + def observe_budget_draft(document, unmerged): + assert document == json.loads(Path(accepted["accepted_result_path"]).read_bytes()) + assert len(unmerged) == (0 if successor == "succeeded" else 1) + raise RuntimeError("observed budget draft boundary") + + # Exercise budget auto-selection before committing a normal final selection. + with pytest.raises(RuntimeError, match="observed budget draft boundary"): + deep.prepare_budget_exhausted_deep_scan( + workbench_db, scan, scan_dir, "Synthetic budget stop", observe_budget_draft + ) + finish = args( + "finish-deep-scan", + "--scan-id", + scan_id, + "--terminal-reason", + "capped", + "--manifest-path", + scan_dir / "scan-manifest.json", + select_finalization=True, + finalization_result_path=accepted["accepted_result_path"], + ) + if successor == "succeeded": + previous = workbench_db.execute( + "SELECT accepted_result_path FROM deep_scan_attempts WHERE worker_id = ?", (first,) + ).fetchone()[0] + stale = SimpleNamespace(**{**vars(finish), "finalization_result_path": previous}) + with pytest.raises(SystemExit, match="committed accepted reducer"): + deep.selected_deep_scan_finalization(workbench_db, run, scan, stale, [], False) + selection = deep.finish_deep_scan_locked(workbench_db, finish, scan_id)["deepScan"][ + "finalizationInput" + ] + replay = deep.finish_deep_scan_locked(workbench_db, stale, scan_id)["deepScan"][ + "finalizationInput" + ] + assert replay == selection + else: + # A pending or failed successor cannot replace the last committed input + # selected for budget publication, which does not await model work. + selection = deep.selected_deep_scan_finalization(workbench_db, run, scan, finish, [], False) + assert ( + selection["resultPath"] + == Path(accepted["accepted_result_path"]).relative_to(scan_dir).as_posix() + ) + assert selection["resultSha256"] == accepted["accepted_result_sha256"] diff --git a/plugins/codex-security/tests/test_finalization_selection_process_loss.py b/plugins/codex-security/tests/test_finalization_selection_process_loss.py new file mode 100644 index 0000000000..a599e73ff7 --- /dev/null +++ b/plugins/codex-security/tests/test_finalization_selection_process_loss.py @@ -0,0 +1,169 @@ +from __future__ import annotations + +import io +import json +import sqlite3 +import subprocess +import sys +from argparse import Namespace +from pathlib import Path + +import pytest +from test_accepted_publication_references import accept_reducer +from test_checkpoint_publication_authority import save_disposition +from test_deep_scan_publication_authority import stage_publication +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan +from test_publication_stop_interleavings import published_bytes, stop_scan + +_CRASH_SELECTION = """ +import io, json, os, runpy, sqlite3, sys +from argparse import Namespace + +api = runpy.run_path(sys.argv[1], run_name="selection_commit_crash_test") +deep = api["deep_scan"] +from types import SimpleNamespace +deep.configure(SimpleNamespace(**{**api, "preserve_stopped_results": api["preserve_stopped_results_after_transition"]})) +class CrashConnection(sqlite3.Connection): + def commit(self): + if sys.argv[4] == "before": + os._exit(72) + super().commit() + os._exit(73) +connection = sqlite3.connect(sys.argv[2], factory=CrashConnection) +connection.row_factory = sqlite3.Row +connection.execute("PRAGMA foreign_keys = ON") +sys.stdin = io.StringIO(json.dumps({"resultPath": sys.argv[5]})) +deep.finish_deep_scan(connection, Namespace(**json.loads(sys.argv[3])), select_finalization=True) +raise AssertionError("selection never reached its commit boundary") +""" + + +@pytest.mark.parametrize("reason", ["saturated", "capped"]) +@pytest.mark.parametrize("cut", ["before", "after"]) +@pytest.mark.parametrize("cause", ["cancel", "cost"]) +@pytest.mark.parametrize("stop_before_replay", [False, True]) +def test_selection_commit_loss_replays_accepted_identity_before_stopping( + workbench_api, + workbench_db, + publication_scan, + tmp_path, + monkeypatch, + reason, + cut, + cause, + stop_before_replay, +): + scan = publication_scan() + result, accepted, _ = accept_reducer(workbench_db, scan) + omissions = [] + if reason == "saturated": + omitted = add_worker(workbench_db, scan) + omitted.write_text(json.dumps(save_disposition(scan, omitted.parent, "reported"))) + save_disposition(scan, omitted.parent, "rejected") + omissions.append(omitted.parent.name) + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_workers SET merge_state = 'buffered' WHERE id = ?", + (omitted.parent.name,), + ) + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_runs SET workflow_version = 'deep-security-scan/v2', " + "status = 'running', phase = 'reducing', terminal_reason = NULL, completed_at = NULL, " + "consecutive_no_new = stop_after_no_new, discovery_runs_dispatched = max_discovery_runs " + "WHERE scan_id = ?", + (scan.scan_id,), + ) + args = Namespace( + scan_id=scan.scan_id, + coordinator_generation=3, + terminal_reason=reason, + manifest_path=str(scan.scan_dir / "scan-manifest.json"), + staged_manifest_path=None, + omitted_worker_id=omissions, + ) + database = tmp_path / "selection.sqlite3" + with sqlite3.connect(database) as connection: + workbench_db.backup(connection) + before = published_bytes(scan) + child = subprocess.run( + [ + sys.executable, + "-c", + _CRASH_SELECTION, + str(Path(__file__).resolve().parents[1] / "scripts" / "workbench_db.py"), + str(database), + json.dumps(vars(args)), + cut, + str(result), + ], + capture_output=True, + text=True, + ) + assert child.returncode == (72 if cut == "before" else 73), (child.stdout, child.stderr) + assert published_bytes(scan) == before + with sqlite3.connect(database) as connection: + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + run = connection.execute("SELECT * FROM deep_scan_runs").fetchone() + selected = json.loads(run["finalization_input_json"]) if cut == "after" else None + if cut == "before": + assert run["finalization_input_json"] is None + assert run["terminal_reason"] is None + # The request names the deleted output. Selection resolves its committed + # accepted attempt rather than reading that replaceable file again. + monkeypatch.setattr(sys, "stdin", io.StringIO(json.dumps({"resultPath": str(result)}))) + if stop_before_replay: + stop_scan(workbench_api, connection, scan, cause) + stopped = published_bytes(scan) + stopped_database = "\n".join(connection.iterdump()) + with pytest.raises(SystemExit, match="running|stopped|failed|canceled"): + workbench_api["deep_scan"].finish_deep_scan( + connection, args, select_finalization=True + ) + assert "\n".join(connection.iterdump()) == stopped_database + assert published_bytes(scan) == stopped + run = connection.execute("SELECT * FROM deep_scan_runs").fetchone() + assert run["status"] == ("canceled" if cause == "cancel" else "failed") + assert run["terminal_reason"] == (reason if cut == "after" else None) + assert ( + json.loads(run["finalization_input_json"]) if cut == "after" else None + ) == selected + assert accepted.read_bytes() == before[accepted.relative_to(scan.scan_dir).as_posix()] + return + replayed = workbench_api["deep_scan"].finish_deep_scan( + connection, args, select_finalization=True + )["deepScan"] + selection = replayed["finalizationInput"] + if selected is not None: + assert selection == selected + assert selection["resultPath"] == accepted.relative_to(scan.scan_dir).as_posix() + assert selection["resultSha256"] == accepted.stem + assert selection["terminalReason"] == reason + assert selection["omittedWorkerIds"] == omissions + assert replayed["terminalReason"] == reason + assert replayed["status"] == "running" + assert published_bytes(scan) == before + stale = stage_publication(scan, generation=2, result_path=accepted, title="Stale aggregate") + with pytest.raises(SystemExit, match="generation"): + workbench_api["saved_results"].write_scan_draft( + workbench_api["_WORKBENCH_DB_CONTEXT"], connection, stale + ) + assert published_bytes(scan) == before + stop_scan(workbench_api, connection, scan, cause) + stopped = published_bytes(scan) + late = stage_publication(scan, generation=3, result_path=accepted, title="Late aggregate") + with pytest.raises(SystemExit, match="stopped"): + workbench_api["saved_results"].write_scan_draft( + workbench_api["_WORKBENCH_DB_CONTEXT"], connection, late + ) + assert published_bytes(scan) == stopped + run = connection.execute("SELECT * FROM deep_scan_runs").fetchone() + assert run["status"] == ("canceled" if cause == "cancel" else "failed") + assert run["terminal_reason"] == reason + assert json.loads(run["finalization_input_json"]) == selection + assert accepted.read_bytes() == before[selection["resultPath"]] + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert len(findings) == 1 + assert findings[0].get("extensions", {}).get("candidateId") != "candidate-disposition" diff --git a/plugins/codex-security/tests/test_publication_stop_interleavings.py b/plugins/codex-security/tests/test_publication_stop_interleavings.py new file mode 100644 index 0000000000..a35778c3b3 --- /dev/null +++ b/plugins/codex-security/tests/test_publication_stop_interleavings.py @@ -0,0 +1,469 @@ +from __future__ import annotations + +import json +import sqlite3 +import subprocess +import sys +from argparse import Namespace +from pathlib import Path + +import pytest +from test_accepted_publication_references import accept_reducer +from test_checkpoint_publication_authority import save_disposition +from test_deep_scan_publication_authority import stage_publication +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan + + +def saved_selection(connection, scan, accepted, omitted=None, *, reason="saturated"): + selection = { + "version": 1, + "resultPath": accepted.relative_to(scan.scan_dir).as_posix(), + "resultSha256": accepted.stem, + "terminalReason": reason, + "omittedWorkerIds": [omitted.parent.name] if omitted is not None else [], + "selectedAt": scan.timestamp, + } + with connection: + connection.execute( + "UPDATE deep_scan_runs SET workflow_version = 'deep-security-scan/v2', " + "finalization_input_json = ?, terminal_reason = ?, phase = 'terminal' " + "WHERE scan_id = ?", + (json.dumps(selection), reason, scan.scan_id), + ) + return selection + + +def stop_scan(api, connection, scan, cause): + if cause == "cancel": + return api["saved_results"].cancel_scan( + api["_WORKBENCH_DB_CONTEXT"], + connection, + Namespace(scan_id=scan.scan_id, thread_id=None), + ) + return api["saved_results"].fail_scan( + api["_WORKBENCH_DB_CONTEXT"], + connection, + Namespace( + scan_id=scan.scan_id, + claim_token=None, + cost_json=None, + message="Scan stopped after reaching the configured cost limit.", + ), + ) + + +def published_bytes(scan): + return { + path.relative_to(scan.scan_dir).as_posix(): path.read_bytes() + for path in scan.scan_dir.rglob("*") + if path.is_file() and "drafts" not in path.relative_to(scan.scan_dir).parts + } + + +_CRASH_STOPPED_PUBLICATION = """ +import os, runpy, sqlite3, sys +from argparse import Namespace + +api = runpy.run_path(sys.argv[1], run_name="stopped_publication_crash_test") +scan_id, cause, boundary = sys.argv[3:] + +class CrashConnection(sqlite3.Connection): + def __exit__(self, *args): + sealing = self.execute( + "SELECT seal_manifest_digest FROM scans WHERE id = ?", (scan_id,) + ).fetchone()[0] is not None + if sealing and boundary == "sqlite-before": + os._exit(72) + result = super().__exit__(*args) + if sealing and boundary == "sqlite-after": + os._exit(73) + return result + +connection = sqlite3.connect(sys.argv[2], factory=CrashConnection) +connection.row_factory = sqlite3.Row +connection.execute("PRAGMA foreign_keys = ON") +import finalize_scan_contract as contract +original_write = contract.write_scan_local_bytes +def crash_after_write(root, relative, contents, **kwargs): + original_write(root, relative, contents, **kwargs) + if relative == boundary: + os._exit(71) +contract.write_scan_local_bytes = crash_after_write +if cause == "cancel": + api["saved_results"].cancel_scan(api["_WORKBENCH_DB_CONTEXT"], connection, Namespace(scan_id=scan_id, thread_id=None)) +else: + api["saved_results"].fail_scan(api["_WORKBENCH_DB_CONTEXT"], connection, Namespace( + scan_id=scan_id, claim_token=None, cost_json=None, + message="Scan stopped after reaching the configured cost limit." + )) +raise AssertionError("stop never reached the requested publication boundary") +""" + +_CRASH_SELECTION_RECOVERY = """ +import os, runpy, sqlite3, sys +from argparse import Namespace + +api = runpy.run_path(sys.argv[1], run_name="selection_recovery_crash_test") +deep = api["deep_scan"] +from types import SimpleNamespace +deep.configure(SimpleNamespace(**{**api, "preserve_stopped_results": api["preserve_stopped_results_after_transition"]})) + +class CrashConnection(sqlite3.Connection): + def __exit__(self, kind, error, traceback): + if kind is None: + if sys.argv[4] == "before": + os._exit(72) + super().__exit__(kind, error, traceback) + os._exit(73) + return super().__exit__(kind, error, traceback) + + def commit(self): + if sys.argv[4] == "before": + os._exit(72) + super().commit() + os._exit(73) + +connection = sqlite3.connect(sys.argv[2], factory=CrashConnection) +connection.row_factory = sqlite3.Row +connection.execute("PRAGMA foreign_keys = ON") +deep.claim_deep_scan_coordinator(connection, Namespace( + scan_id=sys.argv[3], thread_id="fixture-owner", + claim_token=None, coordinator_generation=None, +)) +raise AssertionError("recovery never reached the requested commit boundary") +""" + + +@pytest.mark.parametrize("cause", ["cancel", "cost"]) +@pytest.mark.parametrize("cut", ["before-selection", "selected", "published", "sealed"]) +def test_stop_and_publication_keep_the_winning_terminal_outcome( + workbench_api, workbench_db, publication_scan, tmp_path, cause, cut +): + scan = publication_scan() + _, accepted, coverage = accept_reducer(workbench_db, scan) + omitted = add_worker(workbench_db, scan) + rejected = save_disposition(scan, omitted.parent, "reported") + omitted.write_text(json.dumps(rejected)) + save_disposition(scan, omitted.parent, "rejected") + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_workers SET merge_state = 'buffered' WHERE id = ?", + (omitted.parent.name,), + ) + if cut in {"before-selection", "selected"}: + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', phase = 'reducing', " + "terminal_reason = NULL, completed_at = NULL WHERE scan_id = ?", + (scan.scan_id,), + ) + selection = ( + None + if cut == "before-selection" + else saved_selection(workbench_db, scan, accepted, omitted) + ) + staged = stage_publication( + scan, generation=3, result_path=accepted, title="Selected accepted aggregate" + ) + # The stop path must recover the accepted bytes and the rejection disposition. + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + evidence = {accepted: accepted.read_bytes(), omitted: omitted.read_bytes()} + database_path = tmp_path / "stop-publication.sqlite3" + with sqlite3.connect(database_path) as connection: + workbench_db.backup(connection) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + if cut in {"published", "sealed"}: + workbench_api["saved_results"].write_scan_draft( + workbench_api["_WORKBENCH_DB_CONTEXT"], connection, staged + ) + complete_args = Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + if cut == "sealed": + workbench_api["complete_scan"](connection, complete_args) + before_stop = published_bytes(scan) + if cut == "sealed": + with pytest.raises(SystemExit, match="running|completed"): + stop_scan(workbench_api, connection, scan, cause) + assert published_bytes(scan) == before_stop + else: + stop_scan(workbench_api, connection, scan, cause) + + # Reconnect after the winning commit, then deliver the old publisher response. + with sqlite3.connect(database_path) as connection: + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + row = dict(connection.execute("SELECT * FROM scans").fetchone()) + run = dict(connection.execute("SELECT * FROM deep_scan_runs").fetchone()) + assert row["status"] == ("complete" if cut == "sealed" else "failed") + assert bool(row["canceled_at"]) == (cause == "cancel" and cut != "sealed") + if cause == "cost" and cut != "sealed": + assert row["failure_message"] == ( + "Scan stopped after reaching the configured cost limit." + ) + recorded_selection = json.loads(run["finalization_input_json"] or "null") + if selection is None: + assert recorded_selection is None + else: + digest = recorded_selection.pop("publicationSha256", None) + assert recorded_selection == selection + if cut in {"published", "sealed"}: + assert isinstance(digest, str) and len(digest) == 64 + else: + assert digest is None + if selection is not None: + assert run["terminal_reason"] == selection["terminalReason"] + if cut != "sealed" and cause == "cancel": + assert run["status"] == "canceled" + if cut in {"published", "sealed"}: + assert run["terminal_reason"] == "saturated" + frozen = published_bytes(scan) + with pytest.raises(SystemExit, match="stopped"): + workbench_api["saved_results"].write_scan_draft( + workbench_api["_WORKBENCH_DB_CONTEXT"], connection, staged + ) + if cut == "sealed": + workbench_api["complete_scan"](connection, complete_args) + else: + with pytest.raises(SystemExit): + workbench_api["complete_scan"](connection, complete_args) + workbench_api["saved_results"].preserve_scan_results( + workbench_api["_WORKBENCH_DB_CONTEXT"], + connection, + Namespace( + scan_id=scan.scan_id, + claim_token=None, + thread_id=None, + coordinator_generation=None, + ), + ) + assert published_bytes(scan) == frozen + assert dict(connection.execute("SELECT * FROM scans").fetchone()) == row + assert dict(connection.execute("SELECT * FROM deep_scan_runs").fetchone()) == run + assert all(path.read_bytes() == contents for path, contents in evidence.items()) + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert len(findings) == 1 + assert all( + finding.get("extensions", {}).get("candidateId") != "candidate-disposition" + for finding in findings + ) + if cut != "sealed": + published_coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert coverage["deferred"][0] in published_coverage["deferred"] + assert "completeness" not in published_coverage + + +@pytest.mark.parametrize("cause", ["cancel", "cost"]) +@pytest.mark.parametrize( + "boundary", + ["findings.json", "coverage.json", "scan-manifest.json", "sqlite-before", "sqlite-after"], +) +def test_stopped_publication_process_loss_keeps_frozen_rejection_and_original_selection( + workbench_api, workbench_db, publication_scan, tmp_path, cause, boundary +): + scan = publication_scan() + _, accepted, _ = accept_reducer(workbench_db, scan) + omitted = add_worker(workbench_db, scan) + reported = save_disposition(scan, omitted.parent, "reported") + omitted.write_text(json.dumps(reported)) + save_disposition(scan, omitted.parent, "rejected") + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_workers SET merge_state = 'buffered' WHERE id = ?", + (omitted.parent.name,), + ) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', phase = 'reducing', " + "terminal_reason = NULL, completed_at = NULL WHERE scan_id = ?", + (scan.scan_id,), + ) + selection = saved_selection(workbench_db, scan, accepted, omitted) + staged = stage_publication( + scan, generation=3, result_path=accepted, title="Obsolete selected publication" + ) + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + evidence = {accepted: accepted.read_bytes(), omitted: omitted.read_bytes()} + database_path = tmp_path / "stopped-crash.sqlite3" + with sqlite3.connect(database_path) as connection: + workbench_db.backup(connection) + child = subprocess.run( + [ + sys.executable, + "-c", + _CRASH_STOPPED_PUBLICATION, + str(Path(__file__).resolve().parents[1] / "scripts" / "workbench_db.py"), + str(database_path), + scan.scan_id, + cause, + boundary, + ], + capture_output=True, + text=True, + ) + assert child.returncode == {"sqlite-before": 72, "sqlite-after": 73}.get(boundary, 71), ( + child.stdout, + child.stderr, + ) + with sqlite3.connect(database_path) as connection: + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + row = connection.execute("SELECT * FROM scans").fetchone() + assert row["status"] == "failed" + assert bool(row["canceled_at"]) == (cause == "cancel") + assert bool(row["seal_manifest_digest"]) == (boundary == "sqlite-after") + frozen_sources = row["retained_source_digests_json"] + frozen_heads = row["retained_checkpoint_heads_json"] + assert frozen_sources and frozen_heads + run = dict(connection.execute("SELECT * FROM deep_scan_runs").fetchone()) + workers = [dict(row) for row in connection.execute("SELECT * FROM deep_scan_workers")] + assert json.loads(run["finalization_input_json"]) == selection + assert run["terminal_reason"] == selection["terminalReason"] + assert run["status"] == ("canceled" if cause == "cancel" else "failed") + # The replacement process sees a different live head, but replays the + # already committed stopped selection instead of restoring the candidate. + save_disposition(scan, omitted.parent, "reported") + interrupted = published_bytes(scan) + with pytest.raises(SystemExit, match="stopped"): + workbench_api["saved_results"].write_scan_draft( + workbench_api["_WORKBENCH_DB_CONTEXT"], connection, staged + ) + assert published_bytes(scan) == interrupted + args = Namespace( + scan_id=scan.scan_id, + claim_token=None, + thread_id=None, + coordinator_generation=None, + ) + workbench_api["saved_results"].preserve_scan_results( + workbench_api["_WORKBENCH_DB_CONTEXT"], connection, args + ) + row = connection.execute("SELECT * FROM scans").fetchone() + assert row["status"] == "failed" + assert bool(row["canceled_at"]) == (cause == "cancel") + assert row["retained_source_digests_json"] == frozen_sources + assert row["retained_checkpoint_heads_json"] == frozen_heads + assert row["seal_manifest_digest"] + assert dict(connection.execute("SELECT * FROM deep_scan_runs").fetchone()) == run + assert [ + dict(row) for row in connection.execute("SELECT * FROM deep_scan_workers") + ] == workers + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert len(findings) == 1 + assert findings[0].get("extensions", {}).get("candidateId") != "candidate-disposition" + assert all(path.read_bytes() == contents for path, contents in evidence.items()) + sealed = published_bytes(scan) + workbench_api["saved_results"].preserve_scan_results( + workbench_api["_WORKBENCH_DB_CONTEXT"], connection, args + ) + assert published_bytes(scan) == sealed + assert connection.execute("SELECT COUNT(*) FROM finding_occurrences").fetchone()[0] == 1 + + +@pytest.mark.parametrize("cut", ["before", "after"]) +def test_interrupted_selection_recovery_fences_observers_and_keeps_original_deadline( + workbench_api, workbench_db, publication_scan, tmp_path, cut +): + scan = publication_scan() + _, accepted, _ = accept_reducer(workbench_db, scan) + omitted = add_worker(workbench_db, scan) + selection = saved_selection(workbench_db, scan, accepted, omitted) + with workbench_db: + workbench_db.execute( + "UPDATE scans SET deep_scan_owner_thread_id = 'fixture-owner' WHERE id = ?", + (scan.scan_id,), + ) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', " + "completed_at = NULL, max_time_hours = 1, " + "created_at = '2000-01-01T00:00:00Z', updated_at = '2000-01-01T00:00:00Z' " + "WHERE scan_id = ?", + (scan.scan_id,), + ) + workbench_db.execute( + "UPDATE deep_scan_workers SET merge_state = 'buffered' WHERE id = ?", + (omitted.parent.name,), + ) + database_path = tmp_path / "recovery.sqlite3" + with sqlite3.connect(database_path) as connection: + workbench_db.backup(connection) + original = "\n".join(connection.iterdump()) + before = published_bytes(scan) + child = subprocess.run( + [ + sys.executable, + "-c", + _CRASH_SELECTION_RECOVERY, + str(Path(__file__).resolve().parents[1] / "scripts" / "workbench_db.py"), + str(database_path), + scan.scan_id, + cut, + ], + capture_output=True, + text=True, + ) + assert child.returncode == (72 if cut == "before" else 73), (child.stdout, child.stderr) + deep = workbench_api["deep_scan"] + args = Namespace( + scan_id=scan.scan_id, + thread_id="fixture-owner", + claim_token=None, + coordinator_generation=None, + ) + with sqlite3.connect(database_path) as connection: + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + if cut == "before": + assert "\n".join(connection.iterdump()) == original + run = connection.execute("SELECT * FROM deep_scan_runs").fetchone() + assert run["coordinator_generation"] == (3 if cut == "before" else 4) + workers = [dict(row) for row in connection.execute("SELECT * FROM deep_scan_workers")] + attempts = [dict(row) for row in connection.execute("SELECT * FROM deep_scan_attempts")] + replayed = deep.claim_deep_scan_coordinator(connection, args) + assert replayed["coordinatorDisposition"] == ("adopted" if cut == "before" else "observing") + assert replayed["deepScan"]["coordinatorGeneration"] == 4 + assert replayed["deepScan"]["finalizationInput"] == selection + assert replayed["deepScan"]["createdAt"] == "2000-01-01T00:00:00Z" + assert replayed["deepScan"]["config"]["maxTimeHours"] == 1 + assert replayed["deepScan"]["phase"] == "terminal" + assert replayed["deepScan"]["terminalReason"] == selection["terminalReason"] + assert deep.deep_scan_deadline_reached( + connection.execute("SELECT * FROM deep_scan_runs").fetchone() + ) + stable = "\n".join(connection.iterdump()) + stale_claim = Namespace(**{**vars(args), "coordinator_generation": 3}) + with pytest.raises(SystemExit, match="generation"): + deep.claim_deep_scan_coordinator(connection, stale_claim) + assert "\n".join(connection.iterdump()) == stable + stale = stage_publication(scan, generation=3, result_path=accepted, title="Old coordinator") + with pytest.raises(SystemExit, match="generation"): + workbench_api["saved_results"].write_scan_draft( + workbench_api["_WORKBENCH_DB_CONTEXT"], connection, stale + ) + assert published_bytes(scan) == before + assert [ + dict(row) for row in connection.execute("SELECT * FROM deep_scan_workers") + ] == workers + assert [ + dict(row) for row in connection.execute("SELECT * FROM deep_scan_attempts") + ] == attempts + current = stage_publication( + scan, generation=4, result_path=accepted, title="Recovered selected aggregate" + ) + workbench_api["saved_results"].write_scan_draft( + workbench_api["_WORKBENCH_DB_CONTEXT"], connection, current + ) + assert accepted.read_bytes() == before[accepted.relative_to(scan.scan_dir).as_posix()] + recorded_selection = json.loads( + connection.execute("SELECT finalization_input_json FROM deep_scan_runs").fetchone()[0] + ) + digest = recorded_selection.pop("publicationSha256") + assert isinstance(digest, str) and len(digest) == 64 + assert recorded_selection == selection + # Successful host publication adds only its binding; every other database byte is stable. + with sqlite3.connect(":memory:") as comparison: + connection.backup(comparison) + comparison.execute( + "UPDATE deep_scan_runs SET finalization_input_json = ?", + (json.dumps(selection),), + ) + comparison.commit() + assert "\n".join(comparison.iterdump()) == stable diff --git a/plugins/codex-security/tests/test_report_projection.py b/plugins/codex-security/tests/test_report_projection.py index 4a04015af0..495179aa02 100644 --- a/plugins/codex-security/tests/test_report_projection.py +++ b/plugins/codex-security/tests/test_report_projection.py @@ -888,7 +888,22 @@ def test_projection_keeps_deferred_follow_up_with_open_questions() -> None: assert "Surfaces: parser-surface." in markdown -def test_projection_includes_surface_evidence_receipts() -> None: +@pytest.mark.parametrize( + ("details", "expected"), + [ + ({"notes": "Reviewed parser entrypoints."}, "Reviewed parser entrypoints."), + ({"reason": "Caller policy is unknown."}, "Caller policy is unknown."), + ( + {"notes": "Reviewed parser entrypoints.", "reason": "Caller policy is unknown."}, + "Reviewed parser entrypoints. Caller policy is unknown.", + ), + ({"notes": "Policy checked.", "reason": "Policy checked."}, "Policy checked."), + ({"notes": ""}, ""), + ({"notes": " \t "}, ""), + ({}, "No additional canonical notes were recorded."), + ], +) +def test_projection_includes_surface_evidence_receipts(details: dict, expected: str) -> None: manifest, findings, coverage = canonical_documents() coverage["surfaces"] = [ { @@ -896,13 +911,16 @@ def test_projection_includes_surface_evidence_receipts() -> None: "label": "Parser", "disposition": "no_issue_found", "receiptRefs": ["artifacts/receipts/parser.jsonl"], - "notes": "Reviewed parser entrypoints.", + **details, } ] markdown = PROJECTION.build_report_markdown(manifest, findings, coverage) - assert "Reviewed parser entrypoints. Evidence: artifacts/receipts/parser.jsonl" in markdown + row = next(line for line in markdown.splitlines() if "artifacts/receipts/parser.jsonl" in line) + assert row.endswith( + f"| {expected + ' ' if expected else ''}Evidence: artifacts/receipts/parser.jsonl |" + ) @pytest.mark.parametrize( @@ -935,6 +953,26 @@ def test_projection_preserves_identifier_text_and_code_path_spelling( assert rendered_location in markdown +def test_linked_writeup_retains_distinct_source_fixes() -> None: + manifest, findings, coverage = canonical_documents() + finding = findings["findings"][0] + finding["writeup"] = {"reportPath": "findings/parser/parser.md"} + finding["remediation"] = "Validate the record length." + finding["provenance"] = { + "sourceFindings": [ + {"id": "review-1:0", "finding": {"remediation": "Validate the record length."}}, + {"id": "review-2:0", "finding": {"remediation": "Reject duplicate record keys."}}, + {"id": "review-3:0", "finding": {"remediation": "Reject duplicate record keys."}}, + ] + } + + markdown = PROJECTION.build_report_markdown(manifest, findings, coverage) + + assert "findings/parser/parser.md" in markdown + assert markdown.count("Validate the record length.") == 1 + assert markdown.count("Reject duplicate record keys.") == 1 + + @pytest.mark.parametrize( ("bounds", "rendered_location"), [ diff --git a/plugins/codex-security/tests/test_selected_publication_authority.py b/plugins/codex-security/tests/test_selected_publication_authority.py new file mode 100644 index 0000000000..c87d7304ac --- /dev/null +++ b/plugins/codex-security/tests/test_selected_publication_authority.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import hashlib +import json + +import pytest +from test_deep_scan_publication_authority import stage_publication +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan + + +@pytest.mark.parametrize("publication", ["selected", "mutable", "stale-generation", "unfenced"]) +def test_publication_uses_committed_finalization_selection( + workbench_api, workbench_db, publication_scan, publication +): + scan = publication_scan() + result = add_worker(workbench_db, scan) + contents = json.dumps( + { + "scanId": scan.scan_id, + "complete": True, + "findings": scan.findings, + "coverage": scan.coverage, + } + ).encode() + digest = hashlib.sha256(contents).hexdigest() + accepted = result.parent / "accepted" / f"{digest}.json" + accepted.parent.mkdir() + accepted.write_bytes(contents) + selection = { + "version": 1, + "resultPath": accepted.relative_to(scan.scan_dir).as_posix(), + "resultSha256": digest, + "terminalReason": "saturated", + "omittedWorkerIds": [], + "selectedAt": scan.timestamp, + } + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_workers SET kind = 'dedup', merge_state = 'none' " + "WHERE result_manifest_path = ?", + (str(result),), + ) + workbench_db.execute( + "UPDATE deep_scan_runs SET coordinator_generation = ?, finalization_input_json = ?, " + "workflow_version = 'deep-security-scan/v2' " + "WHERE scan_id = ?", + (1 if publication == "unfenced" else 3, json.dumps(selection), scan.scan_id), + ) + # The accepted bytes survive replacement or deletion of the worker's output. + result.unlink(missing_ok=True) + staged = stage_publication( + scan, + generation=None + if publication == "unfenced" + else 2 + if publication == "stale-generation" + else 3, + result_path=result if publication == "mutable" else accepted, + title="Selected accepted aggregate", + ) + before = {path: path.read_bytes() for path in scan.scan_dir.rglob("*.json")} + + if publication == "selected": + workbench_api["saved_results"].write_scan_draft( + workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, staged + ) + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert findings[0]["title"] == "Selected accepted aggregate" + else: + with pytest.raises(SystemExit, match="coordinator|publication|aggregate"): + workbench_api["saved_results"].write_scan_draft( + workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, staged + ) + assert {path: path.read_bytes() for path in scan.scan_dir.rglob("*.json")} == before + assert accepted.read_bytes() == contents + current = json.loads( + workbench_db.execute( + "SELECT finalization_input_json FROM deep_scan_runs WHERE scan_id = ?", + (scan.scan_id,), + ).fetchone()[0] + ) + if publication == "selected": + digest = current.pop("publicationSha256") + assert len(digest) == 64 and all(char in "0123456789abcdef" for char in digest) + current_scan = workbench_api["require_scan"](workbench_db, scan.scan_id) + db = workbench_api["_WORKBENCH_DB_CONTEXT"] + saved = workbench_api["saved_results"] + prepared = saved._prepare_scan_finalization( + scan.scan_dir, + expected_coverage_mode=db.expected_coverage_mode(current_scan), + completion_binding=db.workbench_completion_binding(current_scan, db.now()), + ) + saved.require_selected_publication(db, workbench_db, current_scan, prepared) + assert current == selection diff --git a/plugins/codex-security/tests/test_selected_publication_compatibility.py b/plugins/codex-security/tests/test_selected_publication_compatibility.py new file mode 100644 index 0000000000..3af0027ea5 --- /dev/null +++ b/plugins/codex-security/tests/test_selected_publication_compatibility.py @@ -0,0 +1,729 @@ +"""Preserve selected Deep Scan publications across main contract normalization.""" + +from __future__ import annotations + +import copy +import hashlib +import json +from argparse import Namespace +from functools import partial + +import pytest +from test_accepted_publication_references import accept_reducer +from test_deep_scan_publication_authority import stage_publication +from test_deep_scan_successful_publication import publication_scan as publication_scan +from test_publication_stop_interleavings import saved_selection +from test_workbench_db import BUDGET_COST + + +def selected_result(connection, scan): + result = scan.scan_dir / "accepted.json" + result.write_text( + json.dumps( + { + "scanId": scan.scan_id, + "complete": True, + "findings": scan.findings, + "sourceCoverage": scan.coverage, + } + ) + ) + selection = { + "version": 1, + "resultPath": result.name, + "resultSha256": hashlib.sha256(result.read_bytes()).hexdigest(), + "terminalReason": "saturated", + "omittedWorkerIds": [], + "selectedAt": scan.timestamp, + } + with connection: + connection.execute( + "UPDATE deep_scan_runs SET workflow_version = 'deep-security-scan/v2', " + "coordinator_generation = 2, finalization_input_json = ? WHERE scan_id = ?", + (json.dumps(selection), scan.scan_id), + ) + return result + + +def selected_publication(api, connection, scan, *, legacy_completeness=None): + if legacy_completeness is not None: + scan.coverage["completeness"] = legacy_completeness + coverage_path = scan.scan_dir / "coverage.json" + coverage = json.loads(coverage_path.read_bytes()) + coverage["completeness"] = legacy_completeness + coverage_path.write_text(json.dumps(coverage)) + result = selected_result(connection, scan) + row = api["require_scan"](connection, scan.scan_id) + prepared = api["_prepare_scan_finalization"]( + scan.scan_dir, + expected_coverage_mode=api["expected_coverage_mode"](row), + completion_binding=api["workbench_completion_binding"](row, api["now"]()), + accepted_findings=api["saved_results"].has_verified_selected_result( + api["_WORKBENCH_DB_CONTEXT"], connection, row + ), + ) + manifest = copy.deepcopy(prepared[2]) + coverage = copy.deepcopy(prepared[4]) + if legacy_completeness is not None: + coverage["completeness"] = legacy_completeness + coverage_bytes = (json.dumps(coverage, indent=2, sort_keys=True) + "\n").encode() + for artifact in manifest["scan"]["artifacts"]: + if artifact["path"] == "coverage.json": + artifact["sha256"] = hashlib.sha256(coverage_bytes).hexdigest() + for key in ("completedAt", "sealedAt"): + manifest["scan"].pop(key, None) + encoded = json.dumps( + [manifest, prepared[3], coverage], sort_keys=True, separators=(",", ":") + ).encode() + selection = json.loads( + connection.execute("SELECT finalization_input_json FROM deep_scan_runs").fetchone()[0] + ) + selection["publicationSha256"] = hashlib.sha256(encoded).hexdigest() + with connection: + connection.execute( + "UPDATE deep_scan_runs SET finalization_input_json = ? WHERE scan_id = ?", + (json.dumps(selection), scan.scan_id), + ) + return result + + +def legacy_finding(scan): + finding = scan.findings[0] + finding.pop("rootCause", None) + finding["root_cause"] = { + "summary": "Previously accepted root cause.", + "evidence_refs": ["legacy-source"], + } + finding["code_evidence"] = [{"id": "legacy-source", "code": "return synthetic_input;"}] + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": scan.findings})) + return {key: copy.deepcopy(finding[key]) for key in ("root_cause", "code_evidence")} + + +def assert_legacy_finding_published(scan, expected): + finding = json.loads((scan.scan_dir / "findings.json").read_text())["findings"][0] + assert {key: finding[key] for key in expected} == expected + report = (scan.scan_dir / "report.md").read_text() + assert "Previously accepted root cause." in report + assert "return synthetic_input;" in report + + +def assert_completed_preparation(api, connection, scan, *, accepted): + files = {path: path.read_bytes() for path in scan.scan_dir.rglob("*") if path.is_file()} + state = "\n".join(connection.iterdump()) + completion = Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + prepared = api["complete_scan"](connection, completion, prepare_only=True) + assert prepared["acceptedFindings"] is accepted + completed = api["complete_scan"](connection, completion) + assert "acceptedFindings" not in completed + assert {path: path.read_bytes() for path in scan.scan_dir.rglob("*") if path.is_file()} == files + assert "\n".join(connection.iterdump()) == state + + +@pytest.mark.parametrize("workflow", ["deep-scan-mcp/v1", "deep-security-scan/v1"]) +@pytest.mark.parametrize("capped", [False, True]) +def test_legacy_v1_accepted_findings_publish_and_resume_completion( + workbench_api, workbench_db, publication_scan, monkeypatch, workflow, capped +): + scan = publication_scan() + expected = legacy_finding(scan) + _, accepted, _ = accept_reducer(workbench_db, scan) + accepted_bytes = accepted.read_bytes() + with workbench_db: + workbench_db.execute("UPDATE deep_scan_runs SET workflow_version = ?", (workflow,)) + if capped: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + staged = stage_publication( + scan, generation=3, result_path=accepted, title=scan.findings[0]["title"] + ) + workbench_api["saved_results"].write_scan_draft( + workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, staged + ) + + def interrupted(*args, **kwargs): + raise RuntimeError("Interrupted legacy completion") + + if capped: + complete = workbench_api["complete_budget_exhausted_scan"] + args = Namespace(scan_id=scan.scan_id, cost_json=json.dumps(BUDGET_COST), message=None) + cut = "complete_scan_locked" + else: + complete = workbench_api["complete_scan"] + args = Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + cut = "_write_prepared_scan_finalization" + with monkeypatch.context() as patch: + patch.setitem(complete.__globals__, cut, interrupted) + with pytest.raises(RuntimeError, match="Interrupted legacy completion"): + complete(workbench_db, args) + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "running" + prepared = workbench_api["complete_scan"]( + workbench_db, + Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None), + prepare_only=True, + ) + assert prepared["acceptedFindings"] is True + complete(workbench_db, args) + assert_legacy_finding_published(scan, expected) + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "complete" + assert ( + workbench_db.execute("SELECT finalization_input_json FROM deep_scan_runs").fetchone()[0] + is None + ) + sealed = {path: path.read_bytes() for path in scan.scan_dir.rglob("*") if path.is_file()} + assert_completed_preparation(workbench_api, workbench_db, scan, accepted=True) + assert {path: path.read_bytes() for path in sealed} == sealed + assert accepted.read_bytes() == accepted_bytes + + +@pytest.mark.parametrize( + "workflow", ["deep-scan-mcp/v1", "deep-security-scan/v1", "deep-security-scan/v2"] +) +@pytest.mark.parametrize("damage", ["accepted-source", "sealed-findings"]) +def test_completed_preparation_rejects_changed_accepted_or_sealed_artifacts( + workbench_api, workbench_db, publication_scan, workflow, damage +): + scan = publication_scan() + legacy_finding(scan) + if workflow.endswith("/v2"): + accepted = selected_publication(workbench_api, workbench_db, scan) + else: + _, accepted, _ = accept_reducer(workbench_db, scan) + with workbench_db: + workbench_db.execute("UPDATE deep_scan_runs SET workflow_version = ?", (workflow,)) + staged = stage_publication( + scan, generation=3, result_path=accepted, title=scan.findings[0]["title"] + ) + workbench_api["saved_results"].write_scan_draft( + workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, staged + ) + completion = Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + workbench_api["complete_scan"](workbench_db, completion) + if damage == "accepted-source": + accepted.write_bytes(accepted.read_bytes() + b"\n") + expected_error = "changed after acceptance" + else: + findings = scan.scan_dir / "findings.json" + findings.write_bytes(findings.read_bytes() + b"\n") + expected_error = "sealed artifact changed" + files = {path: path.read_bytes() for path in scan.scan_dir.rglob("*") if path.is_file()} + state = "\n".join(workbench_db.iterdump()) + with pytest.raises(SystemExit, match=expected_error): + workbench_api["complete_scan"](workbench_db, completion, prepare_only=True) + assert {path: path.read_bytes() for path in scan.scan_dir.rglob("*") if path.is_file()} == files + assert "\n".join(workbench_db.iterdump()) == state + + +def test_legacy_v1_authority_uses_reducer_sequence_instead_of_completion_order( + workbench_api, workbench_db, publication_scan +): + scan = publication_scan() + expected = legacy_finding(scan) + older_result, older_accepted, _ = accept_reducer(workbench_db, scan) + latest_result, latest_accepted, _ = accept_reducer(workbench_db, scan) + with workbench_db: + for sequence, result in enumerate((older_result, latest_result), start=1): + workbench_db.execute( + "UPDATE deep_scan_workers SET prompt_path = ?, completed_at = ? WHERE id = ?", + ( + str(scan.scan_dir / f"dedup-{sequence:04d}" / "prompt.md"), + f"2026-01-0{3 - sequence}", + result.parent.name, + ), + ) + older_accepted.unlink() + staged = stage_publication( + scan, generation=3, result_path=latest_accepted, title=scan.findings[0]["title"] + ) + workbench_api["saved_results"].write_scan_draft( + workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, staged + ) + workbench_api["complete_scan"]( + workbench_db, Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + ) + assert_legacy_finding_published(scan, expected) + + +@pytest.mark.parametrize( + "damage", + [ + "source-before-publication", + "source-before-completion", + "unaccepted", + "no-digest", + "descriptor", + ], +) +def test_legacy_v1_compatibility_requires_the_matching_accepted_source( + workbench_api, workbench_db, publication_scan, damage +): + scan = publication_scan() + legacy_finding(scan) + result, accepted, _ = accept_reducer(workbench_db, scan) + selected = accepted + expected_error = "changed after acceptance" + if damage == "unaccepted": + result.write_bytes(accepted.read_bytes()) + with workbench_db: + workbench_db.execute("DELETE FROM deep_scan_attempts") + selected = result + expected_error = "invalid findings" + elif damage == "no-digest": + with workbench_db: + workbench_db.execute("UPDATE deep_scan_attempts SET accepted_result_sha256 = NULL") + expected_error = "invalid findings" + elif damage == "descriptor": + selected = result + expected_error = "aggregate" + staged = stage_publication( + scan, generation=3, result_path=selected, title=scan.findings[0]["title"] + ) + saved = workbench_api["saved_results"] + completing = damage in {"source-before-completion", "unaccepted", "no-digest"} + if completing: + saved.write_scan_draft(workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, staged) + if damage.startswith("source-"): + accepted.write_bytes(accepted.read_bytes() + b"\n") + files = {path: path.read_bytes() for path in scan.scan_dir.rglob("*") if path.is_file()} + state = "\n".join(workbench_db.iterdump()) + operation = ( + partial( + workbench_api["complete_scan"], + workbench_db, + Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None), + ) + if completing + else partial( + saved.write_scan_draft, workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, staged + ) + ) + with pytest.raises((SystemExit, workbench_api["ContractError"]), match=expected_error): + operation() + assert "\n".join(workbench_db.iterdump()) == state + assert {path: path.read_bytes() for path in scan.scan_dir.rglob("*") if path.is_file()} == files + + +def test_selected_legacy_findings_publish_before_the_first_publication_receipt( + workbench_api, workbench_db, publication_scan +): + scan = publication_scan() + expected = legacy_finding(scan) + accepted = selected_result(workbench_db, scan) + accepted_bytes = accepted.read_bytes() + staged = stage_publication( + scan, generation=2, result_path=accepted, title=scan.findings[0]["title"] + ) + saved = workbench_api["saved_results"] + saved.write_scan_draft(workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, staged) + receipt = workbench_db.execute("SELECT finalization_input_json FROM deep_scan_runs").fetchone()[ + 0 + ] + assert len(json.loads(receipt)["publicationSha256"]) == 64 + workbench_api["complete_scan"]( + workbench_db, Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + ) + assert_legacy_finding_published(scan, expected) + workbench_api["validate_completed_scan_artifacts"]( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_api["require_scan"](workbench_db, scan.scan_id), + ) + assert accepted.read_bytes() == accepted_bytes + assert ( + workbench_db.execute("SELECT finalization_input_json FROM deep_scan_runs").fetchone()[0] + == receipt + ) + + +@pytest.mark.parametrize("cut", ["before", "after"]) +def test_selected_historical_findings_resume_across_interrupted_sealing( + workbench_api, workbench_db, publication_scan, monkeypatch, cut +): + scan = publication_scan() + expected = legacy_finding(scan) + accepted = selected_publication( + workbench_api, workbench_db, scan, legacy_completeness="partial" + ) + accepted_bytes = accepted.read_bytes() + receipt = workbench_db.execute("SELECT finalization_input_json FROM deep_scan_runs").fetchone()[ + 0 + ] + namespace = workbench_api["complete_scan_locked"].__globals__ + write = namespace["_write_prepared_scan_finalization"] + + def interrupted_write(*args, **kwargs): + if cut == "after": + write(*args, **kwargs) + raise RuntimeError("Interrupted historical publication") + + completion = Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + with monkeypatch.context() as patch: + patch.setitem(namespace, "_write_prepared_scan_finalization", interrupted_write) + with pytest.raises(RuntimeError, match="Interrupted historical publication"): + workbench_api["complete_scan"](workbench_db, completion) + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "running" + prepared = workbench_api["complete_scan"](workbench_db, completion, prepare_only=True) + assert prepared["acceptedFindings"] is True + workbench_api["complete_scan"](workbench_db, completion) + assert_legacy_finding_published(scan, expected) + sealed = {p: p.read_bytes() for p in scan.scan_dir.rglob("*") if p.is_file()} + assert_completed_preparation(workbench_api, workbench_db, scan, accepted=True) + assert {p: p.read_bytes() for p in sealed} == sealed + assert accepted.read_bytes() == accepted_bytes + assert ( + workbench_db.execute("SELECT finalization_input_json FROM deep_scan_runs").fetchone()[0] + == receipt + ) + + +@pytest.mark.parametrize("damage", ["canonical-finding", "staged-finding", "accepted-source"]) +def test_selected_legacy_finding_compatibility_retains_publication_integrity( + workbench_api, workbench_db, publication_scan, damage +): + scan = publication_scan() + legacy_finding(scan) + accepted = selected_publication( + workbench_api, workbench_db, scan, legacy_completeness="partial" + ) + if damage == "accepted-source": + accepted.write_bytes(accepted.read_bytes() + b"\n") + elif damage == "canonical-finding": + path = scan.scan_dir / "findings.json" + document = json.loads(path.read_text()) + document["findings"][0]["root_cause"]["summary"] = "Substituted accepted explanation." + path.write_text(json.dumps(document)) + else: + scan.findings[0]["code_evidence"][0]["code"] = "return substituted_input;" + staged = stage_publication( + scan, generation=2, result_path=accepted, title=scan.findings[0]["title"] + ) + before = {p: p.read_bytes() for p in scan.scan_dir.rglob("*") if p.is_file()} + state = "\n".join(workbench_db.iterdump()) + if damage == "staged-finding": + with pytest.raises(workbench_api["ContractError"], match="publication"): + workbench_api["saved_results"].write_scan_draft( + workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, staged + ) + else: + with pytest.raises(SystemExit, match="publication|changed after acceptance"): + workbench_api["complete_scan"]( + workbench_db, Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + ) + assert {p: p.read_bytes() for p in before} == before + assert "\n".join(workbench_db.iterdump()) == state + + +@pytest.mark.parametrize("mode", ["standard", "deep"]) +def test_new_producers_still_reject_legacy_finding_fields( + workbench_api, workbench_db, publication_scan, mode +): + scan = publication_scan(mode=mode) + findings_path = scan.scan_dir / "findings.json" + original_findings = findings_path.read_bytes() + legacy_finding(scan) + with pytest.raises(SystemExit, match="invalid findings.*unexpected schema property"): + workbench_api["complete_scan"]( + workbench_db, Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + ) + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "running" + findings_path.write_bytes(original_findings) + prepared = workbench_api["complete_scan"]( + workbench_db, + Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None), + prepare_only=True, + ) + assert prepared["acceptedFindings"] is False + workbench_api["complete_scan"]( + workbench_db, Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + ) + assert_completed_preparation(workbench_api, workbench_db, scan, accepted=False) + + +def test_zero_result_selection_does_not_admit_legacy_findings( + workbench_api, workbench_db, publication_scan +): + scan = publication_scan() + legacy_finding(scan) + selected_result(workbench_db, scan) + selection = json.loads( + workbench_db.execute("SELECT finalization_input_json FROM deep_scan_runs").fetchone()[0] + ) + selection["resultPath"] = None + selection["resultSha256"] = None + selection["terminalReason"] = "capped" + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_runs SET finalization_input_json = ?, terminal_reason = 'capped'", + (json.dumps(selection),), + ) + with pytest.raises(SystemExit, match="invalid findings.*unexpected schema property"): + workbench_api["complete_scan"]( + workbench_db, Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + ) + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "running" + + +@pytest.mark.parametrize("preselected", [False, True]) +def test_budget_replays_legacy_accepted_findings_after_publication_commit( + workbench_api, workbench_db, publication_scan, monkeypatch, preselected +): + scan = publication_scan() + expected = legacy_finding(scan) + _, accepted, _ = accept_reducer(workbench_db, scan) + accepted_bytes = accepted.read_bytes() + for name in ("scan-manifest.json", "findings.json", "coverage.json"): + (scan.scan_dir / name).unlink() + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', workflow_version = 'deep-security-scan/v2'" + ) + if preselected: + saved_selection(workbench_db, scan, accepted, reason="capped") + budget = workbench_api["complete_budget_exhausted_scan"] + + def interrupted_completion(*args, **kwargs): + raise RuntimeError("Interrupted after accepted budget publication") + + args = Namespace(scan_id=scan.scan_id, cost_json=json.dumps(BUDGET_COST), message=None) + with monkeypatch.context() as patch: + patch.setitem(budget.__globals__, "complete_scan_locked", interrupted_completion) + with pytest.raises(RuntimeError, match="Interrupted after accepted budget publication"): + budget(workbench_db, args) + receipt = workbench_db.execute("SELECT finalization_input_json FROM deep_scan_runs").fetchone()[ + 0 + ] + assert "publicationSha256" in json.loads(receipt) + budget(workbench_db, args) + assert_legacy_finding_published(scan, expected) + assert accepted.read_bytes() == accepted_bytes + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "complete" + assert ( + workbench_db.execute("SELECT finalization_input_json FROM deep_scan_runs").fetchone()[0] + == receipt + ) + + +@pytest.mark.parametrize("legacy_completeness", ["complete", "partial", "unknown"]) +@pytest.mark.parametrize("cut", ["before", "after"]) +def test_selected_replays_historical_publication_across_normalization( + workbench_api, workbench_db, publication_scan, monkeypatch, legacy_completeness, cut +): + scan = publication_scan() + selected_publication(workbench_api, workbench_db, scan, legacy_completeness=legacy_completeness) + selection = workbench_db.execute( + "SELECT finalization_input_json FROM deep_scan_runs" + ).fetchone()[0] + namespace = workbench_api["complete_scan_locked"].__globals__ + write = namespace["_write_prepared_scan_finalization"] + + def interrupted_write(*args, **kwargs): + if cut == "after": + write(*args, **kwargs) + raise RuntimeError("Interrupted parent completion") + + args = Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + with monkeypatch.context() as patch: + patch.setitem(namespace, "_write_prepared_scan_finalization", interrupted_write) + with pytest.raises(RuntimeError, match="Interrupted parent completion"): + workbench_api["complete_scan"](workbench_db, args) + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "running" + if cut == "after": + assert "completeness" not in json.loads((scan.scan_dir / "coverage.json").read_bytes()) + workbench_api["complete_scan"](workbench_db, args) + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "complete" + assert ( + workbench_db.execute("SELECT finalization_input_json FROM deep_scan_runs").fetchone()[0] + == selection + ) + sealed = {p: p.read_bytes() for p in scan.scan_dir.rglob("*.json")} + workbench_api["complete_scan"](workbench_db, args) + assert {p: p.read_bytes() for p in sealed} == sealed + + +@pytest.mark.parametrize("damage", [None, "findings", "coverage", "selected"]) +@pytest.mark.parametrize("legacy_completeness", [None, "complete", "partial", "unknown"]) +def test_selected_completion_validates_saved_publication( + workbench_api, workbench_db, publication_scan, damage, legacy_completeness +): + scan = publication_scan() + selected = selected_publication( + workbench_api, workbench_db, scan, legacy_completeness=legacy_completeness + ) + if damage == "selected": + selected.write_bytes(selected.read_bytes() + b"\n") + elif damage is not None: + path = scan.scan_dir / f"{damage}.json" + document = json.loads(path.read_bytes()) + if damage == "findings": + document["findings"][0]["remediation"] = "Substituted repair." + else: + document["completeness"] = "partial" + document["deferred"] = [ + {"id": "changed-coverage", "reason": "Substituted unresolved review."} + ] + path.write_text(json.dumps(document)) + before = {p: p.read_bytes() for p in scan.scan_dir.rglob("*.json")} + state = "\n".join(workbench_db.iterdump()) + args = Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + if damage is None: + workbench_api["complete_scan"](workbench_db, args) + sealed = {p: p.read_bytes() for p in scan.scan_dir.rglob("*.json")} + workbench_api["complete_scan"](workbench_db, args) + assert {p: p.read_bytes() for p in sealed} == sealed + else: + with pytest.raises(SystemExit, match="publication|changed after acceptance"): + workbench_api["complete_scan"](workbench_db, args) + assert "\n".join(workbench_db.iterdump()) == state + assert {p: p.read_bytes() for p in before} == before + + +def test_selected_historical_publication_still_binds_receipt_bytes( + workbench_api, workbench_db, publication_scan +): + scan = publication_scan() + receipt = scan.scan_dir / "artifacts" / "review.json" + receipt.parent.mkdir() + receipt.write_text('{"review": "original"}\n') + scan.coverage["surfaces"] = [ + { + "id": "saved-review", + "label": "Synthetic saved review", + "disposition": "no_issue_found", + "receiptRefs": ["artifacts/review.json"], + } + ] + (scan.scan_dir / "coverage.json").write_text(json.dumps(scan.coverage)) + selected_publication(workbench_api, workbench_db, scan, legacy_completeness="complete") + receipt.write_text('{"review": "substituted"}\n') + state = "\n".join(workbench_db.iterdump()) + files = {p: p.read_bytes() for p in scan.scan_dir.rglob("*.json")} + with pytest.raises(SystemExit, match="publication"): + workbench_api["complete_scan"]( + workbench_db, Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + ) + assert "\n".join(workbench_db.iterdump()) == state + assert {p: p.read_bytes() for p in files} == files + + +@pytest.mark.parametrize("damage", [None, "draft", "selected"]) +@pytest.mark.parametrize("legacy_completeness", [None, "partial"]) +def test_selected_republication_checks_identity_before_writing_checkpoints( + workbench_api, workbench_db, publication_scan, damage, legacy_completeness +): + scan = publication_scan() + result = selected_publication( + workbench_api, workbench_db, scan, legacy_completeness=legacy_completeness + ) + staged = stage_publication( + scan, + generation=2, + result_path=result, + title="Substituted aggregate" if damage == "draft" else scan.findings[0]["title"], + ) + if damage == "selected": + result.write_bytes(result.read_bytes() + b"\n") + before = {p: p.read_bytes() for p in scan.scan_dir.rglob("*.json")} + state = "\n".join(workbench_db.iterdump()) + if damage is None: + result = workbench_api["saved_results"].write_scan_draft( + workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, staged + ) + assert result["status"] == "draft_written" + workbench_api["complete_scan"]( + workbench_db, Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + ) + return + with pytest.raises( + workbench_api["ContractError"], match="publication|changed after acceptance" + ): + workbench_api["saved_results"].write_scan_draft( + workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, staged + ) + assert {p: p.read_bytes() for p in scan.scan_dir.rglob("*.json")} == before + assert "\n".join(workbench_db.iterdump()) == state + + +def test_selected_zero_discovery_deadline_preserves_the_published_draft( + workbench_api, workbench_db, publication_scan +): + scan = publication_scan() + scan.findings = [] + scan.coverage["deferred"] = [ + { + "id": "deadline", + "reason": "The configured discovery time limit elapsed before source review completed.", + } + ] + selection = { + "version": 1, + "resultPath": None, + "terminalReason": "capped", + "omittedWorkerIds": [], + "selectedAt": scan.timestamp, + } + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_runs SET workflow_version = 'deep-security-scan/v2', " + "coordinator_generation = 2, status = 'running', phase = 'terminal', " + "terminal_reason = 'capped', completed_at = NULL, " + "created_at = '2000-01-01T00:00:00Z', max_time_hours = 1, " + "finalization_input_json = ? WHERE scan_id = ?", + (json.dumps(selection), scan.scan_id), + ) + draft_dir = scan.scan_dir / "drafts" + draft_dir.mkdir() + draft_path = draft_dir / "00000000-0000-4000-8000-000000000001.json" + draft_path.write_text( + json.dumps( + { + "manifest": json.loads((scan.scan_dir / "scan-manifest.json").read_text()), + "findings": {"findings": []}, + "coverage": scan.coverage, + "deepScanPublication": {"coordinatorGeneration": 2, "resultPath": None}, + } + ) + ) + workbench_api["saved_results"].write_scan_draft( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + Namespace( + scan_id=scan.scan_id, + claim_token=None, + draft_path=str(draft_path), + checkpoint_path=None, + expected_draft_digest=None, + ), + ) + selection_json = workbench_db.execute( + "SELECT finalization_input_json FROM deep_scan_runs" + ).fetchone()[0] + assert "publicationSha256" in json.loads(selection_json) + before = {path: path.read_bytes() for path in scan.scan_dir.rglob("*.json")} + finish = Namespace( + scan_id=scan.scan_id, + terminal_reason="capped", + omitted_worker_id=[], + manifest_path=str(scan.scan_dir / "scan-manifest.json"), + staged_manifest_path=None, + coordinator_generation=2, + ) + deep = workbench_api["deep_scan"] + deep.finish_deep_scan(workbench_db, finish) + assert workbench_db.execute("SELECT status FROM deep_scan_runs").fetchone()[0] == "succeeded" + assert {path: path.read_bytes() for path in scan.scan_dir.rglob("*.json")} == before + completion = Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + prepared = workbench_api["complete_scan"](workbench_db, completion, prepare_only=True) + assert prepared["acceptedFindings"] is False + workbench_api["complete_scan"](workbench_db, completion) + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "complete" + sealed = {path: path.read_bytes() for path in scan.scan_dir.rglob("*.json")} + assert_completed_preparation(workbench_api, workbench_db, scan, accepted=False) + deep.finish_deep_scan(workbench_db, finish) + workbench_api["complete_scan"](workbench_db, completion) + assert {path: path.read_bytes() for path in scan.scan_dir.rglob("*.json")} == sealed + assert ( + workbench_db.execute("SELECT finalization_input_json FROM deep_scan_runs").fetchone()[0] + == selection_json + ) diff --git a/plugins/codex-security/tests/test_selected_publication_integrity.py b/plugins/codex-security/tests/test_selected_publication_integrity.py new file mode 100644 index 0000000000..e4dbd8ea8e --- /dev/null +++ b/plugins/codex-security/tests/test_selected_publication_integrity.py @@ -0,0 +1,316 @@ +from __future__ import annotations + +import json +import os +import sqlite3 +from argparse import Namespace +from pathlib import Path + +import pytest +from test_accepted_publication_references import accept_reducer +from test_deep_scan_publication_authority import stage_publication +from test_deep_scan_successful_publication import publication_scan as publication_scan +from test_publication_stop_interleavings import published_bytes, saved_selection +from test_workbench_db import BUDGET_COST + + +def publish_selected(api, connection, scan): + _, accepted, coverage = accept_reducer(connection, scan) + scan.coverage = coverage + selection = saved_selection(connection, scan, accepted) + staged = stage_publication( + scan, generation=3, result_path=accepted, title=scan.findings[0]["title"] + ) + api["saved_results"].write_scan_draft(api["_WORKBENCH_DB_CONTEXT"], connection, staged) + recorded = json.loads( + connection.execute("SELECT finalization_input_json FROM deep_scan_runs").fetchone()[0] + ) + assert {key: recorded[key] for key in selection} == selection + assert len(recorded["publicationSha256"]) == 64 + return accepted, recorded + + +def test_publication_uses_the_staged_draft_checked_for_selection( + workbench_api, workbench_db, publication_scan, monkeypatch +): + scan = publication_scan() + _, accepted, coverage = accept_reducer(workbench_db, scan) + scan.coverage = coverage + saved_selection(workbench_db, scan, accepted) + staged = stage_publication( + scan, generation=3, result_path=accepted, title=scan.findings[0]["title"] + ) + saved = workbench_api["saved_results"] + require_publication = saved._require_current_deep_publication + + def replace_after_selection_check(*args): + require_publication(*args) + staged_path = Path(staged.draft_path) + replacement = json.loads(staged_path.read_bytes()) + replacement["findings"]["findings"][0]["title"] = "Substituted staged finding" + replacement_path = staged_path.with_suffix(".replacement") + replacement_path.write_text(json.dumps(replacement)) + replacement_path.replace(staged_path) + + monkeypatch.setattr(saved, "_require_current_deep_publication", replace_after_selection_check) + saved.write_scan_draft(workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, staged) + workbench_api["complete_scan"]( + workbench_db, Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + ) + published = json.loads((scan.scan_dir / "findings.json").read_bytes()) + assert published["findings"][0]["title"] == scan.findings[0]["title"] + + +def substitute(scan, accepted, defect): + if defect == "accepted-result": + path = accepted + document = json.loads(path.read_bytes()) + document["findings"][0]["title"] = "Substituted accepted finding" + else: + path = scan.scan_dir / ("findings.json" if defect == "findings" else "coverage.json") + document = json.loads(path.read_bytes()) + if defect == "findings": + document["findings"][0]["title"] = "Substituted valid canonical finding" + else: + document["deferred"][0]["reason"] = "Substituted valid unresolved review" + path.write_text(json.dumps(document)) + + +@pytest.mark.parametrize("defect", [None, "findings", "coverage", "accepted-result"]) +@pytest.mark.parametrize("persisted", [False, True], ids=["writer", "persisted-reader"]) +def test_sdk_completion_keeps_selected_publication_binding( + workbench_api, workbench_db, publication_scan, defect, persisted, request +): + scan = publication_scan() + accepted, recorded = publish_selected(workbench_api, workbench_db, scan) + if persisted: + # A resumed reader opens another connection to the persisted selected binding. + recovered = sqlite3.connect(":memory:") + recovered.row_factory = sqlite3.Row + recovered.execute("PRAGMA foreign_keys = ON") + workbench_db.backup(recovered) + request.addfinalizer(recovered.close) + workbench_db = recovered + if defect is not None: + substitute(scan, accepted, defect) + before = published_bytes(scan) + args = Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + + def complete(): + workbench_api["complete_scan"](workbench_db, args, prepare_only=True) + return workbench_api["complete_scan"](workbench_db, args) + + if defect is not None: + with pytest.raises(SystemExit, match="selected.*publication|changed|accepted"): + complete() + assert published_bytes(scan) == before + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "running" + else: + complete() + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "complete" + assert ( + json.loads((scan.scan_dir / "findings.json").read_text())["findings"][0]["title"] + == scan.findings[0]["title"] + ) + # Completion time and seal fields do not invalidate the selected projection. + frozen = published_bytes(scan) + complete() + assert published_bytes(scan) == frozen + assert ( + json.loads( + workbench_db.execute("SELECT finalization_input_json FROM deep_scan_runs").fetchone()[0] + ) + == recorded + ) + + +@pytest.mark.parametrize("defect", [None, "findings", "accepted-result"]) +@pytest.mark.parametrize("lower_bound", [False, True]) +def test_budget_completion_validates_then_binds_intentional_projection( + workbench_api, workbench_db, publication_scan, defect, lower_bound +): + scan = publication_scan() + accepted, recorded = publish_selected(workbench_api, workbench_db, scan) + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + if defect is not None: + substitute(scan, accepted, defect) + before = published_bytes(scan) + args = Namespace( + scan_id=scan.scan_id, + cost_json=json.dumps({"lowerBound": BUDGET_COST} if lower_bound else BUDGET_COST), + message="Synthetic scan reached its configured cost limit.", + ) + if defect is not None: + with pytest.raises(SystemExit, match="selected.*publication|changed|accepted"): + workbench_api["complete_budget_exhausted_scan"](workbench_db, args) + assert published_bytes(scan) == before + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "running" + assert ( + json.loads( + workbench_db.execute( + "SELECT finalization_input_json FROM deep_scan_runs" + ).fetchone()[0] + ) + == recorded + ) + else: + workbench_api["complete_budget_exhausted_scan"](workbench_db, args) + rebound = json.loads( + workbench_db.execute("SELECT finalization_input_json FROM deep_scan_runs").fetchone()[0] + ) + assert {key: rebound[key] for key in recorded if key != "publicationSha256"} == { + key: value for key, value in recorded.items() if key != "publicationSha256" + } + assert rebound["publicationSha256"] != recorded["publicationSha256"] + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "complete" + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert "completeness" not in coverage + assert any(item["id"] == "scan-cost-limit" for item in coverage["deferred"]) + + +@pytest.mark.parametrize("parent_newer_by_ns", [0, 1_000_000_000], ids=["tied", "newer"]) +def test_stopped_recovery_keeps_accepted_reviews_with_later_parent_coverage( + workbench_api, workbench_db, publication_scan, parent_newer_by_ns +): + scan = publication_scan() + _, accepted, coverage = accept_reducer(workbench_db, scan) + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + parent_coverage = scan.scan_dir / "coverage.json" + timestamp = accepted.stat().st_mtime_ns + parent_newer_by_ns + os.utime(parent_coverage, ns=(timestamp, timestamp)) + assert parent_coverage.stat().st_mtime_ns >= accepted.stat().st_mtime_ns + contents = accepted.read_bytes() + + stopped = workbench_api["saved_results"].fail_scan( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + Namespace( + scan_id=scan.scan_id, + claim_token=None, + cost_json=None, + message="Synthetic stopped review.", + ), + )["scan"] + + assert stopped["findingCount"] == 1 + published = json.loads(parent_coverage.read_text()) + assert published["reviews"] == coverage["reviews"] + assert coverage["deferred"][0] in published["deferred"] + manifest = (scan.scan_dir / "scan-manifest.json").read_bytes() + workbench_api["saved_results"].preserve_scan_results( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + Namespace( + scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None + ), + ) + assert (scan.scan_dir / "scan-manifest.json").read_bytes() == manifest + assert accepted.read_bytes() == contents + + +@pytest.mark.parametrize("cut", ["none", "projection", "selected-validation", "completion"]) +def test_budget_completion_uses_one_accepted_snapshot( + workbench_api, workbench_db, publication_scan, monkeypatch, cut +): + scan = publication_scan() + api, connection = workbench_api, workbench_db + if cut == "selected-validation": + accepted, _ = publish_selected(api, connection, scan) + else: + _, accepted, _ = accept_reducer(connection, scan) + for name in ("scan-manifest.json", "findings.json", "coverage.json"): + (scan.scan_dir / name).unlink() + with connection: + connection.execute( + "UPDATE deep_scan_runs SET status='running', phase='discovery', " + "workflow_version='deep-security-scan/v2', manifest_path=NULL, " + "terminal_reason=NULL, completed_at=NULL" + ) + original = accepted.read_bytes() + title = json.loads(original)["findings"][0]["title"] + with connection: + recipe = json.loads(connection.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + connection.execute("UPDATE scans SET recipe_json=?", (json.dumps(recipe),)) + replaced = [] + + def replace_canonical(): + if replaced: + return + path = scan.scan_dir / "findings.json" + document = json.loads(path.read_bytes()) + document["findings"][0]["title"] = "Synthetic replacement after acceptance" + path.write_text(json.dumps(document)) + replaced.append(True) + + if cut == "selected-validation": + original_check = api["saved_results"].require_selected_publication + + def check(*args): + value = original_check(*args) + replace_canonical() + return value + + monkeypatch.setattr(api["saved_results"], "require_selected_publication", check) + elif cut != "none": + namespace = api["complete_budget_exhausted_scan"].__globals__ + key = "budget_exhausted_draft" if cut == "projection" else "complete_scan_locked" + original_call = namespace[key] + + def call(*args, **kwargs): + if cut == "completion": + replace_canonical() + value = original_call(*args, **kwargs) + if cut == "projection": + replace_canonical() + return value + + monkeypatch.setitem(namespace, key, call) + api["complete_budget_exhausted_scan"]( + connection, Namespace(scan_id=scan.scan_id, cost_json=json.dumps(BUDGET_COST), message=None) + ) + assert bool(replaced) is (cut != "none") + assert accepted.read_bytes() == original + assert connection.execute("SELECT status FROM scans").fetchone()[0] == "complete" + published = json.loads((scan.scan_dir / "findings.json").read_bytes()) + assert published["findings"][0]["title"] == title + published["findings"][0]["title"] = "Synthetic corruption after sealing" + (scan.scan_dir / "findings.json").write_text(json.dumps(published)) + with pytest.raises(SystemExit): + api["complete_scan"]( + connection, Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + ) + + +@pytest.mark.parametrize("replace", [False, True]) +def test_completion_checks_the_manifest_it_prepares( + workbench_api, workbench_db, publication_scan, monkeypatch, replace +): + scan = publication_scan() + namespace = workbench_api["complete_scan_locked"].__globals__ + prepare = namespace["_prepare_scan_finalization"] + + def prepare_after_replacement(*args, **kwargs): + if replace: + path = scan.scan_dir / "scan-manifest.json" + manifest = json.loads(path.read_bytes()) + manifest["scan"]["complete"] = False + path.write_text(json.dumps(manifest)) + return prepare(*args, **kwargs) + + monkeypatch.setitem(namespace, "_prepare_scan_finalization", prepare_after_replacement) + args = Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + if replace: + with pytest.raises(SystemExit, match="incomplete"): + workbench_api["complete_scan"](workbench_db, args) + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "running" + assert ( + "sealedAt" + not in json.loads((scan.scan_dir / "scan-manifest.json").read_bytes())["scan"] + ) + else: + workbench_api["complete_scan"](workbench_db, args) + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "complete" diff --git a/plugins/codex-security/tests/test_selected_publication_sealed_replay.py b/plugins/codex-security/tests/test_selected_publication_sealed_replay.py new file mode 100644 index 0000000000..b4e8401c9f --- /dev/null +++ b/plugins/codex-security/tests/test_selected_publication_sealed_replay.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +import json +import sqlite3 +import subprocess +import sys +from argparse import Namespace + +import pytest +from test_accepted_publication_references import accept_reducer +from test_deep_scan_successful_publication import publication_scan as publication_scan +from test_workbench_db import BUDGET_COST + + +@pytest.fixture +def workbench_db(workbench_schema, tmp_path): + with sqlite3.connect(tmp_path / "real-workbench.sqlite3") as connection: + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + workbench_schema.backup(connection) + yield connection + + +@pytest.mark.parametrize( + "replacement", ["none", "findings-only", "coherent-pair", "coherent-unsealed"] +) +def test_sealed_budget_replay_keeps_accepted_projection( + workbench_api, workbench_db, publication_scan, monkeypatch, replacement +): + scan = publication_scan() + _, accepted, _ = accept_reducer(workbench_db, scan) + accepted_bytes = accepted.read_bytes() + accepted_title = json.loads(accepted_bytes)["findings"][0]["title"] + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', workflow_version = 'deep-security-scan/v2'" + ) + budget = workbench_api["complete_budget_exhausted_scan"] + complete = budget.__globals__["complete_scan_locked"] + args = Namespace( + scan_id=scan.scan_id, cost_json=json.dumps(BUDGET_COST), message="Synthetic budget stop." + ) + with monkeypatch.context() as patch: + patch.setitem( + budget.__globals__, + "complete_scan_locked", + lambda *a, **kw: complete(*a, **kw, prepare_only=True), + ) + budget(workbench_db, args) + manifest_path = scan.scan_dir / "scan-manifest.json" + assert json.loads(manifest_path.read_text())["scan"]["sealedAt"] + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "running" + selected_before = json.loads( + workbench_db.execute("SELECT finalization_input_json FROM deep_scan_runs").fetchone()[0] + ) + assert selected_before["publicationSha256"] + saved = workbench_api["saved_results"] + actual_check = saved.require_selected_publication + injected = [] + writer = """ +import hashlib, json, sys +from pathlib import Path +root, mode = Path(sys.argv[1]), sys.argv[2] +p = root / 'findings.json' +findings = json.loads(p.read_bytes()) +findings['findings'][0]['title'] = 'Synthetic replacement after accepted seal' +p.write_text(json.dumps(findings)) +if mode in {'coherent-pair', 'coherent-unsealed'}: + m = root / 'scan-manifest.json' + manifest = json.loads(m.read_bytes()) + for artifact in manifest['scan']['artifacts']: + if artifact['path'] == 'findings.json': + artifact['sha256'] = hashlib.sha256(p.read_bytes()).hexdigest() + if mode == 'coherent-unsealed': + manifest['scan'].pop('sealedAt', None) + manifest['scan'].pop('artifacts', None) + m.write_text(json.dumps(manifest, allow_nan=False, indent=2, sort_keys=True) + chr(10)) +""" + + def check_then_write(*a, **kw): + value = actual_check(*a, **kw) + if replacement != "none" and not injected: + subprocess.run( + [sys.executable, "-I", "-B", "-c", writer, str(scan.scan_dir), replacement], + check=True, + ) + injected.append(True) + return value + + monkeypatch.setattr(saved, "require_selected_publication", check_then_write) + error = None + try: + budget(workbench_db, args) + except SystemExit as exc: + error = str(exc) + selected_after = json.loads( + workbench_db.execute("SELECT finalization_input_json FROM deep_scan_runs").fetchone()[0] + ) + status = workbench_db.execute("SELECT status FROM scans").fetchone()[0] + final_title = json.loads((scan.scan_dir / "findings.json").read_bytes())["findings"][0]["title"] + rows = [dict(r) for r in workbench_db.execute("SELECT * FROM finding_occurrences")] + indexed_replacement = any( + "Synthetic replacement after accepted seal" in json.dumps(r) for r in rows + ) + assert accepted.read_bytes() == accepted_bytes + if replacement != "none": + assert error is not None + assert status == "running" + assert not rows + assert selected_after["publicationSha256"] == selected_before["publicationSha256"] + else: + assert error is None + assert status == "complete" + assert final_title == accepted_title + assert not indexed_replacement + assert selected_after["publicationSha256"] == selected_before["publicationSha256"] diff --git a/plugins/codex-security/tests/test_stopped_accepted_digests.py b/plugins/codex-security/tests/test_stopped_accepted_digests.py new file mode 100644 index 0000000000..b2d427377e --- /dev/null +++ b/plugins/codex-security/tests/test_stopped_accepted_digests.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import copy +import hashlib +import json +import sqlite3 +from argparse import Namespace + +import pytest +from test_accepted_publication_references import accept_reducer +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan + + +@pytest.mark.parametrize("recorded", [True, False], ids=["accepted-receipt", "legacy"]) +@pytest.mark.parametrize("changed", [False, True], ids=["original", "changed"]) +@pytest.mark.parametrize("historical", [False, True], ids=["current-attempt", "prior-attempt"]) +def test_stopped_recovery_checks_recorded_accepted_bytes( + workbench_api, workbench_db, publication_scan, tmp_path, recorded, changed, historical +): + scan = publication_scan() + result, accepted, _ = accept_reducer(workbench_db, scan) + checkpoint = result.parent / "checkpoints" / accepted.name + checkpoint.parent.mkdir() + accepted.rename(checkpoint) + original = checkpoint.read_bytes() + digest = hashlib.sha256(original).hexdigest() + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_attempts SET accepted_result_path = ? WHERE worker_id = ?", + (str(checkpoint), result.parent.name), + ) + if not recorded: + workbench_db.execute( + "DELETE FROM deep_scan_attempts WHERE worker_id = ?", (result.parent.name,) + ) + if historical: + workbench_db.execute( + "UPDATE deep_scan_workers SET attempt = 2, status = 'running' WHERE id = ?", + (result.parent.name,), + ) + if changed: + damaged = json.loads(original) + damaged["findings"][0]["summary"] = "Unaccepted changed evidence." + checkpoint.write_text(json.dumps(damaged)) + accepted_bytes = checkpoint.read_bytes() + # An unrelated valid source must still survive stopped partial preservation. + healthy = add_worker(workbench_db, scan) + finding = copy.deepcopy(scan.findings[0]) + finding["summary"] = "Independent preserved evidence." + finding["identity"]["anchor"] += ".independent" + finding["locations"][0]["startLine"] = 2 + finding["locations"][0]["endLine"] = 2 + healthy.write_text( + json.dumps( + { + "scanId": scan.scan_id, + "complete": True, + "findings": [finding], + "coverage": scan.coverage, + } + ) + ) + healthy_bytes = healthy.read_bytes() + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + database = tmp_path / "accepted-digests.sqlite3" + with sqlite3.connect(database) as connection: + workbench_db.backup(connection) + with sqlite3.connect(database) as connection: + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + stopped = workbench_api["saved_results"].fail_scan( + workbench_api["_WORKBENCH_DB_CONTEXT"], + connection, + Namespace( + scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped." + ), + )["scan"] + summaries = {finding["summary"] for finding in stopped["findings"]} + assert "Independent preserved evidence." in summaries + assert ("Unaccepted changed evidence." in summaries) == (changed and not recorded) + if not changed: + assert scan.findings[0]["summary"] in summaries + if recorded: + assert ( + connection.execute( + "SELECT accepted_result_sha256 FROM deep_scan_attempts WHERE worker_id = ?", + (result.parent.name,), + ).fetchone()[0] + == digest + ) + if recorded and changed: + warnings = json.loads( + connection.execute("SELECT completion_warnings_json FROM scans").fetchone()[0] + ) + assert any("changed after acceptance" in warning for warning in warnings) + assert checkpoint.read_bytes() == accepted_bytes + assert healthy.read_bytes() == healthy_bytes diff --git a/plugins/codex-security/tests/test_stopped_result_version_boundary.py b/plugins/codex-security/tests/test_stopped_result_version_boundary.py new file mode 100644 index 0000000000..334f44ba0d --- /dev/null +++ b/plugins/codex-security/tests/test_stopped_result_version_boundary.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +import hashlib +import json +from argparse import Namespace + +import pytest +from test_checkpoint_publication_authority import save_disposition +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan + + +def snapshot(connection, scan_dir): + return { + "database": "\n".join(connection.iterdump()), + "files": { + path.relative_to(scan_dir).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() + for path in scan_dir.rglob("*") + if path.is_file() + }, + } + + +@pytest.mark.parametrize("operation", ["preserve", "recover"]) +@pytest.mark.parametrize( + "protocol", + ["supported", "supported-v2", "supported-mcp-v1", "future-workflow", "future-selection"], +) +def test_stopped_result_publication_requires_supported_protocol( + workbench_api, workbench_db, publication_scan, monkeypatch, operation, protocol +): + scan = publication_scan() + supported_workflows = { + "supported": "deep-security-scan/v1", + "supported-v2": "deep-security-scan/v2", + "supported-mcp-v1": "deep-scan-mcp/v1", + } + if protocol in supported_workflows: + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_runs SET workflow_version = ? WHERE scan_id = ?", + (supported_workflows[protocol], scan.scan_id), + ) + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + result = add_worker(workbench_db, scan, status="canceled") + draft = save_disposition(scan, result.parent, "reported") + result.write_text(json.dumps(draft)) + + def interrupt_publication(*args, **kwargs): + raise OSError("Synthetic publication interruption") + + with monkeypatch.context() as patch: + patch.setattr( + workbench_api["saved_results"], + "_write_prepared_scan_finalization", + interrupt_publication, + ) + workbench_api["saved_results"].fail_scan( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + Namespace( + scan_id=scan.scan_id, + claim_token=None, + cost_json=None, + message="Original worker stop.", + ), + ) + row = workbench_db.execute("SELECT * FROM scans WHERE id = ?", (scan.scan_id,)).fetchone() + assert row["status"] == "failed" + assert row["retained_source_digests_json"] + assert row["seal_manifest_digest"] is None + + with workbench_db: + if protocol == "future-workflow": + workbench_db.execute( + "UPDATE deep_scan_runs SET workflow_version = 'future/v99' WHERE scan_id = ?", + (scan.scan_id,), + ) + elif protocol == "future-selection": + workbench_db.execute( + "UPDATE deep_scan_runs SET workflow_version = 'deep-security-scan/v2', " + "finalization_input_json = ? WHERE scan_id = ?", + (json.dumps({"version": 99}), scan.scan_id), + ) + before = snapshot(workbench_db, scan.scan_dir) + error = None + try: + if operation == "preserve": + workbench_api["saved_results"].preserve_scan_results( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + Namespace( + scan_id=scan.scan_id, + claim_token=None, + thread_id=None, + coordinator_generation=None, + ), + ) + else: + workbench_api["saved_results"].recover_scan_results( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + Namespace(scan_id=scan.scan_id), + ) + except SystemExit as failure: + error = str(failure) + after = snapshot(workbench_db, scan.scan_dir) + changed_files = sorted( + path + for path in before["files"].keys() | after["files"].keys() + if before["files"].get(path) != after["files"].get(path) + ) + print( + json.dumps( + { + "operation": operation, + "protocol": protocol, + "error": error, + "database_changed": before["database"] != after["database"], + "changed_files": changed_files, + } + ) + ) + if protocol in supported_workflows: + assert error is None + assert before != after + row = workbench_db.execute("SELECT * FROM scans WHERE id = ?", (scan.scan_id,)).fetchone() + assert row["seal_manifest_digest"] + assert row["failure_message"] == "Original worker stop." + run = workbench_db.execute( + "SELECT workflow_version FROM deep_scan_runs WHERE scan_id = ?", (scan.scan_id,) + ).fetchone() + assert run["workflow_version"] == supported_workflows[protocol] + else: + assert error is not None and "unsupported" in error.lower() + assert after == before diff --git a/plugins/codex-security/tests/test_stopped_source_coverage.py b/plugins/codex-security/tests/test_stopped_source_coverage.py new file mode 100644 index 0000000000..4850f07a51 --- /dev/null +++ b/plugins/codex-security/tests/test_stopped_source_coverage.py @@ -0,0 +1,174 @@ +from __future__ import annotations + +import copy +import json +from argparse import Namespace + +import pytest +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan + + +@pytest.mark.parametrize("host_coverage", [True, False], ids=["accepted-projection", "legacy"]) +@pytest.mark.parametrize("parent_draft", [True, False], ids=["parent-draft", "no-parent"]) +def test_stopped_recovery_preserves_accepted_coverage_without_worker_id_collisions( + workbench_api, workbench_db, publication_scan, host_coverage, parent_draft +): + scan = publication_scan() + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + source_coverage = { + "completeness": "partial", + "surfaces": [], + "explicitExclusions": [], + "deferred": [], + "reviews": [], + } + source_files = [] + for disposition in ("needs_follow_up", "rejected"): + result = add_worker(workbench_db, scan) + worker_id = result.parent.name + surface = { + "id": "surface-1", + "candidateId": "candidate-1", + "label": "Independent review", + "disposition": disposition, + "receiptRefs": [], + } + deferred = {"candidateId": "candidate-1", "reason": "Validation remains unresolved."} + coverage = { + "completeness": "partial" if disposition == "needs_follow_up" else "complete", + "surfaces": [surface], + "explicitExclusions": [], + "deferred": [deferred] if disposition == "needs_follow_up" else [], + } + result.write_text( + json.dumps( + {"scanId": scan.scan_id, "complete": True, "findings": [], "coverage": coverage} + ) + ) + source_files.append(result) + prefix = f"{worker_id}-attempt-1" + provenance = {"workerId": worker_id, "attempt": 1, "candidateId": "candidate-1"} + source_coverage["reviews"].append( + {"workerId": worker_id, "attempt": 1, "completeness": coverage["completeness"]} + ) + source_coverage["surfaces"].append( + { + **surface, + "id": f"{prefix}-surface-1", + "provenance": {**provenance, "sourceId": "surface-1"}, + } + ) + if coverage["deferred"]: + source_coverage["deferred"].append( + { + **deferred, + "id": f"{prefix}-deferred-1", + "candidateId": f"{prefix}-candidate-1", + "provenance": provenance, + } + ) + reducer = add_worker(workbench_db, scan) + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_workers SET kind = 'dedup', merge_state = 'none' " + "WHERE result_manifest_path = ?", + (str(reducer),), + ) + aggregate = {"scanId": scan.scan_id, "complete": True, "findings": []} + if host_coverage: + aggregate["sourceCoverage"] = copy.deepcopy(source_coverage) + reducer.write_text(json.dumps(aggregate)) + source_files.append(reducer) + saved_bytes = {path: path.read_bytes() for path in source_files} + if not parent_draft: + for filename in ("scan-manifest.json", "findings.json", "coverage.json"): + (scan.scan_dir / filename).unlink() + + stopped = workbench_api["saved_results"].fail_scan( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."), + )["scan"] + + assert stopped["progress"]["status"] == "failed" + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert "completeness" not in coverage + assert len(coverage["deferred"]) == 2 + assert coverage["deferred"][-1]["id"] == "scan-stopped" + assert len(coverage["surfaces"]) == 2 + if host_coverage: + for field in ("reviews", "surfaces", "deferred"): + assert ( + coverage[field][:-1] if field == "deferred" else coverage[field] + ) == source_coverage[field] + else: + assert coverage["deferred"][0]["candidateId"] == "candidate-1" + manifest = (scan.scan_dir / "scan-manifest.json").read_bytes() + workbench_api["saved_results"].preserve_scan_results( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + Namespace( + scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None + ), + ) + assert (scan.scan_dir / "scan-manifest.json").read_bytes() == manifest + assert all(path.read_bytes() == contents for path, contents in saved_bytes.items()) + + +def test_stopped_recovery_keeps_unmerged_coverage_after_accepted_review( + workbench_api, workbench_db, publication_scan +): + scan = publication_scan() + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + accepted = add_worker(workbench_db, scan) + accepted.write_text( + json.dumps( + {"scanId": scan.scan_id, "complete": True, "findings": [], "coverage": scan.coverage} + ) + ) + reducer = add_worker(workbench_db, scan) + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_workers SET kind = 'dedup', merge_state = 'none' " + "WHERE result_manifest_path = ?", + (str(reducer),), + ) + reviews = [{"workerId": accepted.parent.name, "attempt": 1, "completeness": "complete"}] + reducer.write_text( + json.dumps( + { + "scanId": scan.scan_id, + "complete": True, + "findings": [], + "sourceCoverage": {**scan.coverage, "reviews": reviews}, + } + ) + ) + pending = add_worker(workbench_db, scan, status="canceled") + deferred = {"id": "pending-review", "reason": "The independent review remains unresolved."} + pending.write_text( + json.dumps( + { + "scanId": scan.scan_id, + "complete": False, + "findings": [], + "coverage": {**scan.coverage, "completeness": "partial", "deferred": [deferred]}, + } + ) + ) + + workbench_api["saved_results"].fail_scan( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."), + ) + + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert "completeness" not in coverage + assert coverage["reviews"] == reviews + assert { + **deferred, + "id": f"{pending.parent.name}/{deferred['id']}", + "workerId": pending.parent.name, + } in coverage["deferred"] diff --git a/plugins/codex-security/tests/test_workbench_completion_binding.py b/plugins/codex-security/tests/test_workbench_completion_binding.py index 5e885f6a1c..38770b6360 100644 --- a/plugins/codex-security/tests/test_workbench_completion_binding.py +++ b/plugins/codex-security/tests/test_workbench_completion_binding.py @@ -59,6 +59,39 @@ def _start_deep_scan_with_draft_findings(tmp_path: Path) -> tuple[Path, str, Pat return state_dir, scan_id, scan_dir +@pytest.mark.parametrize("completeness", ["partial", "complete", "unknown"]) +def test_deep_completion_returns_sealed_coverage_on_replay( + tmp_path: Path, completeness: str +) -> None: + state_dir, scan_id, scan_dir = _start_deep_scan_with_draft_findings(tmp_path) + coverage_path = scan_dir / "coverage.json" + coverage = json.loads(coverage_path.read_text()) + coverage["completeness"] = completeness + coverage["deferred"] = ( + [{"id": "external-control", "reason": "Deployment configuration is unavailable."}] + if completeness == "partial" + else [] + ) + coverage_path.write_text(json.dumps(coverage)) + completed = run_workbench(state_dir, "complete-scan", "--scan-id", scan_id) + canonical = json.loads(coverage_path.read_text()) + assert "completeness" not in canonical + assert len(canonical["deferred"]) == (1 if completeness == "partial" else 0) + expected = { + "surfaceCount": len(canonical["surfaces"]), + "deferredCount": len(canonical["deferred"]), + "explicitExclusionCount": len(canonical["explicitExclusions"]), + } + artifacts = ["scan-manifest.json", "findings.json", "coverage.json", "report.md"] + sealed = [(scan_dir / name).read_bytes() for name in artifacts] + replayed = run_workbench(state_dir, "complete-scan", "--scan-id", scan_id) + for result in (completed, replayed): + assert result["scan"]["progress"]["status"] == "complete" + assert result["coverageSummary"] == expected + assert result["scan"]["usage"] == completed["scan"]["usage"] + assert [(scan_dir / name).read_bytes() for name in artifacts] == sealed + + def register_cli_scan( state_dir: Path, target: Path, diff --git a/plugins/codex-security/tests/test_workbench_db.py b/plugins/codex-security/tests/test_workbench_db.py index 9381029f2d..b44e8256fb 100644 --- a/plugins/codex-security/tests/test_workbench_db.py +++ b/plugins/codex-security/tests/test_workbench_db.py @@ -14,6 +14,7 @@ from typing import Any import pytest +import workbench_test_support from workbench_test_support import ( BUDGET_COST, SCRIPT, @@ -68,6 +69,9 @@ BUDGET_WARNING = "Scan stopped: estimated cost $0.00625 exceeded the $0.005 cost limit." EXPECTED_TABLES = { + "deep_scan_attempts", + "deep_scan_attempt_sessions", + "deep_scan_merge_claims", "deep_scan_dedup_inputs", "deep_scan_runs", "deep_scan_worker_threads", @@ -183,6 +187,198 @@ def budget_scan_fixture( return state_dir, target, scan_dir, scan_id, ledger +@pytest.mark.parametrize("operation", ["complete-scan", "complete-budget-exhausted-scan"]) +@pytest.mark.parametrize("protocol", ["supported", "future-workflow", "future-selection"]) +def test_completion_rejects_unknown_protocol_before_mutation( + workbench_api, monkeypatch, tmp_path, operation, protocol +): + script = str(workbench_api["__file__"]) + monkeypatch.setattr(workbench_test_support, "SCRIPT", script) + monkeypatch.setenv("CODEX_HOME", str(tmp_path / "home")) + state_dir, _, scan_dir, scan_id, _ = budget_scan_fixture(tmp_path) + environment = {**os.environ, "CODEX_SECURITY_STATE_DIR": str(state_dir)} + cost_args = ["--scan-id", scan_id, "--cost-json", json.dumps(BUDGET_COST)] + if operation == "complete-scan": + cut_program = """ +import os, runpy, sys +script, *args = sys.argv[1:] +api = runpy.run_path(script, run_name="completion_version_test") +namespace = api["main"].__globals__ +original = namespace["_write_prepared_scan_finalization"] +def after_seal(*args, **kwargs): + original(*args, **kwargs) + os._exit(86) +namespace["_write_prepared_scan_finalization"] = after_seal +sys.argv = [script, *args] +api["main"]() +""" + cut = subprocess.run( + [ + sys.executable, + "-I", + "-B", + "-c", + cut_program, + script, + "complete-budget-exhausted-scan", + *cost_args, + "--message", + BUDGET_WARNING, + ], + env=environment, + capture_output=True, + text=True, + ) + assert cut.returncode == 86, cut.stderr + assert json.loads((scan_dir / "scan-manifest.json").read_text())["scan"]["sealedAt"] + database = state_dir / "workbench.sqlite3" + with sqlite3.connect(database) as connection: + assert connection.execute("SELECT status FROM scans").fetchone() == ("running",) + if protocol == "future-workflow": + connection.execute("UPDATE deep_scan_runs SET workflow_version = 'future/v99'") + elif protocol == "future-selection": + connection.execute( + "UPDATE deep_scan_runs SET workflow_version = 'deep-security-scan/v2', " + "finalization_input_json = ?", + (json.dumps({"version": 99}),), + ) + + def snapshot(): + with sqlite3.connect(database) as connection: + return list(connection.iterdump()), { + str(path.relative_to(scan_dir)): path.read_bytes() + for path in scan_dir.rglob("*") + if path.is_file() + } + + before = snapshot() + result = subprocess.run( + [ + sys.executable, + "-I", + "-B", + script, + operation, + *cost_args, + *([] if operation == "complete-scan" else ["--message", BUDGET_WARNING]), + ], + env=environment, + capture_output=True, + text=True, + ) + after = snapshot() + if protocol == "supported": + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout)["scan"]["progress"]["status"] == "complete" + if operation == "complete-scan": + assert after[1] == before[1] + else: + assert result.returncode != 0 + assert "unsupported" in result.stderr.lower() + assert after == before + + +@pytest.mark.parametrize("legacy_digest", [False, True]) +@pytest.mark.parametrize( + "protocol", ["supported", "legacy-no-run", "future-workflow", "future-selection"] +) +def test_completed_replay_checks_protocol_before_cost_or_digest_writes( + workbench_api, monkeypatch, tmp_path, protocol, legacy_digest +): + monkeypatch.setattr(workbench_test_support, "SCRIPT", workbench_api["__file__"]) + monkeypatch.setenv("CODEX_HOME", str(tmp_path / "home")) + state_dir, _, scan_dir, scan_id, _ = budget_scan_fixture(tmp_path) + complete_budget_scan(state_dir, scan_id) + database = state_dir / "workbench.sqlite3" + with sqlite3.connect(database) as connection: + assert connection.execute("SELECT status FROM scans").fetchone() == ("complete",) + if legacy_digest: + connection.execute("UPDATE scans SET seal_manifest_digest = NULL") + if protocol == "future-workflow": + connection.execute("UPDATE deep_scan_runs SET workflow_version = 'future/v99'") + elif protocol == "future-selection": + connection.execute( + "UPDATE deep_scan_runs SET workflow_version = 'deep-security-scan/v2', " + "finalization_input_json = ?", + (json.dumps({"version": 99}),), + ) + elif protocol == "legacy-no-run": + connection.execute("DELETE FROM deep_scan_runs") + + def snapshot(): + with sqlite3.connect(database) as connection: + return list(connection.iterdump()), { + str(path.relative_to(scan_dir)): path.read_bytes() + for path in scan_dir.rglob("*") + if path.is_file() + } + + before = snapshot() + cost = {**BUDGET_COST, "inputTokens": 1500, "estimatedUsd": 0.0075} + result = run_workbench( + state_dir, + "complete-scan", + "--scan-id", + scan_id, + "--cost-json", + json.dumps(cost), + check=not protocol.startswith("future-"), + ) + after = snapshot() + if protocol.startswith("future-"): + assert after == before + assert result["returncode"] != 0 + assert "unsupported" in result["stderr"].lower() + else: + assert result["scan"]["progress"]["status"] == "complete" + assert result["scan"]["cost"] == cost + assert after[1] == before[1] + with sqlite3.connect(database) as connection: + assert connection.execute("SELECT seal_manifest_digest FROM scans").fetchone()[0] + + +@pytest.mark.parametrize("terminal", [False, True]) +@pytest.mark.parametrize("continuation", ["current", "pending", "claimed"]) +def test_budget_completion_checks_continuation_before_draft_writes( + workbench_api, monkeypatch, tmp_path, terminal, continuation +): + monkeypatch.setattr(workbench_test_support, "SCRIPT", workbench_api["__file__"]) + monkeypatch.setenv("CODEX_HOME", str(tmp_path / "home")) + state_dir, _, scan_dir, scan_id, _ = budget_scan_fixture(tmp_path, terminal=terminal) + database = state_dir / "workbench.sqlite3" + with sqlite3.connect(database) as connection: + connection.execute("UPDATE deep_scan_runs SET workflow_version = 'deep-security-scan/v2'") + if continuation != "current": + connection.execute("UPDATE scans SET handoff_status = 'pending'") + if continuation == "claimed": + run_workbench( + state_dir, + "claim-handoff-delivery", + "--scan-id", + scan_id, + "--claim-token", + str(uuid.uuid4()), + ) + + def snapshot(): + with sqlite3.connect(database) as connection: + return list(connection.iterdump()), { + str(path.relative_to(scan_dir)): path.read_bytes() + for path in scan_dir.rglob("*") + if path.is_file() + } + + before = snapshot() + result = complete_budget_scan(state_dir, scan_id, check=continuation == "current") + after = snapshot() + if continuation == "current": + assert result["scan"]["progress"]["status"] == "complete" + else: + assert after == before + assert result["returncode"] != 0 + assert "owned by another continuation" in result["stderr"] + + def complete_budget_scan(state_dir: Path, scan_id: str, *, check: bool = True) -> dict[str, object]: return scan_command( state_dir, @@ -448,14 +644,33 @@ def test_budget_exhaustion_rejects_scan_below_configured_limit(tmp_path: Path) - assert "has not exceeded its configured cost limit" in str(rejected["stderr"]) -def test_budget_exhaustion_rejects_incomplete_discovery(tmp_path: Path) -> None: - state_dir, _, _, scan_id, _ = budget_scan_fixture(tmp_path, terminal=False) - - rejected = complete_budget_scan(state_dir, scan_id, check=False) - - assert rejected["returncode"] != 0 - assert "requires successfully completed Deep Scan discovery" in str(rejected["stderr"]) - assert get_scan(state_dir, scan_id)["scan"]["progress"]["status"] == "running" +@pytest.mark.parametrize("workflow", ["deep-security-scan/v1", "deep-security-scan/v2"]) +def test_budget_exhaustion_during_discovery_preserves_workflow_contract( + tmp_path: Path, workflow: str +) -> None: + state_dir, _, scan_dir, scan_id, ledger = budget_scan_fixture(tmp_path, terminal=False) + original = ledger.read_bytes() + with sqlite3.connect(state_dir / "workbench.sqlite3") as connection: + connection.execute( + "UPDATE deep_scan_runs SET workflow_version = ? WHERE scan_id = ?", (workflow, scan_id) + ) + result = complete_budget_scan(state_dir, scan_id, check=False) + if workflow == "deep-security-scan/v1": + assert result["returncode"] != 0 + assert "requires successfully completed Deep Scan discovery" in str(result["stderr"]) + assert ( + run_workbench(state_dir, "get-scan", "--scan-id", scan_id)["scan"]["progress"]["status"] + == "running" + ) + else: + assert result["returncode"] == 0 + saved = run_workbench(state_dir, "get-scan", "--scan-id", scan_id)["scan"] + assert saved["progress"]["status"] == "complete" + assert saved["findings"] == [] + assert saved["cost"] == BUDGET_COST + assert saved["warnings"] == [BUDGET_WARNING] + assert "completeness" not in json.loads((scan_dir / "coverage.json").read_text()) + assert ledger.read_bytes() == original def test_budget_exhaustion_rejects_standard_scan(tmp_path: Path) -> None: diff --git a/plugins/codex-security/tests/test_workbench_deep_scan.py b/plugins/codex-security/tests/test_workbench_deep_scan.py index d974a04683..1a0f1bcbf9 100644 --- a/plugins/codex-security/tests/test_workbench_deep_scan.py +++ b/plugins/codex-security/tests/test_workbench_deep_scan.py @@ -272,7 +272,9 @@ def claim() -> dict[str, object]: return claim_deep_scan_coordinator(state_dir, codex_home, scan_id) with sqlite3.connect(state_dir / "workbench.sqlite3") as connection: - schema_version = connection.execute("SELECT MAX(version) FROM schema_migrations").fetchone() + migrations = connection.execute( + "SELECT * FROM schema_migrations ORDER BY version" + ).fetchall() assert claim()["deepScan"]["coordinatorGeneration"] == 2 assert claim()["coordinatorDisposition"] == "observing" expire_deep_scan_coordinator(state_dir, scan_id) @@ -306,8 +308,8 @@ def claim() -> dict[str, object]: assert {result["deepScan"]["coordinatorGeneration"] for result in results} == {3} with sqlite3.connect(state_dir / "workbench.sqlite3") as connection: assert ( - connection.execute("SELECT MAX(version) FROM schema_migrations").fetchone() - == schema_version + connection.execute("SELECT * FROM schema_migrations ORDER BY version").fetchall() + == migrations ) @@ -1750,10 +1752,21 @@ def test_failed_reducer_rebuffers_claimed_inputs_for_same_generation_replacement error="fixture reducer exhausted its attempts", coordinator_generation=2, )["deepScan"] - replayed_workers = {worker["id"]: worker for worker in replayed["workers"]} + assert replayed["workerReceipt"] == failed["workerReceipt"] assert replayed["phase"] == "reducing" - assert replayed["consecutiveErrors"] == counter_before_failure - assert all(replayed_workers[worker]["mergeState"] == "merging" for worker in replacement_inputs) + current = run_workbench( + state_dir, + "get-deep-scan", + "--scan-id", + scan_id, + "--thread-id", + "thread-deep-scan", + environment=deep_environment(codex_home), + )["deepScan"] + current_workers = {worker["id"]: worker for worker in current["workers"]} + assert current["phase"] == "reducing" + assert current["consecutiveErrors"] == counter_before_failure + assert all(current_workers[worker]["mergeState"] == "merging" for worker in replacement_inputs) upsert_worker( state_dir, @@ -3154,7 +3167,7 @@ def test_capped_completion_does_not_discard_canceled_reducer_inputs(tmp_path: Pa rejected = run_workbench( state_dir, *finish_args, environment=deep_environment(codex_home), check=False ) - assert "while discovery output is merging" in str(rejected["stderr"]) + assert "cannot declare omitted buffered workers" in str(rejected["stderr"]) with sqlite3.connect(state_dir / "workbench.sqlite3") as connection: assert connection.execute( "SELECT status FROM deep_scan_runs WHERE scan_id = ?", diff --git a/plugins/codex-security/tests/test_workbench_saved_source_order.py b/plugins/codex-security/tests/test_workbench_saved_source_order.py index 719f296181..e4757095ac 100644 --- a/plugins/codex-security/tests/test_workbench_saved_source_order.py +++ b/plugins/codex-security/tests/test_workbench_saved_source_order.py @@ -21,6 +21,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) import workbench_db import workbench_saved_results as saved +from workbench import saved_sources def call_workbench(monkeypatch, state, codex_home, *args): @@ -105,7 +106,7 @@ def test_head_capture_includes_checkpoint_published_after_enumeration( result.write_bytes(result.read_bytes()) os.utime(result, ns=(300, 300)) directory = (output / "checkpoints").relative_to(scan_dir).as_posix() - children = saved._children + children = saved_sources._children published = [] def publish_after_listing(root, relative): @@ -117,7 +118,7 @@ def publish_after_listing(root, relative): published.append(checkpoint) return names - monkeypatch.setattr(saved, "_children", publish_after_listing) + monkeypatch.setattr(saved_sources, "_children", publish_after_listing) call_workbench( monkeypatch, state, diff --git a/plugins/codex-security/tests/test_workbench_scan_usage.py b/plugins/codex-security/tests/test_workbench_scan_usage.py index b8f96436d9..e62e4f2315 100644 --- a/plugins/codex-security/tests/test_workbench_scan_usage.py +++ b/plugins/codex-security/tests/test_workbench_scan_usage.py @@ -1,11 +1,14 @@ from __future__ import annotations import json +import os import runpy import sqlite3 +import subprocess import sys import tempfile import uuid +from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, replace from datetime import datetime, timedelta from pathlib import Path @@ -14,6 +17,7 @@ import pytest from workbench_test_support import ( + SCRIPT, begin_deep_scan, create_saved_workspace, fail_scan, @@ -41,7 +45,7 @@ class ScanFixture: diff_target: dict[str, Any] | None = None -def _start_scan(tmp_path: Path, *, mode: str = "standard") -> ScanFixture: +def _start_scan(tmp_path: Path, *, mode: str = "standard", dedicated: bool = False) -> ScanFixture: state_dir = tmp_path / "workbench-state" target = tmp_path / "target" environment = { @@ -88,14 +92,38 @@ def _start_scan(tmp_path: Path, *, mode: str = "standard") -> ScanFixture: workspace = create_saved_workspace(state_dir, target, thread_id="scan-parent", mode=mode) workspace_id = str(workspace["id"]) - started = start_delivered_scan( - state_dir, - "--workspace-id", - workspace_id, - "--scan-root", - str(tmp_path / "scans"), - environment=environment, - )["results"] + if dedicated: + scan_dir = tmp_path / "cli-scan" + scan_dir.mkdir(mode=0o700) + started = run_workbench( + state_dir, + "register-cli-scan", + "--scan-dir", + str(scan_dir), + "--repository", + str(target), + "--registration-json-stdin", + input_text=json.dumps( + { + "recipe": { + "config": {}, + "mode": mode, + "repository": str(target), + "target": {"kind": "repository", "paths": []}, + } + } + ), + environment=environment, + ) + else: + started = start_delivered_scan( + state_dir, + "--workspace-id", + workspace_id, + "--scan-root", + str(tmp_path / "scans"), + environment=environment, + )["results"] if mode == "deep": begin_deep_scan( state_dir, @@ -178,10 +206,19 @@ def _rollout( source: Any = "cli" if parent_thread_id is not None: source = {"subagent": {"thread_spawn": {"parent_thread_id": parent_thread_id}}} + own_started_at = ( + task_started_at.isoformat().replace("+00:00", "Z") + if task_started_at is not None + else next((item["timestamp"] for item in events if "timestamp" in item), None) + ) records: list[dict[str, Any]] = [ { "type": "session_meta", - "payload": {"id": recorded_thread_id or thread_id, "source": source}, + "payload": { + "id": recorded_thread_id or thread_id, + "source": source, + "timestamp": own_started_at, + }, }, *(copied_events or []), ] @@ -192,7 +229,15 @@ def _rollout( if parent_thread_id is not None and include_task_start: event: dict[str, Any] = { "type": "event_msg", - "payload": {"type": "task_started", "turn_id": f"turn-{thread_id}"}, + "payload": { + "type": "task_started", + "turn_id": f"turn-{thread_id}", + "started_at": int( + datetime.fromisoformat(own_started_at.replace("Z", "+00:00")).timestamp() + ) + if own_started_at is not None + else None, + }, } if include_task_start_timestamp: event["timestamp"] = ( @@ -1266,14 +1311,406 @@ def test_completion_counts_deep_worker_retries_and_descendants(tmp_path: Path) - assert ( usage.items() >= { - "coverage": "complete", + "coverage": "partial", "source": "codex_rollout", - **_counts(67, 0, 13), - "threadCount": 4, + **_counts(57, 0, 10), + "threadCount": 3, + "missingThreadCount": 1, + "warnings": ["scan_owner_turn_unavailable"], + "modelUsage": [{"model": None, **_counts(57, 0, 10)}], }.items() ) +@pytest.mark.parametrize( + "worker_home", + [ + "recorded", + "current", + "inherited-sqlite", + "current-prefix", + "recorded-prefix", + "current-equal-incomplete", + "recorded-equal-incomplete", + "current-unreadable", + "current-mismatched", + "external-sqlite", + "external-archived", + "external-home-alias", + "external-archived-home-alias", + "external-shared-home", + "external-missing-copy", + "external-missing-child", + "external-current-middle-missing", + "external-current-middle-complete", + "unavailable", + ], +) +def test_completion_keeps_owner_and_workers_in_their_recorded_homes( + tmp_path: Path, worker_home: str +) -> None: + current_home = tmp_path / "current-home" + copied_rollout = worker_home in { + "current-prefix", + "recorded-prefix", + "current-equal-incomplete", + "recorded-equal-incomplete", + "current-unreadable", + "current-mismatched", + } + environment = { + "CODEX_HOME": str(current_home), + "CODEX_SQLITE_HOME": str(current_home / "sqlite"), + "CODEX_STATE_DB": str(current_home / "sqlite" / "state_5.sqlite"), + } + owners = { + f"owner-{index}": _rollout( + tmp_path, + f"owner-{index}", + [_event(datetime.now().astimezone(), "turn_context", {"turn_id": "original"})], + ) + for index in (1, 2) + } + _state_graph(environment, owners, []) + + def complete(index: int) -> dict[str, Any]: + root = tmp_path / f"scan-{index}" + root.mkdir(mode=0o700) + target = root / "target" + target.mkdir() + selected_home = current_home if worker_home == "current" else root / "original-home" + if worker_home == "external-shared-home": + selected_home = tmp_path / "shared-original-home" + if worker_home in {"external-home-alias", "external-archived-home-alias"}: + actual_home = root / "actual-home" + actual_home.mkdir() + selected_home.symlink_to(actual_home, target_is_directory=True) + process = subprocess.run( + [ + sys.executable, + "-c", + ( + "import runpy, sys; script = sys.argv.pop(1); " + "runpy.run_path(script)['main'](with_execution_settings=True)" + ), + str(SCRIPT), + "begin-deep-scan", + "--thread-id", + f"owner-{index}", + "--target-path", + str(target), + "--scan-root", + str(root / "scans"), + ], + env={**os.environ, **environment, "CODEX_SECURITY_STATE_DIR": str(root / "state")}, + input=json.dumps( + { + "executionSettings": { + "codexHome": str(selected_home), + "codexPath": sys.executable, + } + } + ), + capture_output=True, + text=True, + timeout=30, + ) + assert process.returncode == 0, process.stderr + deep = json.loads(process.stdout)["deepScan"] + scan_id, scan_dir = deep["scanId"], Path(deep["scanDir"]) + snapshot = scan_dir / "artifacts/deep_discovery/execution-settings.json" + original_bytes = snapshot.read_bytes() + fixture = ScanFixture( + root / "state", + target, + scan_id, + scan_dir, + datetime.fromisoformat(deep["createdAt"].replace("Z", "+00:00")), + environment, + "deep", + ) + counted = fixture.started_at + timedelta(microseconds=1) + with owners[f"owner-{index}"].open("a") as stream: + stream.write(json.dumps(_token_event(counted, index * 10, 2)) + "\n") + stream.write(json.dumps(_event(counted, "turn_context", {"turn_id": "later"})) + "\n") + stream.write(json.dumps(_token_event(counted, 9000, 900)) + "\n") + worker_threads = {} + for kind in ("discovery", "second-discovery"): + thread_id = f"{kind}-{index}" + artifact = scan_dir / "artifacts" / thread_id + artifact.mkdir(mode=0o700) + prompt = artifact / "prompt.md" + prompt.write_text("Review the synthetic target.\n") + run_workbench( + fixture.state_dir, + "upsert-deep-scan-worker", + "--scan-id", + scan_id, + "--worker-id", + str(uuid.uuid4()), + "--kind", + "discovery", + "--status", + "running", + "--prompt-path", + str(prompt), + "--artifact-dir", + str(artifact), + "--sdk-thread-id", + thread_id, + environment=environment, + ) + worker_threads[thread_id] = _rollout( + root, + thread_id, + [ + *( + [_event(counted, "turn_context", {"model": "model-alpha"})] + if copied_rollout and kind == "discovery" + else [] + ), + _token_event(counted, index * 20, 3), + _event( + counted, + "turn_context", + { + "turn_id": "resumed", + **( + {"model": "model-beta"} + if copied_rollout and kind == "discovery" + else {} + ), + }, + ), + _token_event(counted, index * 30, 5), + ], + ) + child_id = f"child-{index}" + worker_threads[child_id] = _rollout( + root, + child_id, + [_token_event(counted, index * 7, 1)], + parent_thread_id=f"discovery-{index}", + ) + if worker_home in {"current", "inherited-sqlite"}: + with sqlite3.connect(environment["CODEX_STATE_DB"]) as connection: + connection.executemany( + "INSERT INTO threads (id, rollout_path) VALUES (?, ?)", + [(key, str(path)) for key, path in worker_threads.items()], + ) + connection.execute( + "INSERT INTO thread_spawn_edges VALUES (?, ?)", (f"discovery-{index}", child_id) + ) + if worker_home == "inherited-sqlite": + # Earlier launches used A; the resumed process forwards its + # explicit SQLite home C even while workers keep Codex home A. + _state_graph( + {"CODEX_SQLITE_HOME": str(selected_home)}, + {f"discovery-{index}": worker_threads[f"discovery-{index}"]}, + [], + ) + elif worker_home in { + "recorded", + "current-prefix", + "recorded-prefix", + "current-equal-incomplete", + "recorded-equal-incomplete", + "current-unreadable", + "current-mismatched", + }: + recorded_threads = dict(worker_threads) + if worker_home != "recorded": + thread_id = f"discovery-{index}" + full = worker_threads[thread_id] + copied = full.with_name(f"copied-{thread_id}.jsonl") + copied.write_bytes(b"\n".join(full.read_bytes().splitlines()[:3]) + b"\n") + if worker_home in {"current-equal-incomplete", "recorded-equal-incomplete"}: + copied.write_bytes(full.read_bytes() + b'{"type":"event_msg"') + if worker_home == "current-unreadable": + copied.write_text("invalid session metadata\n") + elif worker_home == "current-mismatched": + copied.write_text(full.read_text().replace(thread_id, "unrelated-thread")) + current_copy = copied + if worker_home in {"recorded-prefix", "recorded-equal-incomplete"}: + recorded_threads[thread_id] = copied + current_copy = full + with sqlite3.connect(environment["CODEX_STATE_DB"]) as connection: + connection.execute( + "INSERT INTO threads (id, rollout_path) VALUES (?, ?)", + (thread_id, str(current_copy)), + ) + _state_graph( + {"CODEX_SQLITE_HOME": str(selected_home)}, + recorded_threads, + [(f"discovery-{index}", child_id)], + ) + elif worker_home in { + "external-sqlite", + "external-archived", + "external-home-alias", + "external-archived-home-alias", + "external-shared-home", + "external-missing-copy", + "external-missing-child", + "external-current-middle-missing", + "external-current-middle-complete", + }: + if worker_home.startswith("external-current-middle"): + grandchild_id = f"grandchild-{index}" + worker_threads[grandchild_id] = _rollout( + root, + grandchild_id, + [_token_event(counted, index * 11, 2)], + parent_thread_id=child_id, + ) + # Native keeps rollouts in its Codex home even when its SQLite + # index lives elsewhere and recovery chooses a different index. + sessions = ( + selected_home + / ("archived_sessions" if "archived" in worker_home else "sessions") + / "2026" + / "01" + / "01" + ) + sessions.mkdir(parents=True, exist_ok=True) + for thread_id, path in worker_threads.items(): + recorded = sessions / f"rollout-{thread_id}.jsonl" + path.rename(recorded) + worker_threads[thread_id] = recorded + _state_graph( + {"CODEX_SQLITE_HOME": str(root / "original-external-sqlite")}, + worker_threads, + [(f"discovery-{index}", child_id)], + ) + if worker_home in {"external-missing-copy", "external-missing-child"}: + first_id = f"discovery-{index}" + with sqlite3.connect(environment["CODEX_STATE_DB"]) as connection: + connection.execute( + "INSERT INTO threads (id, rollout_path) VALUES (?, ?)", + ( + first_id, + str(root / "missing-copy.jsonl") + if worker_home == "external-missing-copy" + else str(worker_threads[first_id]), + ), + ) + if worker_home == "external-missing-child": + connection.execute( + "INSERT INTO thread_spawn_edges VALUES (?, ?)", (first_id, child_id) + ) + worker_threads[child_id].unlink() + if worker_home.startswith("external-current-middle"): + # Recovery's current index sees the grandchild while its parent + # rollout is restored later from the worker's original home. + with sqlite3.connect(environment["CODEX_STATE_DB"]) as connection: + connection.executemany( + "INSERT INTO threads (id, rollout_path) VALUES (?, ?)", + [ + (f"discovery-{index}", str(worker_threads[f"discovery-{index}"])), + ( + child_id, + str(root / "missing-middle.jsonl") + if worker_home.endswith("missing") + else str(worker_threads[child_id]), + ), + (grandchild_id, str(worker_threads[grandchild_id])), + ], + ) + connection.executemany( + "INSERT INTO thread_spawn_edges VALUES (?, ?)", + [(f"discovery-{index}", child_id), (child_id, grandchild_id)], + ) + result = _complete_scan(fixture)["scan"]["usage"] + assert snapshot.read_bytes() == original_bytes + return result + + # Both completions share current home B; each scan retains its own worker home A. + with ThreadPoolExecutor(max_workers=2) as pool: + results = list(pool.map(complete, (1, 2))) + for index, usage in enumerate(results, 1): + if worker_home == "unavailable": + assert usage["coverage"] == "partial" + assert usage["inputTokens"] == index * 10 + assert usage["outputTokens"] == 2 + assert usage["threadCount"] == 1 + assert usage["missingThreadCount"] == 2 + elif worker_home == "external-missing-child": + assert ( + usage.items() + >= { + "coverage": "partial", + "source": "codex_rollout", + **_counts(index * 70, 0, 12), + "threadCount": 3, + "missingThreadCount": 1, + "warnings": [ + "codex_state_unavailable", + "rollout_unavailable", + "scan_root_unavailable", + ], + "modelUsage": [{"model": None, **_counts(index * 70, 0, 12)}], + }.items() + ) + else: + assert ( + usage.items() + >= { + "coverage": "complete", + "source": "codex_rollout", + **_counts( + index * (88 if worker_home.startswith("external-current-middle") else 77), + 0, + 15 if worker_home.startswith("external-current-middle") else 13, + ), + "threadCount": 5 if worker_home.startswith("external-current-middle") else 4, + "modelUsage": ( + [ + {"model": None, **_counts(index * 47, 0, 8)}, + {"model": "model-alpha", **_counts(index * 20, 0, 3)}, + {"model": "model-beta", **_counts(index * 10, 0, 2)}, + ] + if copied_rollout + else [ + { + "model": None, + **_counts( + index + * ( + 88 + if worker_home.startswith("external-current-middle") + else 77 + ), + 0, + 15 if worker_home.startswith("external-current-middle") else 13, + ), + } + ] + ), + }.items() + ) + + +def test_recorded_home_alias_keeps_individual_rollout_links_excluded( + tmp_path: Path, workbench_api +) -> None: + reader = sys.modules["workbench_scan_usage"] + home = tmp_path / "home" + sessions = home / "sessions" + sessions.mkdir(parents=True) + alias = tmp_path / "alias" + alias.symlink_to(home, target_is_directory=True) + canonical = _rollout(tmp_path, "worker", []) + path = sessions / "worker.jsonl" + canonical.rename(path) + linked = _rollout(tmp_path, "linked-worker", []) + (sessions / "linked-worker.jsonl").symlink_to(linked) + (sessions / "linked-directory").symlink_to(linked.parent, target_is_directory=True) + + discovered = reader._discover_recorded_worker_sessions(alias, {"worker", "linked-worker"}) + assert [(session.thread_id, session.path) for session in discovered] == [("worker", path)] + assert reader._discover_recorded_worker_sessions(tmp_path / "missing-home", {"worker"}) == [] + + @pytest.mark.parametrize("owner_first", [True, False]) def test_usage_counts_registered_worker_when_its_parent_rollout_is_missing( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, owner_first: bool @@ -1396,7 +1833,7 @@ def test_completion_preserves_explicit_legacy_cost( def test_completion_persists_direct_control_zero_without_hiding_missing_workers( tmp_path: Path, worker_state: str ) -> None: - fixture = _start_scan(tmp_path, mode="deep") + fixture = _start_scan(tmp_path, mode="deep", dedicated=True) counted = fixture.started_at + timedelta(microseconds=1) threads = { # The app-server client persists this originator while injecting context, @@ -1473,7 +1910,7 @@ def test_completion_persists_direct_control_zero_without_hiding_missing_workers( def test_direct_control_resumed_model_turn_requires_a_usage_receipt( tmp_path: Path, turn_start: str ) -> None: - fixture = _start_scan(tmp_path, mode="deep") + fixture = _start_scan(tmp_path, mode="deep", dedicated=True) counted = fixture.started_at + timedelta(microseconds=1) timestamp = ( fixture.started_at - timedelta(seconds=1) @@ -1528,7 +1965,7 @@ def test_direct_control_resumed_model_turn_requires_a_usage_receipt( def test_metadata_only_deep_owner_requires_direct_control_originator( tmp_path: Path, originator: str | None ) -> None: - fixture = _start_scan(tmp_path, mode="deep") + fixture = _start_scan(tmp_path, mode="deep", dedicated=True) _state_graph( fixture.environment, {"scan-parent": _rollout(tmp_path, "scan-parent", [], originator=originator)}, @@ -1878,6 +2315,666 @@ def test_failed_scan_preserves_legacy_failure_behavior(tmp_path: Path) -> None: assert "usage" not in failed +@pytest.mark.parametrize("complete_first", [False, True]) +@pytest.mark.parametrize("tail", ["complete", "partial", "invalid"]) +def test_rollout_copies_prefer_parsed_cache_and_model_corrections( + tmp_path: Path, workbench_api, complete_first: bool, tail: str +) -> None: + reader = sys.modules["workbench_scan_usage"] + start = datetime.fromisoformat("2026-01-01T00:00:00+00:00") + prefix = _rollout( + tmp_path, + "worker", + [ + _event(start, "turn_context", {"turn_id": "scan-turn", "model": "model-before"}), + _token_event(start, 1_000_000, 0), + ], + ) + complete = tmp_path / "complete.jsonl" + tokens = dict( + input_tokens=1_000_000, cached_input_tokens=900_000, output_tokens=0, total_tokens=1_000_000 + ) + receipt = _event( + start, + "token_usage_record", + dict( + response_id="response-one", + thread_id="worker", + turn_id="scan-turn", + model="model-receipt", + usage=tokens, + thread_token_usage=tokens, + ), + ) + complete.write_bytes(prefix.read_bytes() + json.dumps(receipt).encode() + b"\n") + if tail == "partial": + with complete.open("ab") as stream: + stream.write(b'{"type":"event_msg"') + elif tail == "invalid": + with complete.open("ab") as stream: + stream.write(b"invalid-record\n") + paths = [prefix, complete] if not complete_first else [complete, prefix] + models = {} + measurement = reader._read_rollout_copies_usage( + [reader.RolloutSession("worker", None, path) for path in paths], + started_at=start, + completed_at=None, + owner_turn_id="scan-turn", + model_usage=models, + ) + counts, warnings = measurement.tokens, measurement.warnings + expected = _counts(1_000_000, 900_000, 0) + assert counts == expected + assert models == {"model-receipt": expected} + assert warnings == ( + set() + if tail == "complete" + else {"rollout_record_incomplete" if tail == "partial" else "rollout_record_invalid"} + ) + + +def test_rollout_usage_reconciles_stale_cumulative_events_and_models( + tmp_path: Path, workbench_api +) -> None: + usage_reader = sys.modules["workbench_scan_usage"] + start = datetime.fromisoformat("2026-01-01T00:00:00+00:00") + events = [ + _event(start, "turn_context", {"turn_id": "own-turn", "model": "gpt-5.6-sol"}), + _token_event(start, 100, 10), + _token_event(start, 60, 6), + _event(start, "turn_context", {"turn_id": "own-turn", "model": "gpt-6-astra"}), + _token_event(start, 200, 20), + ] + rollout = _rollout(tmp_path, "worker", events) + measurement = usage_reader._read_rollout_usage( + usage_reader.RolloutSession("worker", None, rollout), + started_at=start, + completed_at=None, + ) + counts, warnings = measurement.tokens, measurement.warnings + assert counts == _counts(200, 0, 20) + assert warnings == {"token_counter_regressed"} + models = {} + measurement = usage_reader._read_rollout_usage( + usage_reader.RolloutSession("worker", None, rollout), + started_at=start, + completed_at=None, + model_usage=models, + ) + counts, warnings = measurement.tokens, measurement.warnings + assert counts == _counts(200, 0, 20) + assert warnings == {"token_counter_regressed"} + assert models == {"gpt-5.6-sol": _counts(100, 0, 10), "gpt-6-astra": _counts(100, 0, 10)} + + +@pytest.mark.parametrize("prior_receipt", [False, True]) +def test_shared_parent_usage_requires_original_turn_and_scan_interval( + tmp_path: Path, workbench_api, prior_receipt: bool +) -> None: + usage_reader = sys.modules["workbench_scan_usage"] + start = datetime.fromisoformat("2026-01-01T00:00:00+00:00") + end = start + timedelta(seconds=5) + events = [ + _event( + start - timedelta(seconds=1), + "turn_context", + {"turn_id": "prior", "model": "gpt-5.6-sol"}, + ), + _token_event(start - timedelta(seconds=1), 100, 10), + _event(start, "turn_context", {"turn_id": "scan-turn", "model": "gpt-6-astra"}), + _token_event(start, 110, 12), + _event(start, "turn_context", {"turn_id": "unrelated", "model": "gpt-5.6-sol"}), + _token_event(start, 910, 92), + _event( + end + timedelta(seconds=1), + "turn_context", + {"turn_id": "scan-turn", "model": "gpt-6-astra"}, + ), + _token_event(end + timedelta(seconds=1), 1000, 100), + ] + if prior_receipt: + events.insert( + 0, + _response_event( + start - timedelta(seconds=1), + "parent", + "prior-response", + 100, + 10, + turn_id="prior", + ), + ) + models = {} + measurement = usage_reader._read_rollout_usage( + usage_reader.RolloutSession("parent", None, _rollout(tmp_path, "parent", events)), + started_at=start, + completed_at=end, + owner_turn_id="scan-turn", + model_usage=models, + ) + counts, warnings = measurement.tokens, measurement.warnings + assert counts == _counts(10, 0, 2) + assert warnings == set() + assert models == {"gpt-6-astra": _counts(10, 0, 2)} + + +@pytest.mark.parametrize( + "counter_info,receipt_state,expected_warning", + [ + (None, "complete", None), + ({}, "complete", "token_record_invalid"), + ({"total_token_usage": {"input_tokens": -1}}, "complete", "token_record_invalid"), + (None, "missing-response", "token_receipts_incomplete"), + (None, "incomplete-line", "rollout_record_incomplete"), + (None, "invalid-timestamp", "token_record_invalid"), + ], + ids=[ + "null-counter", + "empty-info", + "malformed-usage", + "missing-response", + "incomplete-line", + "invalid-timestamp", + ], +) +def test_completion_handles_no_usage_counter_without_hiding_incomplete_receipts( + tmp_path: Path, counter_info: Any, receipt_state: str, expected_warning: str | None +) -> None: + fixture = _start_scan(tmp_path) + counted = fixture.started_at + timedelta(microseconds=1) + tokens = dict(input_tokens=100, cached_input_tokens=20, output_tokens=10, total_tokens=110) + response = _event( + counted, + "token_usage_record", + dict( + response_id="response-one", + thread_id="scan-parent", + model="gpt-5.6-sol", + usage=tokens, + thread_token_usage=( + {**tokens, "input_tokens": 150, "total_tokens": 160} + if receipt_state == "missing-response" + else tokens + ), + ), + ) + counter = _event( + counted, + "event_msg", + {"type": "token_count", "info": counter_info, "rate_limits": None}, + ) + events = [counter, response, counter] + if receipt_state == "invalid-timestamp": + events.append( + { + **response, + "timestamp": None, + "payload": {**response["payload"], "response_id": "response-two"}, + } + ) + parent = _rollout(tmp_path, "scan-parent", events) + if receipt_state == "incomplete-line": + with parent.open("a") as stream: + stream.write('{"type":"token_usage_record"') + _state_graph(fixture.environment, {"scan-parent": parent}, []) + usage = _complete_scan(fixture)["scan"]["usage"] + assert usage["totalTokens"] == 110 + assert usage["modelUsage"] == [{"model": "gpt-5.6-sol", **_counts(100, 20, 10)}] + assert usage["coverage"] == ("partial" if expected_warning else "complete") + assert usage.get("warnings", []) == ([expected_warning] if expected_warning else []) + + +@pytest.mark.parametrize("counters", [False, True]) +def test_response_receipts_count_compaction_once_across_resets( + tmp_path: Path, workbench_api, counters: bool +) -> None: + reader = sys.modules["workbench_scan_usage"] + start = datetime.fromisoformat("2026-01-01T00:00:00+00:00") + + def usage(input_tokens, cached, output): + return dict( + input_tokens=input_tokens, + cached_input_tokens=cached, + cache_write_input_tokens=0, + output_tokens=output, + reasoning_output_tokens=0, + total_tokens=input_tokens + output, + ) + + def receipt(response, count, cumulative, model="gpt-5.6-sol", turn="scan-turn", second=1): + return _event( + start + timedelta(seconds=second), + "token_usage_record", + dict( + response_id=response, + thread_id="parent", + turn_id=turn, + model=model, + usage=count, + thread_token_usage=cumulative, + ), + ) + + first = receipt("first", usage(100, 80, 10), usage(100, 80, 10)) + compact = receipt("compaction", usage(50, 40, 5), usage(150, 120, 15), model="gpt-6-astra") + second = receipt("second", usage(120, 90, 12), usage(120, 90, 12)) + events = [ + first, + *([_token_event(start + timedelta(seconds=1), 100, 10)] if counters else []), + compact, + _event(start + timedelta(seconds=1), "compacted", {"message": "Synthetic summary"}), + compact, + second, + *([_token_event(start + timedelta(seconds=1), 220, 22)] if counters else []), + first, + _event( + start + timedelta(seconds=1), + "event_msg", + {"type": "task_complete", "turn_id": "scan-turn"}, + ), + receipt("other", usage(900, 0, 0), usage(900, 0, 0), turn="other-turn"), + receipt("post", usage(800, 0, 0), usage(1700, 0, 0), second=11), + ] + models = {} + measurement = reader._read_rollout_usage( + reader.RolloutSession("parent", None, _rollout(tmp_path, "parent", events)), + started_at=start, + completed_at=start + timedelta(seconds=10), + owner_turn_id="scan-turn", + model_usage=models, + ) + total, warnings = measurement.tokens, measurement.warnings + assert total == _counts(270, 210, 27) + assert warnings == set() + assert models == {"gpt-5.6-sol": _counts(220, 170, 22), "gpt-6-astra": _counts(50, 40, 5)} + + +def test_delayed_response_receipt_resolves_missing_cumulative_usage( + tmp_path: Path, workbench_api +) -> None: + reader = sys.modules["workbench_scan_usage"] + start = datetime.fromisoformat("2026-01-01T00:00:00+00:00") + + def receipt(response, tokens, cumulative): + def usage(value): + return dict(input_tokens=value, output_tokens=0, total_tokens=value) + + return _event( + start, + "token_usage_record", + dict( + response_id=response, + thread_id="parent", + model="gpt-5.6-sol", + usage=usage(tokens), + thread_token_usage=usage(cumulative), + ), + ) + + rollout = _rollout(tmp_path, "parent", [receipt("first", 100, 100), receipt("third", 50, 180)]) + session = reader.RolloutSession("parent", None, rollout) + measurement = reader._read_rollout_usage(session, started_at=start, completed_at=None) + total, warnings = measurement.tokens, measurement.warnings + assert total == _counts(150, 0, 0) + assert warnings == {"token_receipts_incomplete"} + with rollout.open("a") as source: + source.write(json.dumps(receipt("second", 30, 130)) + "\n") + measurement = reader._read_rollout_usage(session, started_at=start, completed_at=None) + total, warnings = measurement.tokens, measurement.warnings + assert total == _counts(180, 0, 0) + assert warnings == set() + + +@pytest.mark.parametrize("missing_receipt", [False, True]) +def test_completion_retains_child_receipt_gap_across_compaction( + tmp_path: Path, missing_receipt: bool +) -> None: + fixture = _start_scan(tmp_path) + counted = fixture.started_at + timedelta(microseconds=1) + + def receipt(thread_id: str, response: str, tokens: int, cumulative: int): + return _event( + counted, + "token_usage_record", + { + "response_id": response, + "thread_id": thread_id, + "turn_id": f"turn-{thread_id}", + "model": "gpt-5.6-sol", + "usage": {"input_tokens": tokens, "output_tokens": 0, "total_tokens": tokens}, + "thread_token_usage": { + "input_tokens": cumulative, + "output_tokens": 0, + "total_tokens": cumulative, + }, + }, + ) + + parent = _rollout( + tmp_path, + "scan-parent", + [ + receipt("scan-parent", "parent", 10, 10), + _event(counted, "event_msg", {"type": "task_complete", "turn_id": "turn-scan-parent"}), + ], + ) + first = receipt("scan-worker", "first", 50 if missing_receipt else 100, 100) + second = receipt("scan-worker", "second", 100, 100) + worker = _rollout( + tmp_path, + "scan-worker", + [ + first, + _event(counted, "compacted", {"message": "Synthetic summary"}), + first, + second, + second, + ], + parent_thread_id="scan-parent", + ) + _state_graph( + fixture.environment, + {"scan-parent": parent, "scan-worker": worker}, + [("scan-parent", "scan-worker")], + ) + usage = _complete_scan(fixture)["scan"]["usage"] + expected = _counts(160 if missing_receipt else 210, 0, 0) + assert usage["totalTokens"] == expected["totalTokens"] + assert usage["modelUsage"] == [{"model": "gpt-5.6-sol", **expected}] + assert usage["coverage"] == ("partial" if missing_receipt else "complete") + assert usage.get("warnings", []) == (["token_receipts_incomplete"] if missing_receipt else []) + + +@pytest.mark.parametrize("prior_turn", [False, True]) +@pytest.mark.parametrize("outside_after", [False, True]) +def test_compaction_before_owned_receipts_does_not_taint_scan_coverage( + tmp_path: Path, workbench_api, prior_turn: bool, outside_after: bool +) -> None: + reader = sys.modules["workbench_scan_usage"] + start = datetime.fromisoformat("2026-01-01T00:00:00+00:00") + + def receipt(response: str, tokens: int, timestamp: datetime, turn: str): + return _event( + timestamp, + "token_usage_record", + { + "response_id": response, + "thread_id": "parent", + "turn_id": turn, + "model": "gpt-5.6-sol", + "usage": {"input_tokens": tokens, "output_tokens": 0, "total_tokens": tokens}, + "thread_token_usage": { + "input_tokens": 100, + "output_tokens": 0, + "total_tokens": 100, + }, + }, + ) + + previous = start if prior_turn else start - timedelta(seconds=1) + if outside_after: + previous = start + timedelta(seconds=1 if prior_turn else 10) + outside = receipt("prior", 50, previous, "other-turn" if prior_turn else "scan-turn") + current = receipt("current", 100, start, "scan-turn") + compacted = _event(previous, "compacted", {"message": "Synthetic previous summary"}) + rollout = _rollout( + tmp_path, + "parent", + [ + *([current, compacted, outside] if outside_after else [outside, compacted, current]), + _event(start, "event_msg", {"type": "task_complete", "turn_id": "scan-turn"}), + ], + ) + measurement = reader._read_rollout_usage( + reader.RolloutSession("parent", None, rollout), + started_at=start, + completed_at=start + timedelta(seconds=5), + owner_turn_id="scan-turn", + ) + counts, warnings = measurement.tokens, measurement.warnings + assert counts == _counts(100, 0, 0) + assert warnings == set() + + +def test_exact_receipts_replace_overlapping_legacy_counter(tmp_path: Path, workbench_api) -> None: + reader = sys.modules["workbench_scan_usage"] + start = datetime.fromisoformat("2026-01-01T00:00:00+00:00") + + def receipt(response, tokens, cumulative): + def usage(value): + return dict(input_tokens=value, output_tokens=0, total_tokens=value) + + return _event( + start, + "token_usage_record", + dict( + response_id=response, + thread_id="parent", + model="gpt-5.6-sol", + usage=usage(tokens), + thread_token_usage=usage(cumulative), + ), + ) + + rollout = _rollout( + tmp_path, + "parent", + [ + _token_event(start, 100, 0), + receipt("new", 10, 110), + receipt("old", 100, 100), + _token_event(start, 10, 0), + ], + ) + models = {} + measurement = reader._read_rollout_usage( + reader.RolloutSession("parent", None, rollout), + started_at=start, + completed_at=None, + model_usage=models, + ) + total, warnings = measurement.tokens, measurement.warnings + assert total == _counts(110, 0, 0) + assert warnings == set() + assert models == {"gpt-5.6-sol": _counts(110, 0, 0)} + + +@pytest.mark.parametrize( + "receipt,counter,expected", + [ + ((150, 15), (100, 200), (150, 200)), + ((150, 15), (100, 20), (150, 20)), + ((100, 50), (110, 10), (110, 50)), + ((100, 50), (200, 10), (200, 50)), + ((100, 10), (100, 10), (100, 10)), + ], + ids=[ + "counter-output", + "smaller-counter-output", + "smaller-counter-input", + "counter-input", + "equal-control", + ], +) +def test_completion_retains_receipt_and_counter_categories(tmp_path, receipt, counter, expected): + fixture = _start_scan(tmp_path) + counted = fixture.started_at + timedelta(microseconds=1) + response = _event( + counted, + "token_usage_record", + { + "response_id": "category-response", + "thread_id": "scan-parent", + "model": "gpt-5.6-sol", + "usage": { + "input_tokens": receipt[0], + "output_tokens": receipt[1], + "total_tokens": sum(receipt), + }, + "thread_token_usage": { + "input_tokens": counter[0], + "output_tokens": counter[1], + "total_tokens": sum(counter), + }, + }, + ) + parent = _rollout(tmp_path, "scan-parent", [response, _token_event(counted, *counter)]) + _state_graph(fixture.environment, {"scan-parent": parent}, []) + usage = _complete_scan(fixture)["scan"]["usage"] + assert (usage["inputTokens"], usage["outputTokens"], usage["totalTokens"]) == ( + *expected, + sum(expected), + ) + for category in ("inputTokens", "outputTokens", "totalTokens"): + assert sum(row[category] for row in usage["modelUsage"]) == usage[category] + assert usage["coverage"] == ("complete" if receipt == counter else "partial") + + +@pytest.mark.parametrize("counter_input,expected_cached", [(100, 20), (250, 90)]) +def test_completion_keeps_cached_subsets_valid_when_categories_diverge( + tmp_path, counter_input, expected_cached +): + fixture = _start_scan(tmp_path) + counted = fixture.started_at + timedelta(microseconds=1) + response = _event( + counted, + "token_usage_record", + { + "response_id": "cache-response", + "thread_id": "scan-parent", + "model": "gpt-5.6-sol", + "usage": { + "input_tokens": 150, + "cached_input_tokens": 20, + "output_tokens": 15, + "total_tokens": 165, + }, + }, + ) + parent = _rollout( + tmp_path, + "scan-parent", + [response, _token_event(counted, counter_input, 200, cached_input_tokens=90)], + ) + _state_graph(fixture.environment, {"scan-parent": parent}, []) + usage = _complete_scan(fixture)["scan"]["usage"] + assert (usage["inputTokens"], usage["cachedInputTokens"], usage["outputTokens"]) == ( + max(150, counter_input), + expected_cached, + 200, + ) + for row in usage["modelUsage"]: + assert row["cachedInputTokens"] + row["cacheWriteInputTokens"] <= row["inputTokens"] + for key in ("inputTokens", "cachedInputTokens", "outputTokens", "totalTokens"): + assert sum(row[key] for row in usage["modelUsage"]) == usage[key] + + +@pytest.mark.parametrize("receipt_owner", ["other-turn", "outside-window"]) +def test_owned_legacy_model_survives_unrelated_first_receipt( + tmp_path: Path, workbench_api, receipt_owner: str +) -> None: + reader = sys.modules["workbench_scan_usage"] + start = datetime.fromisoformat("2026-01-01T00:00:00+00:00") + end = start + timedelta(seconds=5) + events = [ + _event(start, "turn_context", {"turn_id": "scan-turn", "model": "gpt-5.6-sol"}), + _token_event(start, 1000, 10), + _event( + end + timedelta(seconds=1) if receipt_owner == "outside-window" else start, + "token_usage_record", + { + "thread_id": "parent", + "turn_id": "other-turn" if receipt_owner == "other-turn" else "scan-turn", + "response_id": "unrelated-response", + "model": "gpt-6-astra", + "usage": { + "input_tokens": 50, + "cached_input_tokens": 0, + "cache_write_input_tokens": 0, + "output_tokens": 0, + "reasoning_output_tokens": 0, + "total_tokens": 50, + }, + }, + ), + ] + models = {} + measurement = reader._read_rollout_usage( + reader.RolloutSession("parent", None, _rollout(tmp_path, "parent", events)), + started_at=start, + completed_at=end, + owner_turn_id="scan-turn", + model_usage=models, + ) + counts, warnings = measurement.tokens, measurement.warnings + assert counts == _counts(1000, 0, 10) + assert warnings == set() + assert models == {"gpt-5.6-sol": _counts(1000, 0, 10)} + + +@pytest.mark.parametrize("thread_id", ["legacy-worker", "12345678-1234-4234-8234-123456789abc"]) +def test_legacy_worker_usage_excludes_replayed_parent_turns( + tmp_path: Path, workbench_api, thread_id: str +) -> None: + reader = sys.modules["workbench_scan_usage"] + start = datetime.fromisoformat("2026-01-01T00:00:00+00:00") + created = start + timedelta(seconds=10, microseconds=500_000) + rollout = _rollout( + tmp_path, + thread_id, + [_token_event(created + timedelta(seconds=1), 130, 13)], + parent_thread_id="parent", + task_started_at=created, + copied_events=[ + _event( + created, + "event_msg", + { + "type": "task_started", + "turn_id": "parent-turn", + "started_at": int(start.timestamp()), + }, + ), + _token_event(created, 100, 10), + ], + ) + measurement = reader._read_rollout_usage( + reader.RolloutSession(thread_id, "parent", rollout), started_at=start, completed_at=None + ) + total, warnings = measurement.tokens, measurement.warnings + assert total == _counts(30, 0, 3) + assert warnings == set() + + +@pytest.mark.parametrize( + "missing", ["session timestamp", "task started_at", "boolean task started_at"] +) +def test_legacy_worker_usage_requires_turn_ownership_timestamps( + tmp_path: Path, workbench_api, missing: str +) -> None: + reader = sys.modules["workbench_scan_usage"] + start = datetime.fromisoformat("2026-01-01T00:00:00+00:00") + rollout = _rollout( + tmp_path, "legacy-worker", [_token_event(start, 100, 10)], parent_thread_id="parent" + ) + records = [json.loads(line) for line in rollout.read_text().splitlines()] + if missing == "session timestamp": + records[0]["payload"].pop("timestamp") + elif missing == "task started_at": + records[1]["payload"].pop("started_at") + else: + records[1]["payload"]["started_at"] = True + rollout.write_text("".join(json.dumps(record) + "\n" for record in records)) + measurement = reader._read_rollout_usage( + reader.RolloutSession("legacy-worker", "parent", rollout), + started_at=start, + completed_at=None, + ) + total, warnings = measurement.tokens, measurement.warnings + assert total == _counts(0, 0, 0) + assert warnings == {"thread_ownership_unavailable"} + + @pytest.mark.parametrize("worker_kind", ["discovery", "dedup"]) def test_completion_recovers_replaced_deep_worker_threads(tmp_path: Path, worker_kind: str) -> None: fixture = _start_scan(tmp_path, mode="deep") @@ -1931,6 +3028,15 @@ def test_completion_recovers_replaced_deep_worker_threads(tmp_path: Path, worker thread_id, environment=fixture.environment, ) + # Legacy runs have no durable owner turn or earlier worker session records. + with sqlite3.connect(fixture.state_dir / "workbench.sqlite3") as connection: + connection.execute( + "UPDATE deep_scan_runs SET usage_owner_json = NULL WHERE scan_id = ?", + (fixture.scan_id,), + ) + connection.execute( + "DELETE FROM deep_scan_attempt_sessions WHERE scan_id = ?", (fixture.scan_id,) + ) _state_graph( fixture.environment, { @@ -2324,7 +3430,13 @@ def test_closed_turn_coverage_survives_later_success( started = datetime.fromisoformat("2026-01-01T00:00:00+00:00") thread_id = "worker" if worker else "scan-parent" parent_id = "scan-parent" if worker else None - events = [_event(started, "event_msg", {"type": "task_started", "turn_id": "first"})] + events = [ + _event( + started, + "event_msg", + {"type": "task_started", "turn_id": "first", "started_at": int(started.timestamp())}, + ) + ] if receipt in {"native", "zero"}: events.append( _response_event( diff --git a/plugins/codex-security/tests/test_workbench_setup_and_migrations.py b/plugins/codex-security/tests/test_workbench_setup_and_migrations.py index f1f7ec12d6..c98b093430 100644 --- a/plugins/codex-security/tests/test_workbench_setup_and_migrations.py +++ b/plugins/codex-security/tests/test_workbench_setup_and_migrations.py @@ -94,6 +94,12 @@ (50, "persist authorized source excerpt scopes"), (51, "preserve deep scan worker thread history"), (52, "record severity classification review counts"), + (53, "preserve original deep scan discovery context"), + (54, "retain deep scan attempts and exact merge inputs"), + (55, "persist selected deep scan finalization input"), + (56, "freeze stopped scan checkpoint selections"), + (57, "bind original deep scan parent usage turn"), + (58, "bind original deep scan execution settings"), ] @@ -2922,3 +2928,96 @@ def test_workbench_reconciles_profile_and_public_warning_histories( ).fetchone() is not None ) is supported + + +@pytest.mark.parametrize("snapshot", ["early", "previous", "current", "before-refresh", "f57"]) +def test_original_deep_scan_history_upgrades_without_skipping_local_embeddings( + snapshot: str, +) -> None: + apply_migrations = runpy.run_path(str(SCRIPT), run_name="codex_security_workbench_db")[ + "apply_migrations" + ] + original_versions = { + "preserve original deep scan discovery context": 44, + "retain deep scan attempts and exact merge inputs": 45, + "persist selected deep scan finalization input": 46, + "freeze stopped scan checkpoint selections": 47, + "bind original deep scan parent usage turn": 48, + "bind original deep scan execution settings": 51, + } + if snapshot in {"previous", "current", "before-refresh", "f57"}: + original_versions = { + name: version - {"previous": 6, "current": 5, "before-refresh": 3, "f57": 1}[snapshot] + for version, name, _ in SCHEMA.MIGRATIONS + if name in original_versions + } + connection = sqlite3.connect(":memory:") + connection.row_factory = sqlite3.Row + create_migration_history(connection) + timestamp = "2026-01-01T00:00:00Z" + historical = [ + (original_versions.get(name, version), name, sql) + for version, name, sql in SCHEMA.MIGRATIONS + if version + <= ({"early": 43, "previous": 46, "current": 47, "before-refresh": 49, "f57": 51}[snapshot]) + or name in original_versions + ] + apply_historical_migrations(connection, historical, timestamp) + connection.execute( + "INSERT INTO workspaces (id, created_at, updated_at) VALUES ('workspace', ?, ?)", + (timestamp, timestamp), + ) + connection.execute( + """INSERT INTO scans (id, workspace_id, target_path, target_revision, scope, + mode, scan_dir, status, phase, started_at, created_at, updated_at) + VALUES ('scan', 'workspace', '/synthetic/repository', 'revision', '.', + 'deep', '/synthetic/scan', 'running', 'discovery', ?, ?, ?)""", + (timestamp, timestamp, timestamp), + ) + connection.execute( + """INSERT INTO deep_scan_runs (scan_id, schema_version, workflow_version, + status, phase, workers, subagents, stop_after_no_new, max_discovery_runs, + discovery_user_context, finalization_input_json, usage_owner_json, + execution_settings_json, created_at, updated_at) + VALUES ('scan', 1, 'deep-scan-mcp/v1', 'running', 'discovery', 2, 0, 2, 10, + 'original context', '{"result":"original"}', '{"turn":"original"}', + '{"model":"synthetic"}', ?, ?)""", + (timestamp, timestamp), + ) + connection.commit() + original = dict(connection.execute("SELECT * FROM deep_scan_runs").fetchone()) + + apply_migrations(connection) + + upgraded = dict(connection.execute("SELECT * FROM deep_scan_runs").fetchone()) + assert {key: upgraded[key] for key in original} == original + assert upgraded["discovery_user_context_json"] is None + assert connection.execute("SELECT * FROM local_finding_embeddings").fetchall() == [] + assert "cache_key" in { + row["name"] for row in connection.execute("PRAGMA table_info(finding_embeddings)") + } + assert ( + connection.execute( + "SELECT name FROM sqlite_master WHERE type = 'trigger' " + "AND name = 'invalidate_local_finding_embedding'" + ).fetchone() + is not None + ) + recorded = [ + tuple(row) + for row in connection.execute( + "SELECT version, name, applied_at FROM schema_migrations ORDER BY version" + ) + ] + assert [(version, name) for version, name, _ in recorded] == EXPECTED_MIGRATIONS + assert all( + applied_at == timestamp for _, name, applied_at in recorded if name in original_versions + ) + apply_migrations(connection) + assert [ + tuple(row) + for row in connection.execute( + "SELECT version, name, applied_at FROM schema_migrations ORDER BY version" + ) + ] == recorded + connection.close() diff --git a/plugins/codex-security/tests/test_workbench_standard_deep_results.py b/plugins/codex-security/tests/test_workbench_standard_deep_results.py index 2b67175892..dc9fc7f773 100644 --- a/plugins/codex-security/tests/test_workbench_standard_deep_results.py +++ b/plugins/codex-security/tests/test_workbench_standard_deep_results.py @@ -82,8 +82,10 @@ def test_stopped_deep_scan_ignores_late_worker_checkpoints_without_reducer( # The latest incomplete attempt need not be parseable for a saved checkpoint to survive. result_path.write_text("{incomplete") with sqlite3.connect(state_dir / "workbench.sqlite3") as connection: + # This is new incomplete work, not a rewrite of the accepted attempt. connection.execute( - "UPDATE deep_scan_workers SET status = 'running' WHERE id = ?", (worker_id,) + "UPDATE deep_scan_workers SET status = 'running', attempt = 2 WHERE id = ?", + (worker_id,), ) environment = {"CODEX_HOME": str(codex_home)} if termination == "canceled": @@ -410,6 +412,9 @@ def test_explicit_recovery_preserves_sealed_parent_with_empty_source_map( state_dir, codex_home, target, scan_dir, scan_id = deep_scan_fixture(tmp_path) _, result_path = accepted_standard_worker(state_dir, codex_home, scan_dir, scan_id) result_path.unlink() + # Remove the immutable accepted copy too, leaving no recoverable worker source. + for checkpoint in (result_path.parent / "checkpoints").glob("*.json"): + checkpoint.unlink() contract_dir = tmp_path / "contract" contract_dir.mkdir() scripts_dir = Path(__file__).resolve().parents[1] / "scripts" @@ -888,6 +893,9 @@ def test_canceled_scan_reports_noop_coordinator_publication(tmp_path: Path) -> N state_dir, codex_home, _, scan_dir, scan_id = deep_scan_fixture(tmp_path) _, result_path = accepted_standard_worker(state_dir, codex_home, scan_dir, scan_id) result_path.write_text("{incomplete") + # A valid immutable copy would let publication recover despite this corruption. + for checkpoint in (result_path.parent / "checkpoints").glob("*.json"): + checkpoint.unlink() wrapper = tmp_path / "fail_before_canceled_sources_are_frozen.py" canceled = run_workbench_with_fault( @@ -1075,12 +1083,20 @@ def deep_scan_fixture( ) scan_id = str(registered["scanId"]) begin_deep_scan( - state_dir, "standard-worker-thread", "--scan-id", scan_id, environment=environment + state_dir, + "standard-worker-thread", + "--workflow-version", + "deep-security-scan/v1", + "--scan-id", + scan_id, + environment=environment, ) else: begun = begin_deep_scan( state_dir, "standard-worker-thread", + "--workflow-version", + "deep-security-scan/v1", "--target-path", str(target), "--scope", @@ -1143,6 +1159,19 @@ def accepted_standard_worker( str(result_path), environment=environment, ) + # These fixtures represent workers saved before immutable attempt acceptance. + with sqlite3.connect(state_dir / "workbench.sqlite3") as connection: + accepted = connection.execute( + "SELECT accepted_result_path FROM deep_scan_attempts " + "WHERE worker_id = ? AND attempt = 1", + (worker_id,), + ).fetchone() + if accepted is not None and accepted[0] is not None: + Path(accepted[0]).unlink() + connection.execute( + "DELETE FROM deep_scan_attempts WHERE worker_id = ? AND attempt = 1", + (worker_id,), + ) return worker_id, result_path @@ -1252,6 +1281,7 @@ def test_failure_preserves_last_committed_reducer_without_parent_draft( state_dir, codex_home, scan_dir, scan_id, worker_id, result_path ) reduced = json.loads(reducer_path.read_text()) + accepted_summary = reduced["findings"][0]["summary"] reduced["findings"][0]["summary"] = ( "The reducer retained additional independently reviewed evidence." ) @@ -1262,7 +1292,8 @@ def test_failure_preserves_last_committed_reducer_without_parent_draft( failed = get_scan(state_dir, scan_id)["scan"] assert failed["progress"]["status"] == "failed" assert failed["findingCount"] == 1 - assert failed["findings"][0]["summary"] == reduced["findings"][0]["summary"] + assert failed["findings"][0]["summary"] == accepted_summary + assert json.loads(reducer_path.read_text()) == reduced assert json.loads((scan_dir / "coverage.json").read_text())["mode"] == "deep_repository" @@ -1628,7 +1659,7 @@ def test_recovery_selects_strongest_same_finding_checkpoint(tmp_path: Path) -> N strong["confidence"]["level"] = "high" strong["summary"] = "Later strong checkpoint evidence." checkpoint_dir = result_path.parent / "checkpoints" - checkpoint_dir.mkdir() + checkpoint_dir.mkdir(exist_ok=True) for name, finding in (("0" * 64, weak), ("f" * 64, strong)): (checkpoint_dir / f"{name}.json").write_text( json.dumps(saved_draft(scan_id, findings=[finding])) diff --git a/plugins/codex-security/tests/workbench_test_support.py b/plugins/codex-security/tests/workbench_test_support.py index 421dc01b7a..2b56b1a4f4 100644 --- a/plugins/codex-security/tests/workbench_test_support.py +++ b/plugins/codex-security/tests/workbench_test_support.py @@ -725,7 +725,7 @@ def worker_paths(scan_dir: Path, name: str) -> tuple[Path, Path, Path]: def mark_deep_coordinator_succeeded(state_dir: Path, scan_id: str, scan_dir: Path) -> Path: manifest = scan_dir / "artifacts" / "deep_discovery" / "coordinator-manifest.json" - manifest.parent.mkdir(parents=True) + manifest.parent.mkdir(parents=True, exist_ok=True) manifest.write_text('{"status":"succeeded"}\n') with sqlite3.connect(state_dir / "workbench.sqlite3") as connection: connection.execute( diff --git a/sdk/typescript/README.md b/sdk/typescript/README.md index 95f0367248..50dc80a9ca 100644 --- a/sdk/typescript/README.md +++ b/sdk/typescript/README.md @@ -388,6 +388,17 @@ stops, the scan combines and returns completed findings. See Deep discovery retains its enforced deadline, with a default and maximum of 5760 minutes. `maxTimeHours` remains a deprecated Deep-only alias in hours. +New Deep scans preserve the selected aggregate and original terminal reason through +publication and completion retries. Resuming a selected result reuses accepted +findings instead of starting another discovery or reducer run; older scans keep +their recorded workflow. Remaining workers use their recorded execution settings, +while authentication continues to use the original configured account. + +The SDK's `onDeepProgress` callback reports completed, active and maximum independent +reviews. Saved execution attribution and recorded worker homes let resumed usage and +session logs retain their original scan membership. A known priced lower bound can +still stop an explicit budget when another model's price is unavailable. + ### Progress and cost Use `onProgress` for scan progress and `onCost` for cost updates. diff --git a/sdk/typescript/scripts/fixtures/package-deep-codex.mjs b/sdk/typescript/scripts/fixtures/package-deep-codex.mjs index 1ec06b490b..74f4eed2cc 100644 --- a/sdk/typescript/scripts/fixtures/package-deep-codex.mjs +++ b/sdk/typescript/scripts/fixtures/package-deep-codex.mjs @@ -69,6 +69,7 @@ async function run() { }, }, })}\n`, + { mode: 0o600 }, ); result = {}; break; @@ -168,20 +169,17 @@ async function run() { } } else { assert.equal(layout, "reducer"); + let document = ""; let cursor; - let json = ""; do { const page = await server.call( "get_codex_security_deep_reducer_inputs", - { - maxBytes: 64_000, - ...(cursor === undefined ? {} : { cursor }), - }, + { maxBytes: 4096, ...(cursor === undefined ? {} : { cursor }) }, ); - json += page.json; + document += page.json; cursor = page.nextCursor; } while (cursor !== undefined); - const inputs = JSON.parse(json); + const inputs = JSON.parse(document); assert.ok(inputs.discoveries.length > 0); await server.call("record_codex_security_deep_reduction", { scanId: env.CODEX_SECURITY_SCAN_ID, diff --git a/sdk/typescript/scripts/fixtures/package-deep-scan.mjs b/sdk/typescript/scripts/fixtures/package-deep-scan.mjs index 25676eb2eb..a16aa4f88a 100644 --- a/sdk/typescript/scripts/fixtures/package-deep-scan.mjs +++ b/sdk/typescript/scripts/fixtures/package-deep-scan.mjs @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { execFile } from "node:child_process"; -import { randomUUID } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import { chmod, copyFile, @@ -15,7 +15,7 @@ import { } from "node:fs/promises"; import { tmpdir } from "node:os"; import { createRequire } from "node:module"; -import { dirname, join } from "node:path"; +import { basename, dirname, join, resolve } from "node:path"; import { pathToFileURL } from "node:url"; import { promisify } from "node:util"; import { startRpc } from "./package-rpc.mjs"; @@ -362,6 +362,7 @@ async function runDetachedPlugin(pluginRoot, executable) { } finally { await rpc.close(); } + await assertSavedState(f, scanId, owner); await assertExecutions(f, scanId, 4); } @@ -391,6 +392,11 @@ async function runInstalledSdk(pluginRoot, executable) { await readFile(join(pluginRoot, ".codex-plugin", "plugin.json"), "utf8"), ); const owner = "package-sdk-owner"; + const postScanPrompt = "Explain the completed synthetic scan."; + const prompts = []; + let threadCount = 0; + let resumedThreads = 0; + let manifestBeforeFollowUp; let scanId; const client = new sdk.CodexSecurity( { pythonPath: f.env.PYTHON }, @@ -410,19 +416,52 @@ async function runInstalledSdk(pluginRoot, executable) { version: manifest.version, }, }), - // The installed SDK runs the direct engine without a parent model turn. - createCodex() { + // The native control session starts the real packaged engine. Only the + // post-scan model response is synthetic, after authoritative completion. + createCodex({ env }) { return { startThread() { + threadCount += 1; return { - id: owner, + id: null, async runStreamed() { - assert.fail( - "The direct engine must not start a parent model turn.", + throw new Error( + "The direct Deep Scan must not start a parent model turn.", ); }, }; }, + resumeThread(threadId) { + assert.equal(threadId, owner); + resumedThreads += 1; + return { + id: threadId, + async runStreamed(prompt) { + prompts.push(prompt); + assert.equal(prompt, postScanPrompt); + manifestBeforeFollowUp = await readFile( + join(env.CODEX_SECURITY_SCAN_DIR, "scan-manifest.json"), + "utf8", + ); + const completed = JSON.parse(manifestBeforeFollowUp); + assert.equal(completed.scan.status, "completed"); + assert.ok(completed.scan.sealedAt); + return { + events: (async function* () { + yield { type: "thread.started", thread_id: owner }; + yield { + type: "turn.completed", + usage: { + input_tokens: 100_000, + cached_input_tokens: 0, + output_tokens: 100_000, + }, + }; + })(), + }; + }, + }; + }, }; }, }, @@ -435,16 +474,26 @@ async function runInstalledSdk(pluginRoot, executable) { subagents: 0, maxDiscoveryRuns: 2, stopAfterNoNew: 1, + postScanPrompt, outputDir: join(f.directory, "output"), onScanRegistered(scan) { scanId = scan.scanId; }, }); + assert.equal(threadCount, 1); + assert.equal(resumedThreads, 1); + assert.deepEqual(prompts, [postScanPrompt]); assert.equal(result.threadId, owner); assert.equal(result.manifest.scan.status, "completed"); assert.ok(result.manifest.scan.sealedAt); assert.equal(result.manifest.scan.id, scanId); assert.deepEqual(result.findings.findings, []); + assert.equal( + await readFile(result.manifestPath, "utf8"), + manifestBeforeFollowUp, + ); + assert.ok(result.cost === null || result.cost.inputTokens < 100_000); + assert.equal(result.toJSON().threadId, owner); assert.ok( (await readFile(join(f.directory, "output", "report.md"), "utf8")) .length > 0, @@ -452,9 +501,79 @@ async function runInstalledSdk(pluginRoot, executable) { } finally { await client.close(); } + await assertSavedState(f, scanId, owner); await assertExecutions(f, scanId, 5); } +async function assertSavedState(f, scanId, owner) { + const { deepScan } = await workbench(f, [ + "get-deep-scan", + "--scan-id", + scanId, + "--thread-id", + owner, + ]); + assert.equal(deepScan.status, "succeeded"); + if (deepScan.workflowVersion === "deep-security-scan/v2") { + const selected = deepScan.finalizationInput; + assert.ok( + selected, + "The completed v2 scan retains its finalization input.", + ); + assert.equal(selected.version, 1); + assert.equal(selected.terminalReason, deepScan.terminalReason); + assert.deepEqual(selected.omittedWorkerIds, []); + await assertDigest( + resolve(deepScan.scanDir, selected.resultPath), + selected.resultSha256, + ); + const workers = deepScan.workers.filter( + (worker) => worker.status === "succeeded", + ); + assert.equal(workers.length, 3); + for (const worker of workers) { + const attempt = deepScan.attempts.find( + (entry) => + entry.workerId === worker.id && entry.attempt === worker.attempt, + ); + assert.ok(attempt, "Each accepted worker retains its execution attempt."); + assert.equal(attempt.status, "succeeded"); + await assertDigest( + attempt.acceptedResultPath, + attempt.acceptedResultSha256, + ); + if (worker.kind === "dedup") { + assert.equal(selected.resultSha256, attempt.acceptedResultSha256); + } else { + const input = deepScan.dedupInputs.find( + (entry) => entry.discoveryWorkerId === worker.id, + ); + assert.ok(input, "The reducer retains each accepted discovery input."); + assert.equal(input.attempt, worker.attempt); + assert.equal(input.resultManifestSha256, attempt.acceptedResultSha256); + await assertDigest( + input.resultManifestPath, + input.resultManifestSha256, + ); + } + } + assert.equal(deepScan.dedupInputs.length, 2); + } + console.log( + JSON.stringify({ + fixture: basename(f.directory), + workflowVersion: deepScan.workflowVersion, + attempts: deepScan.attempts?.length ?? null, + selectedFinalization: deepScan.finalizationInput != null, + }), + ); +} + +async function assertDigest(path, expected) { + const bytes = await readFile(path); + assert.equal(createHash("sha256").update(bytes).digest("hex"), expected); +} + async function assertDraft(path) { const document = JSON.parse(await readFile(path, "utf8")); const findings = JSON.parse( diff --git a/sdk/typescript/scripts/package-dist-files.mjs b/sdk/typescript/scripts/package-dist-files.mjs index 5f73d814c4..2ebbced254 100644 --- a/sdk/typescript/scripts/package-dist-files.mjs +++ b/sdk/typescript/scripts/package-dist-files.mjs @@ -1,5 +1,10 @@ const distModules = [ "api", + "reasoning-summary", + "preflight-config", + "deep-scan-finalization", + "codex-session", + "accepted-audit", "scan-executor", "host-protocol", "hosted-remediation", @@ -75,6 +80,7 @@ const distModules = [ "scan-activity", "scan-comparison", "scan-dashboard", + "scan-files", "scan-history-renderer", "scan-inputs", "scan-logs", @@ -84,6 +90,7 @@ const distModules = [ "validation-records", "suggest-owners", "scan-sessions", + "selected-scan-recovery", "server/api", "deduplication/codex-review", "deduplication/checkpointed-review", diff --git a/sdk/typescript/src/accepted-audit.ts b/sdk/typescript/src/accepted-audit.ts new file mode 100644 index 0000000000..ee526a955f --- /dev/null +++ b/sdk/typescript/src/accepted-audit.ts @@ -0,0 +1,49 @@ +export interface ScanDraftInput { + scanId: string; + complete?: boolean; + handoffClaimToken?: string; + scope?: Record; + threatModel?: Record; + findings: Record[]; + coverage: Record; +} + +/** Accepted evidence may still describe partial or unknown source coverage. */ +export interface AuditEvidence { + checkpoint?: ScanDraftInput; + accepted?: ScanDraftInput; +} + +export type AuditOutcome = AuditEvidence & + ( + | { status: "accepted"; execution: Execution; accepted: ScanDraftInput } + | { status: "checkpoint"; execution: Execution } + ); + +/** One attempt; enclosing callers own retries and public completion. */ +export async function runAcceptedAudit(input: { + signal: AbortSignal; + execute: () => Promise; + accept: (execution: Execution) => Promise; +}): Promise> { + input.signal.throwIfAborted(); + const execution = await input.execute(); + input.signal.throwIfAborted(); + const evidence = await input.accept(execution); + input.signal.throwIfAborted(); + return evidence.accepted === undefined + ? { ...evidence, execution, status: "checkpoint" } + : { + ...evidence, + execution, + status: "accepted", + accepted: evidence.accepted, + }; +} + +/** Process completion alone does not accept an unfinished audit checkpoint. */ +export function auditEvidence(checkpoint: ScanDraftInput): AuditEvidence { + return checkpoint.complete === false + ? { checkpoint } + : { checkpoint, accepted: checkpoint }; +} diff --git a/sdk/typescript/src/api.ts b/sdk/typescript/src/api.ts index bd5c74ea70..06f8a4480e 100644 --- a/sdk/typescript/src/api.ts +++ b/sdk/typescript/src/api.ts @@ -1,7 +1,16 @@ /// import { workbenchArgs } from "./workbench-arguments.js"; -import { isSafeNonNegativeInteger as safeInteger } from "./value.js"; +import { scanPreflightCodexConfig } from "./preflight-config.js"; +import { captureOriginalReasoningSummary } from "./reasoning-summary.js"; +export { scanPreflightCodexConfig } from "./preflight-config.js"; +import { resumeSelectedDeepScan } from "./deep-scan-finalization.js"; +import { recoverSelectedScanStartup } from "./selected-scan-recovery.js"; +import { + auditEvidence, + runAcceptedAudit, + type ScanDraftInput, +} from "./accepted-audit.js"; import { ScanExecutionError, ScanExecutionResultSchema, @@ -35,7 +44,6 @@ import { win32, } from "node:path"; import { - Codex, type CodexOptions, type ModelReasoningEffort, type ThreadOptions, @@ -53,6 +61,12 @@ import { type ProviderProfile, } from "./provider-profile.js"; +import { + createCodexClient, + type CodexSessionClient as CodexClientLike, + type CodexSessionThread as CodexThreadLike, + type CodexSessionEvent as ScanEvent, +} from "./codex-session.js"; import { CODEX_AUTH_CONFIG_KEYS, NO_CREDENTIALS_MESSAGE, @@ -96,6 +110,10 @@ import { type ScanWorkerEvent, } from "./cost.js"; import type { ScanTokenUsage } from "./cost-model.js"; +import { + cachedScanAttributionReader, + type ScanExecutionAttribution, +} from "./scan-sessions.js"; import { ScanProgressTracker, type DeepScanProgress } from "./scan-progress.js"; import { findScanSession } from "./scan-logs.js"; import { DIRECT_DEEP_SCAN_CONTROL_USAGE, runDeepScan } from "./deep-scan.js"; @@ -144,6 +162,7 @@ import { CodexSecurityError, ConfigurationError, IncompleteScanError, + InvalidTargetError, OutputDirectoryError, OutputDirectoryNotEmptyError, errorMessage, @@ -265,24 +284,6 @@ import { type InspectedExecutable, } from "./trusted-executable.js"; -interface CodexThreadLike { - readonly id: string | null; - runStreamed( - input: string, - options: TurnOptions, - ): Promise<{ events: AsyncGenerator }>; -} - -interface ScanEvent { - readonly type: string; - readonly [key: string]: unknown; -} - -interface CodexClientLike { - startThread(options: ThreadOptions): CodexThreadLike; - resumeThread?(threadId: string, options: ThreadOptions): CodexThreadLike; -} - interface PreparedRuntime { codexHome: string; bootstrapWorkspace?: string; @@ -648,7 +649,7 @@ interface ClientDependencies { export const DEFAULT_DEPENDENCIES: ClientDependencies = { createCodex: ({ nativeProfile, ...options }) => nativeProfile === undefined - ? new Codex(options) + ? createCodexClient(options) : createProfileCodex(options, nativeProfile), environment: process.env, }; @@ -1438,12 +1439,7 @@ export class CodexSecurity { `permissions.${POLICY_PERMISSION_PROFILE}.filesystem=${inlineToml(policyFilesystem)}`, ], ); - const reportCost = ( - current: Readonly, - complete = true, - ): void => { - const total = addScanCosts(accumulatedCost, current); - if (completeCost && complete) notifyObserver(options, "onCost")(total); + const enforceCostLimit = (total: Readonly): void => { if ( options.maxCostUsd !== undefined && total.estimatedUsd > options.maxCostUsd @@ -1455,6 +1451,14 @@ export class CodexSecurity { ); } }; + const reportCost = ( + current: Readonly, + complete = true, + ): void => { + const total = addScanCosts(accumulatedCost, current); + if (completeCost && complete) notifyObserver(options, "onCost")(total); + enforceCostLimit(total); + }; const outputSchema = securityPolicyStageOutputSchema(); const run = async ( stage: SecurityPolicyStage, @@ -1478,6 +1482,10 @@ export class CodexSecurity { options.onCost === undefined && options.maxCostUsd === undefined ? undefined : reportCost, + onCostLowerBound: + options.maxCostUsd === undefined + ? undefined + : (cost) => enforceCostLimit(addScanCosts(accumulatedCost, cost)), onError: (error) => { if (options.maxCostUsd !== undefined) budgetController.abort(error); else @@ -1637,14 +1645,90 @@ export class CodexSecurity { let preparedTargetWarnings: string[] = []; let runPostScan: (() => ReturnType) | null = null; + let selectedDeepFinalization = false; + let deepFinalizationStatusUnavailable = false; + let observedScanThreadId: string | undefined; let activeScan: { id: string; options: WorkbenchCommandOptions; + mode: ScanMode; } | null = null; const prepareArtifactRestorer = this.#dependencies.prepareScanArtifactRestorer ?? prepareScanArtifactRestorer; const workbench = this.#dependencies.runWorkbench ?? runWorkbench; + let recoveryTarget: NormalizedTarget | undefined; + const recoverStartup = async ( + startupError: unknown, + ): Promise => { + if (startupError instanceof DeepScanPluginCompatibilityError) + throw startupError; + const recovered = await recoverSelectedScanStartup({ + repository, + normalizedTarget: recoveryTarget, + options, + config: this.config, + environment: this.#dependencies.environment, + signal, + startupError, + workbench, + python: this.#dependencies.resolvePluginPython ?? resolvePluginPython, + warn: notifyObserver(options, "onWarning"), + registered: (scan, command, threadId) => { + activeScan = { id: scan.scanId, options: command, mode: "deep" }; + observedScanThreadId = threadId; + selectedDeepFinalization = true; + scanDir = scan.scanDir; + notifyObserver(options, "onOutputDirReady")(scan.scanDir); + notifyObserver(options, "onScanRegistered")(scan); + }, + cost: (cost) => + notifyObserver(options, "onCost")(cost, options.maxCostUsd), + collect: async (context) => + collectResult( + { + ...context.turnResult, + finalResponse: ( + await readScanFile( + context.scanDir, + "report.md", + "report.md", + signal, + ) + ).toString("utf8"), + }, + context.threadId, + context.scanDir, + context.pluginRoot, + context.expectation, + signal, + true, + context.python, + context.expectation.repository, + ), + }); + if (recovered === null) throw startupError; + activeScan = null; + return recovered; + }; + const recoverCompletedScan = async ( + commandOptions: WorkbenchCommandOptions, + scanId: string, + error: unknown, + completionArgs: readonly string[], + ): Promise => { + const saved = await workbench(commandOptions, [ + "get-scan", + "--scan-id", + scanId, + ]).catch(() => null); + const savedScan = saved?.["scan"]; + const progress = isRecord(savedScan) ? savedScan["progress"] : null; + if (!isRecord(progress) || progress["status"] !== "complete") throw error; + // A lost response can follow a durable seal. Only the existing normal + // completion command can validate and return that committed receipt. + return workbench(commandOptions, completionArgs); + }; const terminalScanTurnStatus = () => scanTurnStatus === "completed" || scanTurnStatus === "failed" ? scanTurnStatus @@ -1695,16 +1779,26 @@ export class CodexSecurity { // Workflows reuse the prepared prompts and deep settings, but validate the // output only when starting new work; a completed workflow may already own it. - const inputs = - preparedInputs === undefined - ? await this.#prepareLocalInputs(repository, options, signal) - : { - ...preparedInputs, - outputDir: await prepareScanOutputDir( - options, - preparedInputs.protectedRoots, - ), - }; + let inputs: LocalScanInputs; + try { + inputs = + preparedInputs === undefined + ? await this.#prepareLocalInputs(repository, options, signal) + : { + ...preparedInputs, + outputDir: await prepareScanOutputDir( + options, + preparedInputs.protectedRoots, + ), + }; + } catch (error) { + if ( + options.resumeScanId !== undefined && + error instanceof InvalidTargetError + ) + return await recoverStartup(error); + throw error; + } const { repository: repo, target: normalized, @@ -1715,6 +1809,7 @@ export class CodexSecurity { deepScanConfiguration, prompts, } = inputs; + recoveryTarget = normalized; options = { ...options, ...prompts }; checkOpen(); let temporaryRoot: string | undefined; @@ -1755,23 +1850,29 @@ export class CodexSecurity { } checkOpen(); - const session = await this.#prepareSession( - { protectedRoot }, - options, - signal, - temporaryRoot, - mode === "deep", - ); + let session: PreparedSession; + try { + session = await this.#prepareSession( + { protectedRoot }, + options, + signal, + temporaryRoot, + mode === "deep", + ); + } catch (error) { + if (options.resumeScanId !== undefined) + return await recoverStartup(error); + throw error; + } const { runtime, runtimeHome, - effectiveConfig, - preflightConfig, modelProvider, authentication, approvalPolicy, python, } = session; + let { effectiveConfig, preflightConfig } = session; let git: InspectedExecutable = { executable: null, environment: selectedScanEnvironment( @@ -1833,6 +1934,49 @@ export class CodexSecurity { } checkOpen(); + if (mode === "deep" && options.resumeScanId === undefined) { + const summary = await captureOriginalReasoningSummary({ + config: session.sessionConfig, + ...(runtime.providerProfile === undefined + ? {} + : { nativeProfile: runtime.providerProfile.name }), + command: this.#codexCommand(), + cwd: scanDir, + environment: { + ...withoutOpenAiApiKeys( + this.#createSessionEnvironment(session, {}, options.auth), + ), + ...(session.externalProvider === null && session.apiKey !== null + ? { CODEX_API_KEY: session.apiKey } + : {}), + }, + signal, + }); + if (summary !== undefined) { + effectiveConfig = { + ...effectiveConfig, + model_reasoning_summary: summary, + }; + preflightConfig = scanPreflightCodexConfig(effectiveConfig); + session.effectiveConfig = effectiveConfig; + session.preflightConfig = preflightConfig; + session.sessionConfig = { + ...session.sessionConfig, + model_reasoning_summary: summary, + }; + if (runtime.configPath !== undefined) + await writeCodexConfig(runtime.configPath, { + ...preflightConfig, + ...(options.cyberAccessProgram === undefined + ? {} + : { + codex_security: { + cyber_access_program: options.cyberAccessProgram, + }, + }), + }); + } + } const workerProviderEnvironment = { ...withoutOpenAiApiKeys( selectedScanEnvironment( @@ -2021,6 +2165,14 @@ export class CodexSecurity { "onWarning", )(`Could not track scan activity: ${errorMessage(error)}`); }; + const enforceCostLimit = (cost: Readonly): boolean => { + if (maxCostUsd === undefined || cost.estimatedUsd <= maxCostUsd) + return false; + costAbortController.abort( + new ScanCostLimitExceededError(maxCostUsd, cost, scanDir), + ); + return true; + }; const tracker = new ScanCostTracker({ codexHome: runtime.codexHome, model, @@ -2060,15 +2212,7 @@ export class CodexSecurity { latestCostComplete = complete; if (complete) notifyObserver(options, "onCost")(cost, maxCostUsd); - if ( - maxCostUsd !== undefined && - cost.estimatedUsd > maxCostUsd - ) { - costAbortController.abort( - new ScanCostLimitExceededError(maxCostUsd, cost, scanDir), - ); - return; - } + if (enforceCostLimit(cost)) return; const request = options.onBudgetApproaching; if ( request === undefined || @@ -2128,6 +2272,8 @@ export class CodexSecurity { } }); }, + onCostLowerBound: + options.maxCostUsd === undefined ? undefined : enforceCostLimit, onError: reportTrackingError, }); costTracker = tracker; @@ -2222,8 +2368,10 @@ export class CodexSecurity { savedSession === null || savedSession.workingDirectory !== scanDir ) { - throw new CodexSecurityError( - `The original Codex session for scan ${scanId} is unavailable. Restore its session logs in the original Codex Security state directory before resuming.`, + return await recoverStartup( + new CodexSecurityError( + `The original Codex session for scan ${scanId} is unavailable. Restore its session logs in the original Codex Security state directory before resuming.`, + ), ); } if (typeof registration["sealedProducerVersion"] === "string") { @@ -2287,6 +2435,8 @@ export class CodexSecurity { } const targetRevision = registeredRevision === "unversioned" ? null : registeredRevision; + if (registration["selectedFinalization"] === true) + expectation.repositoryRevision = targetRevision; const registeredFileCount = registration["scopeFileCount"]; scopeFileCount = typeof registeredFileCount === "number" && @@ -2305,7 +2455,7 @@ export class CodexSecurity { filesTotal: scopeFileCount, }); } - activeScan = { id: scanId, options: workbenchOptions }; + activeScan = { id: scanId, options: workbenchOptions, mode }; notifyObserver( options, "onScanRegistered", @@ -2329,6 +2479,24 @@ export class CodexSecurity { notifyObserver(options, "onOutputDirReady")(scanDir); } throwIfAborted(signal, scanDir); + if (mode === "deep") { + tracker.setAttributionReader( + cachedScanAttributionReader(stateDirectory, async () => { + const context = await workbench( + { ...workbenchOptions, signal: undefined }, + ["get-scan", "--scan-id", scanId], + ); + const scan = context["scan"]; + if (isRecord(scan) && !("executionAttribution" in scan)) + return undefined; + return isRecord(scan) && isRecord(scan["executionAttribution"]) + ? (scan[ + "executionAttribution" + ] as unknown as ScanExecutionAttribution) + : null; + }), + ); + } if ( this.#dependencies.hostedScan === undefined && ((mode !== "deep" && options.onProgress !== undefined) || @@ -2525,6 +2693,7 @@ export class CodexSecurity { ); } thread = codex.resumeThread(resumeThreadId, threadOptions); + observedScanThreadId = resumeThreadId; tracker.start(resumeThreadId); if (budgetRecovery !== null) budgetRecovery.threadId = resumeThreadId; await tracker.refresh().catch(reportTrackingError); @@ -2565,23 +2734,93 @@ export class CodexSecurity { ), )}`; } - const events = directDeepScan - ? (this.#dependencies.runDeepScan ?? runDeepScan)({ + const recoverSelectedCompletion = async (required = false) => { + const threadId = observedScanThreadId ?? thread.id; + if (mode !== "deep" || !threadId || signal.aborted) return null; + const saved = await workbench(workbenchOptions, [ + "get-deep-scan", + "--scan-id", + scanId, + "--thread-id", + threadId, + ]).catch((error: unknown) => { + deepFinalizationStatusUnavailable = true; + if (required) throw error; + return null; + }); + if (saved !== null) deepFinalizationStatusUnavailable = false; + const deep = saved?.["deepScan"]; + if ( + !isRecord(deep) || + !isRecord(deep["finalizationInput"]) || + (deep["status"] !== "running" && deep["status"] !== "succeeded") + ) { + if (required) + throw new CodexSecurityError( + "The saved Deep Scan selection is unavailable for finalization.", + ); + return null; + } + selectedDeepFinalization = true; + await resumeSelectedDeepScan({ + scanId, + threadId, + pluginRoot: runtime.plugin.installedRoot, + signal, + runWorkbench: (args) => workbench(workbenchOptions, args), + }); + return { + status: "completed" as const, + threadId, + finalResponse: "", + usage: null, + lastStreamError: null, + }; + }; + const savedCompletion = resumeThreadId + ? await recoverSelectedCompletion( + registration["selectedFinalization"] === true, + ) + : null; + const directRunner = this.#dependencies.runDeepScan ?? runDeepScan; + const directPreflightCommand = directDeepScan + ? await this.#providerPreflightCommand(effectiveConfig) + : undefined; + const events = (async function* () { + if (savedCompletion) return; + if (directDeepScan) { + yield* directRunner({ codexOptions, - preflightCommand: - await this.#providerPreflightCommand(effectiveConfig), + preflightCommand: directPreflightCommand!, pluginRoot: runtime.plugin.pluginRoot, repository: repo, scanDir, scanId, prompt, signal, - }) - : (await thread.runStreamed(prompt, turnOptions)).events; - scanTurnStatus = "in_progress"; + }); + } else { + yield* (await thread.runStreamed(prompt, turnOptions)).events; + } + })(); + if (!savedCompletion) scanTurnStatus = "in_progress"; checkOpen(); + let preparedAcceptedFindings = false; let result = await runScanEvents({ + acceptedFindings: () => preparedAcceptedFindings, + savedCompletion: savedCompletion ?? undefined, + recoverCompletion: recoverSelectedCompletion, + reconcileCheckpoint: + mode === "deep" + ? undefined + : async () => { + await workbench(workbenchOptions, [ + "prepare-scan-completion", + "--scan-id", + scanId, + ]); + }, thread, events, signal, @@ -2599,6 +2838,7 @@ export class CodexSecurity { scanTurnStatus = status; }, onThreadStarted: async (threadId) => { + observedScanThreadId = threadId; if (resumeThreadId !== undefined && threadId !== resumeThreadId) { throw new CodexSecurityError( "Codex did not resume the original scan session.", @@ -2635,6 +2875,7 @@ export class CodexSecurity { } }, onFinalize: async (usage) => { + await recoverSelectedCompletion(); if (options.validationPrompt !== undefined) { tracker.recordUsage(usage); await tracker.refresh().catch(reportTrackingError); @@ -2700,7 +2941,10 @@ export class CodexSecurity { return { usage, cost: estimateScanCost(model, usage) }; }); throwIfAborted(signal, scanDir); - if (options.maxCostUsd !== undefined && snapshot.cost === null) { + if ( + options.maxCostUsd !== undefined && + (snapshot.cost === null || snapshot.cost.coverage === "partial") + ) { notifyObserver( options, "onWarning", @@ -2737,6 +2981,7 @@ export class CodexSecurity { } throw error; } + preparedAcceptedFindings = preparation["acceptedFindings"] === true; preparedTargetWarnings = Array.isArray(preparation["targetWarnings"]) ? preparation["targetWarnings"].filter( (warning): warning is string => typeof warning === "string", @@ -2765,14 +3010,20 @@ export class CodexSecurity { onObserverError: options.onObserverError, }); checkOpen(); - const completion = await workbench(workbenchOptions, [ + const completionArgs = [ "complete-scan", "--scan-id", scanId, ...(completionCost === null ? [] : ["--cost-json", JSON.stringify(completionCost)]), - ]); + ]; + const completion = await workbench( + workbenchOptions, + completionArgs, + ).catch((error) => + recoverCompletedScan(workbenchOptions, scanId, error, completionArgs), + ); activeScan = null; const completedScan = completion["scan"]; await finalizeScanUsage( @@ -2842,10 +3093,27 @@ export class CodexSecurity { let artifactRestorer: ScanArtifactRestorer | null = null; try { artifactRestorer = await prepareArtifactRestorer( - workbenchOptions, + { ...workbenchOptions, signal: undefined }, scanDir, ); + const savedDocument = (name: string) => + JSON.parse( + completedArtifacts + .find((artifact) => artifact.name === name)! + .contents.toString("utf8"), + ); + const acceptedDocuments = preparedAcceptedFindings + ? { + manifest: savedDocument("scan-manifest.json"), + findings: savedDocument("findings.json"), + coverage: savedDocument("coverage.json"), + } + : undefined; const followUpResult = await runScanEvents({ + // A new model edit cannot inherit the saved publication's authority. + acceptedFindings: (documents) => + preparedAcceptedFindings && + isDeepStrictEqual(documents, acceptedDocuments), thread, events: (await followUp()).events, signal, @@ -3031,16 +3299,56 @@ export class CodexSecurity { options.signal?.aborted !== true ) { try { - const completion = await workbench( - { ...activeScan.options, signal: undefined }, - [ - "complete-budget-exhausted-scan", + const budgetScanId = activeScan.id; + const budgetCost = snapshot?.cost ?? { lowerBound: failure.cost }; + const completionSignal = AbortSignal.any([ + this.#abortController.signal, + ...(options.signal === undefined ? [] : [options.signal]), + ]); + const completionOptions = { + ...activeScan.options, + signal: completionSignal, + }; + const saved = await workbench(completionOptions, [ + "get-deep-scan", + "--scan-id", + activeScan.id, + "--thread-id", + budgetRecovery.threadId, + ]).catch(() => null); + const deep = saved?.["deepScan"]; + if ( + isRecord(deep) && + deep["status"] === "running" && + isRecord(deep["finalizationInput"]) + ) { + // Cost stops model work, but an already selected result can still + // finish through the local publisher. Caller cancellation remains live. + selectedDeepFinalization = true; + await resumeSelectedDeepScan({ + scanId: activeScan.id, + threadId: budgetRecovery.threadId, + pluginRoot: budgetRecovery.pluginRoot, + signal: completionSignal, + runWorkbench: (args) => workbench(completionOptions, args), + }); + } + const completion = await workbench(completionOptions, [ + "complete-budget-exhausted-scan", + "--scan-id", + budgetScanId, + "--cost-json", + JSON.stringify(budgetCost), + `--message=${failure.message.slice(0, 2400)}`, + ]).catch((error) => + recoverCompletedScan(completionOptions, budgetScanId, error, [ + "complete-scan", "--scan-id", - activeScan.id, - "--cost-json", - JSON.stringify(snapshot?.cost ?? failure.cost), - `--message=${failure.message.slice(0, 2400)}`, - ], + budgetScanId, + ...(snapshot?.cost + ? ["--cost-json", JSON.stringify(snapshot.cost)] + : []), + ]), ); if (terminalTurnStatus !== null) await finalizeScanUsage( @@ -3061,10 +3369,7 @@ export class CodexSecurity { scanDir, budgetRecovery.pluginRoot, budgetRecovery.expectation, - AbortSignal.any([ - this.#abortController.signal, - ...(options.signal === undefined ? [] : [options.signal]), - ]), + completionSignal, true, budgetRecovery.pythonPath, budgetRecovery.protectedRoot, @@ -3127,9 +3432,34 @@ export class CodexSecurity { ); } } - // A failed attachment must not turn a resumable coordinator into a terminal failure. - // Deep Scan orchestration persists its own terminal failures and cancellations. - if (activeScan !== null && options.resumeScanId === undefined) { + const callerCanceledDeepScan = + activeScan?.mode === "deep" && options.signal?.aborted; + if (activeScan !== null && callerCanceledDeepScan) { + const workbenchOptions = { ...activeScan.options, signal: undefined }; + // The workbench owns the running-state check and repeated cancellation. + // Selection may have committed before the SDK received its response. + await workbench(workbenchOptions, [ + "cancel-scan", + "--scan-id", + activeScan.id, + ...(observedScanThreadId === undefined + ? [] + : ["--thread-id", observedScanThreadId]), + ]).catch(() => undefined); + } + // Publication failures remain resumable. A cost stop or explicit client close + // still uses the existing failure path to retain partial results and stop work. + if ( + activeScan !== null && + !callerCanceledDeepScan && + ((options.resumeScanId === undefined && + !selectedDeepFinalization && + !deepFinalizationStatusUnavailable) || + ((selectedDeepFinalization || deepFinalizationStatusUnavailable) && + !options.signal?.aborted && + (failure instanceof ScanCostLimitExceededError || + this.#abortController.signal.aborted))) + ) { if ( options.validationPrompt !== undefined && !customValidationComplete @@ -3398,6 +3728,53 @@ export class CodexSecurity { } } + #createSessionEnvironment( + session: PreparedSession, + runtimePaths: Record, + auth: ScanAuthMode = "auto", + git?: InspectedExecutable, + ): ProcessEnvironment { + const { + runtime, + python, + modelProvider, + externalProvider, + apiKey, + sessionConfig, + } = session; + const commandAuth = hasCommandAuth(sessionConfig); + const environment: ProcessEnvironment = { + ...environmentWithGit( + pluginExecutionEnvironment( + python, + withoutCodexHome( + selectedScanEnvironment( + commandAuth + ? withoutOpenAiApiKeys(runtime.environment) + : runtime.environment, + auth, + modelProvider, + ), + ), + ), + git, + ), + ...(externalProvider === null + ? {} + : { [externalProvider.env_key]: apiKey! }), + CODEX_HOME: runtime.codexHome, + ...runtimePaths, + }; + for (const name of Object.keys(environment)) { + if (name.toUpperCase() === SAFETY_IDENTIFIER_ENV) + delete environment[name]; + } + if (session.safetyIdentifier !== undefined) { + environment[SAFETY_IDENTIFIER_ENV] = session.safetyIdentifier; + } + return environment; + } + async #createSessionCodex( session: PreparedSession, { command, mode }: { command: string; mode?: ScanMode }, @@ -3495,45 +3872,14 @@ export class CodexSecurity { }, }; } - const { - runtime, - runtimeHome, - python, - modelProvider, - externalProvider, - apiKey, - sessionConfig, - } = session; - const commandAuth = hasCommandAuth(sessionConfig); - const environment: ProcessEnvironment = { - ...environmentWithGit( - pluginExecutionEnvironment( - python, - withoutCodexHome( - selectedScanEnvironment( - commandAuth - ? withoutOpenAiApiKeys(runtime.environment) - : runtime.environment, - auth, - modelProvider, - ), - ), - ), - git, - ), - ...(externalProvider === null - ? {} - : { [externalProvider.env_key]: apiKey! }), - CODEX_HOME: runtime.codexHome, - ...runtimePaths, - }; - for (const name of Object.keys(environment)) { - if (name.toUpperCase() === SAFETY_IDENTIFIER_ENV) - delete environment[name]; - } - if (session.safetyIdentifier !== undefined) { - environment[SAFETY_IDENTIFIER_ENV] = session.safetyIdentifier; - } + const { runtime, runtimeHome, externalProvider, apiKey, sessionConfig } = + session; + const environment = this.#createSessionEnvironment( + session, + runtimePaths, + auth, + git, + ); const sdkCodexConfig = structuredCodexConfig(config ?? sessionConfig); // This snapshot outlives a Deep Scan when the client reuses its runtime. if (runtime.deepScanConfigPath !== undefined) { @@ -4541,9 +4887,22 @@ async function removeTargetPathsFile(path: string | null): Promise { } } +interface CanonicalScanDraftDocuments { + manifest: JsonObject; + findings: JsonObject; + coverage: JsonObject; +} + interface ScanEventRunOptions extends Pick { scanId?: string; - thread: Pick; + savedCompletion?: Awaited>; + recoverCompletion?: () => Promise + > | null>; + reconcileCheckpoint?: () => Promise; + acceptedFindings?: (documents: CanonicalScanDraftDocuments) => boolean; + thread: Pick & + Partial>; events: AsyncGenerator; signal: AbortSignal; scanDir: string; @@ -4575,123 +4934,174 @@ export async function runScanEvents( let tacStatusReported = false; try { options.signal.throwIfAborted(); - const turn = await readCodexTurn({ - thread: options.thread, - events: options.events, - onEvent: async (event) => { - if (event.type === "turn.completed") - options.onTurnStatus?.("completed"); - if (event.type === "turn.failed") options.onTurnStatus?.("failed"); - if ( - !tacStatusReported && - options.modelProvider !== "amazon-bedrock" && - options.authentication?.method !== "aws_credentials" - ) { - const tacStatus = trustedAccessStatusFromEvent(event); - if (tacStatus !== null) { - tacStatusReported = true; - notifyObserver(options, "onTrustedAccessStatus")(tacStatus); - if (tacStatus !== "granted") { - notifyObserver( - options, - "onWarning", - )(trustedAccessWarning(tacStatus, options.authentication)); + let completedTurn: + | (Awaited> & { + threadId: string; + status: "completed"; + }) + | undefined; + const execute = async () => { + const turn = + options.savedCompletion ?? + (await readCodexTurn({ + thread: options.thread, + events: options.events, + onEvent: async (event) => { + if (event.type === "turn.completed") + options.onTurnStatus?.("completed"); + if (event.type === "turn.failed") options.onTurnStatus?.("failed"); + if ( + !tacStatusReported && + options.modelProvider !== "amazon-bedrock" && + options.authentication?.method !== "aws_credentials" + ) { + const tacStatus = trustedAccessStatusFromEvent(event); + if (tacStatus !== null) { + tacStatusReported = true; + notifyObserver(options, "onTrustedAccessStatus")(tacStatus); + if (tacStatus !== "granted") { + notifyObserver( + options, + "onWarning", + )(trustedAccessWarning(tacStatus, options.authentication)); + } + } } - } - } - for (const activity of scanActivitiesFromEvent( - event, - options.expectation.repository, - )) { - notifyObserver(options, "onActivity")(activity); - } - for (const progress of scanProgressUpdatesFromEvent(event)) { - notifyObserver(options, "onProgress")(progress); - } - const workerStatus = workerStatusFromEvent(event); - if (workerStatus !== null) { - notifyObserver(options, "onWorkerStatus")(workerStatus); - } - if (event.type === "thread.started") { - const startedThreadId = event["thread_id"]; - if (typeof startedThreadId === "string") { - await options.onThreadStarted?.(startedThreadId); - } - if (!scanStarted) { - scanStarted = true; - notifyObserver(options, "onScanStarted")(); - } - } - }, - onReconnect: (message, reconnect) => { - notifyObserver(options, "onReconnect")( - ...reconnect, - reconnectDetails(message), + for (const activity of scanActivitiesFromEvent( + event, + options.expectation.repository, + )) { + notifyObserver(options, "onActivity")(activity); + } + for (const progress of scanProgressUpdatesFromEvent(event)) { + notifyObserver(options, "onProgress")(progress); + } + const workerStatus = workerStatusFromEvent(event); + if (workerStatus !== null) { + notifyObserver(options, "onWorkerStatus")(workerStatus); + } + if (event.type === "thread.started") { + const startedThreadId = event["thread_id"]; + if (typeof startedThreadId === "string") { + await options.onThreadStarted?.(startedThreadId); + } + if (!scanStarted) { + scanStarted = true; + notifyObserver(options, "onScanStarted")(); + } + } + }, + onReconnect: (message, reconnect) => { + notifyObserver(options, "onReconnect")( + ...reconnect, + reconnectDetails(message), + ); + }, + })); + const { status, threadId, lastStreamError } = turn; + if (status !== "completed") { + throw new IncompleteScanError( + lastStreamError ?? + "Codex Security event stream ended before the turn completed.", ); - }, - }); - const { status, threadId, finalResponse, lastStreamError } = turn; - let { usage } = turn; - if (options.signal.aborted) { - throw new ScanInterruptedError( - `Codex Security scan was interrupted; partial output remains at ${options.scanDir}.`, - options.scanDir, - ); - } - if (status !== "completed") { - throw new IncompleteScanError( - lastStreamError ?? - "Codex Security event stream ended before the turn completed.", - ); - } - if (threadId === null) { - throw new IncompleteScanError( - "Codex Security did not report a thread ID.", + } + if (threadId === null) { + throw new IncompleteScanError( + "Codex Security did not report a thread ID.", + ); + } + return (completedTurn = { ...turn, threadId, status }); + }; + let managedFinalized = false; + const accept = async () => { + // Managed completion reconciles saved checkpoints before canonical admission. + if ( + options.workbenchValidated && + options.onFinalize !== undefined && + completedTurn !== undefined && + !managedFinalized + ) { + const finalizedUsage = await options.onFinalize(completedTurn.usage); + if (finalizedUsage !== undefined) completedTurn.usage = finalizedUsage; + managedFinalized = true; + } + options.signal.throwIfAborted(); + const readDocuments = async () => { + await requireScanArtifacts( + options.scanDir, + ["scan-manifest.json", "findings.json", "coverage.json"], + options.signal, + ); + return Promise.all( + ["scan-manifest.json", "findings.json", "coverage.json"].map( + async (name) => + JSON.parse( + new TextDecoder("utf-8", { fatal: true }).decode( + await readScanFile( + options.scanDir, + name, + name, + options.signal, + ), + ), + ), + ), + ); + }; + let [manifest, findings, coverage] = await readDocuments(); + if (manifest?.scan?.complete === false && options.reconcileCheckpoint) { + await options.reconcileCheckpoint(); + [manifest, findings, coverage] = await readDocuments(); + } + const helper = ( + await import( + pathToFileURL(join(await bundledPluginRoot(), "mcp/helpers.mjs")).href + ) + ).default; + const documents = { manifest, findings, coverage }; + const draft: ScanDraftInput = helper.parseCanonicalScanDraft( + { scanId: options.scanId ?? manifest.scan.id, ...documents }, + options.acceptedFindings?.(documents) === true, ); + return auditEvidence(draft); + }; + let audit; + try { + audit = await runAcceptedAudit({ + signal: options.signal, + execute, + accept, + }); + } catch (error) { + if (options.signal.aborted) throw error; + const saved = await options.recoverCompletion?.(); + if (!saved || saved.status !== "completed" || saved.threadId === null) + throw error; + // The enclosing scan publishes its saved selection; acceptance then reads it. + const recovered = completedTurn ?? { + ...saved, + threadId: saved.threadId, + status: "completed" as const, + }; + completedTurn = recovered; + audit = await runAcceptedAudit({ + signal: options.signal, + execute: async () => recovered, + accept, + }); } - // Managed scans reconcile saved checkpoints before admitting canonical artifacts. - if (options.workbenchValidated && options.onFinalize !== undefined) { - const finalizedUsage = await options.onFinalize(usage); - if (finalizedUsage !== undefined) usage = finalizedUsage; - } - options.signal.throwIfAborted(); - await requireScanArtifacts( - options.scanDir, - ["scan-manifest.json", "findings.json", "coverage.json"], - options.signal, - ); - const [manifest, findings, coverage] = await Promise.all( - ["scan-manifest.json", "findings.json", "coverage.json"].map( - async (name) => - JSON.parse( - new TextDecoder("utf-8", { fatal: true }).decode( - await readScanFile(options.scanDir, name, name, options.signal), - ), - ), - ), - ); - const helper = ( - await import( - pathToFileURL(join(await bundledPluginRoot(), "mcp/helpers.mjs")).href - ) - ).default; - const draft = helper.parseCanonicalScanDraft({ - scanId: options.scanId ?? manifest.scan.id, - manifest, - findings, - coverage, - }); - options.signal.throwIfAborted(); - if (draft.complete === false) { + if (audit.status === "checkpoint") throw new IncompleteScanError( "Codex Security produced only an unfinished audit checkpoint.", ); - } + const { status, threadId, finalResponse } = audit.execution; + let { usage } = audit.execution; if (!options.workbenchValidated && options.onFinalize !== undefined) { const finalizedUsage = await options.onFinalize(usage); if (finalizedUsage !== undefined) usage = finalizedUsage; } - const result = await collectResult( + options.signal.throwIfAborted(); + return await collectResult( { status, finalResponse, @@ -4707,18 +5117,15 @@ export async function runScanEvents( options.pythonPath, options.protectedRoot, ); - if (options.signal.aborted) { - throw new ScanInterruptedError( - `Codex Security scan was interrupted; partial output remains at ${options.scanDir}.`, - options.scanDir, - ); - } - return result; } catch (error) { if (options.signal.reason instanceof ScanCostLimitExceededError) { throw options.signal.reason; } - if (options.signal.aborted && !(error instanceof ScanInterruptedError)) { + if ( + options.signal.aborted && + !(error instanceof ScanInterruptedError) && + !(error instanceof ScanExecutionError) + ) { throw new ScanInterruptedError( `Codex Security scan was interrupted; partial output remains at ${options.scanDir}.`, options.scanDir, @@ -4878,7 +5285,7 @@ function scanPrompt( : []), ...(mode === "deep" ? [ - `The SDK has already registered this scan. Call start_codex_security_deep_scan with ${JSON.stringify({ scanId })}; never pass targetPath or create another scan.`, + `The SDK has already registered this scan. Call start_codex_security_deep_scan with ${JSON.stringify({ scanId })}; never pass targetPath or create another scan. Leave finalization to the SDK after this turn; do not call a completion tool.`, ] : skillName === "security-scan" || customValidation ? [ @@ -4934,7 +5341,7 @@ function scanPrompt( ] : skillName === "deep-security-scan" ? [ - "The Deep Scan coordinator already wrote the canonical scan artifacts. Call complete_codex_security_scan exactly once without submitting another semantic draft; the workbench owns authoritative metadata, finalization, report generation, and sealing.", + "The Deep Scan coordinator already wrote the canonical scan artifacts. Leave them unchanged and end the scan turn without calling a completion tool; the SDK accounts for the completed turn, enforces its budget, and owns finalization, report generation, and sealing.", ] : [ "Use record_codex_security_scan_draft and complete_codex_security_scan as directed by the selected skill; the workbench owns authoritative metadata, finalization, report generation, and sealing.", @@ -5065,6 +5472,19 @@ function addScanCosts( previous.cacheWriteInputTokens + current.cacheWriteInputTokens, outputTokens: previous.outputTokens + current.outputTokens, estimatedUsd: previous.estimatedUsd + current.estimatedUsd, + ...(previous.coverage === "partial" || current.coverage === "partial" + ? { coverage: "partial" as const } + : {}), + ...(previous.modelCosts || + current.modelCosts || + previous.model !== current.model + ? { + modelCosts: [ + ...(previous.modelCosts ?? [previous]), + ...(current.modelCosts ?? [current]), + ], + } + : {}), ...(previous.cacheWriteInputTokensReported === false || current.cacheWriteInputTokensReported === false ? { cacheWriteInputTokensReported: false } @@ -5579,173 +5999,7 @@ function sharedCredentialCodexConfig( return scanRuntimeCodexConfig(shared, credentialHome); } -export function scanPreflightCodexConfig(config: JsonObject): JsonObject { - const safeString = (value: unknown): value is string => - typeof value === "string" && - value.length > 0 && - !/[\u0000-\u001f\u007f]/u.test(value); - const safeProfileName = (value: unknown): value is string => - safeString(value) && /^[A-Za-z0-9_-]+$/u.test(value); - const capabilityFeatures = (value: unknown): JsonObject => { - if (!isRecord(value)) return {}; - const result: JsonObject = {}; - for (const key of [ - "goals", - "multi_agent", - "enable_fanout", - "cli_daybreak", - "api_key_cyber_access_programs", - "api_key_model_discovery", - ]) { - if (typeof value[key] === "boolean") result[key] = value[key]; - } - const multiAgent = value["multi_agent_v2"]; - if (typeof multiAgent === "boolean") { - result["multi_agent_v2"] = multiAgent; - } else if (isRecord(multiAgent)) { - const sanitized: JsonObject = {}; - if (typeof multiAgent["enabled"] === "boolean") { - sanitized["enabled"] = multiAgent["enabled"]; - } - const capacity = multiAgent["max_concurrent_threads_per_session"]; - if (safeInteger(capacity)) { - sanitized["max_concurrent_threads_per_session"] = capacity; - } - if (Object.keys(sanitized).length > 0) { - result["multi_agent_v2"] = sanitized; - } - } - return result; - }; - const executionConfig = (source: JsonObject): JsonObject => { - const result: JsonObject = {}; - if (typeof source["daybreak"] === "boolean") { - result["daybreak"] = source["daybreak"]; - } - for (const key of [ - "model", - "model_reasoning_effort", - "model_reasoning_summary", - "model_provider", - "service_tier", - ]) { - const value = source[key]; - if (safeString(value)) result[key] = value; - } - const features = capabilityFeatures(source["features"]); - if (Object.keys(features).length > 0) result["features"] = features; - const skills = source["skills"]; - if (isRecord(skills) && Array.isArray(skills["config"])) { - result["skills"] = { - config: skills["config"].flatMap((entry) => - isRecord(entry) && - safeString(entry["path"]) && - typeof entry["enabled"] === "boolean" - ? [{ path: entry["path"], enabled: entry["enabled"] }] - : [], - ), - }; - } - const agents = source["agents"]; - if (isRecord(agents)) { - const sanitized: JsonObject = {}; - for (const key of ["max_threads", "max_depth"]) { - const value = agents[key]; - if (safeInteger(value)) sanitized[key] = value; - } - if (Object.keys(sanitized).length > 0) result["agents"] = sanitized; - } - const multiagent = source["multiagent_config"]; - if (isRecord(multiagent) && safeInteger(multiagent["max_concurrency"])) { - result["multiagent_config"] = { - max_concurrency: multiagent["max_concurrency"], - }; - } - return result; - }; - const resolved = resolveCodexProfile(config); - const result = executionConfig( - safeProfileName(config["profile"]) ? config : resolved, - ); - // Keep effective execution settings even when preflight filters the profile name. - for (const key of ["model_reasoning_summary", "service_tier"]) { - const value = resolved[key]; - if (safeString(value)) result[key] = value; - } - if (typeof resolved["daybreak"] === "boolean") { - result["daybreak"] = resolved["daybreak"]; - } - const resolvedFeatures = capabilityFeatures(resolved["features"]); - for (const key of [ - "cli_daybreak", - "api_key_cyber_access_programs", - "api_key_model_discovery", - ]) { - if (resolvedFeatures[key] !== undefined) { - result["features"] = { - ...(isRecord(result["features"]) ? result["features"] : {}), - [key]: resolvedFeatures[key], - }; - } - } - const selectedProfile = safeProfileName(config["profile"]) - ? config["profile"] - : undefined; - if (selectedProfile !== undefined) { - result["profile"] = selectedProfile; - } - const profiles = config["profiles"]; - if (isRecord(profiles)) { - const sanitized: JsonObject = {}; - for (const [name, profile] of Object.entries(profiles)) { - if (!safeProfileName(name) || !isRecord(profile)) continue; - const projected = executionConfig(profile as JsonObject); - if (Object.keys(projected).length === 0) continue; - sanitized[name] = projected; - } - if (Object.keys(sanitized).length > 0) result["profiles"] = sanitized; - } - const modelProvider = scanModelProvider(result); - if (isExternalModelProvider(modelProvider)) { - result["model_providers"] = { - [modelProvider]: { ...EXTERNAL_CODEX_PROVIDERS[modelProvider] }, - }; - } else if (modelProvider === "amazon-bedrock") { - const providers = resolved["model_providers"]; - const provider = isRecord(providers) ? providers[modelProvider] : undefined; - const aws = isRecord(provider) ? provider["aws"] : undefined; - if (isRecord(aws)) { - const sanitized: JsonObject = {}; - for (const key of ["region", "profile"]) { - const value = aws[key]; - if (safeString(value)) sanitized[key] = value; - } - if (Object.keys(sanitized).length > 0) { - result["model_providers"] = { - [modelProvider]: { aws: sanitized }, - }; - } - } - } - const rootMarkers = config["project_root_markers"]; - if (Array.isArray(rootMarkers)) { - result["project_root_markers"] = rootMarkers.filter(safeString); - } - const projects = config["projects"]; - if (isRecord(projects)) { - const sanitized: JsonObject = {}; - for (const [path, project] of Object.entries(projects)) { - if (!safeString(path) || !isAbsolute(path) || !isRecord(project)) { - continue; - } - const trust = project["trust_level"]; - if (trust !== "trusted" && trust !== "untrusted") continue; - sanitized[path] = { trust_level: trust }; - } - if (Object.keys(sanitized).length > 0) result["projects"] = sanitized; - } - return result; -} +class DeepScanPluginCompatibilityError extends CodexSecurityError {} async function requireCurrentDeepScanPlugin( runtime: PreparedRuntime, @@ -5765,7 +6019,7 @@ async function requireCurrentDeepScanPlugin( capabilities["directDeepScanEngine"] !== 1 || capabilities["workerProviderSnapshot"] !== 4 ) { - throw new CodexSecurityError( + throw new DeepScanPluginCompatibilityError( "Deep Scan requires the current plugin's direct engine and isolated worker runtime support. Update the custom plugin or use the bundled plugin.", ); } diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts index 307254f140..ee29210eb4 100644 --- a/sdk/typescript/src/cli.ts +++ b/sdk/typescript/src/cli.ts @@ -2479,6 +2479,7 @@ export async function main( output: scanOutputSchema("SCAN_FAILED", "SCAN_RESUME_UNAVAILABLE"), async run({ args, error: incurError, format, options }) { let scanArguments: ScanArguments; + let selectedFinalization = false; try { const saved = await dependencies.runWorkbench( workbenchArgs("get-cli-scan-resume", { scanId: args.scanId }), @@ -2491,6 +2492,7 @@ export async function main( "The workbench returned invalid scan resume context.", ); } + selectedFinalization = saved["selectedFinalization"] === true; scanArguments = await prepareScanArgumentsFromRecipe( saved["recipe"], saved["scanId"], @@ -2501,6 +2503,7 @@ export async function main( : undefined, }, dependencies.currentDirectory(), + selectedFinalization, ); scanArguments.resumeScanId = saved["scanId"]; scanArguments.outputDir = resolveCliPath( @@ -2508,11 +2511,27 @@ export async function main( saved["scanDir"], ); if (scanArguments.knowledgeBasePaths.length > 0) { - scanArguments.knowledgeBaseSnapshot = await restoreScanKnowledge( - scanArguments.outputDir, - scanArguments.repository!, - (saved["recipe"] as JsonObject)["scanInputs"], - ); + try { + scanArguments.knowledgeBaseSnapshot = await restoreScanKnowledge( + scanArguments.outputDir, + scanArguments.repository!, + (saved["recipe"] as JsonObject)["scanInputs"], + ); + } catch (error) { + const cause = error instanceof Error ? error.cause : undefined; + // A missing original checkout can use selected artifact recovery. + if ( + !selectedFinalization || + typeof cause !== "object" || + cause === null || + !("code" in cause) || + cause.code !== "ENOENT" || + !("path" in cause) || + cause.path !== scanArguments.repository + ) { + throw error; + } + } } scanArguments.parentScanId = undefined; // Resume uses the installed engine with the saved recipe and checkpoints. @@ -2532,7 +2551,12 @@ export async function main( scanArguments.includeFailureResults = argv.includes("--full-output") && (format === "json" || format === "jsonl"); - const outcome = await runScan(scanArguments, errorOutput, dependencies); + const outcome = await runScan( + scanArguments, + errorOutput, + dependencies, + !selectedFinalization, + ); return finishScan(outcome, format, incurError); }, }) @@ -4923,20 +4947,24 @@ export async function main( ); return undefined; } - const session = - typeof saved["threadId"] === "string" - ? await findScanSession( - codexSecurityCredentialHome( - dependencies.environment, - ), - saved["threadId"], - ) - : null; - if (session?.workingDirectory !== scanDir) { - errorOutput.write( - `codex-security: ${scan.scanId}: Original session logs are unavailable. Preserving this attempt and starting a new one.\n`, - ); - return undefined; + const selectedFinalization = + saved["selectedFinalization"] === true; + if (!selectedFinalization) { + const session = + typeof saved["threadId"] === "string" + ? await findScanSession( + codexSecurityCredentialHome( + dependencies.environment, + ), + saved["threadId"], + ) + : null; + if (session?.workingDirectory !== scanDir) { + errorOutput.write( + `codex-security: ${scan.scanId}: Original session logs are unavailable. Preserving this attempt and starting a new one.\n`, + ); + return undefined; + } } const recipe = await prepareScanArgumentsFromRecipe( saved["recipe"], @@ -4948,6 +4976,7 @@ export async function main( : undefined, }, currentDirectory, + selectedFinalization, ); const security = dependencies.createSecurity({ pluginPath: options.pluginPath, @@ -6587,6 +6616,7 @@ async function prepareScanArgumentsFromRecipe( "scanPrompt" | "scanPromptFile" | "validationPromptFile" >, directory: string, + selectedFinalization = false, ): Promise { if (recipe === undefined || !isJsonObject(recipe)) { throw new CodexSecurityError( @@ -6594,6 +6624,7 @@ async function prepareScanArgumentsFromRecipe( ); } if ( + !selectedFinalization && recipe["requiresScanPrompt"] === true && scanPrompt === undefined && scanPromptFile === undefined @@ -6603,6 +6634,7 @@ async function prepareScanArgumentsFromRecipe( ); } if ( + !selectedFinalization && recipe["validationMode"] === "custom" && validationPromptFile === undefined ) { @@ -6761,7 +6793,11 @@ async function prepareScanArgumentsFromRecipe( repository, directory, ); - if (recipe["requiresScanPrompt"] === true && !prompts.scanPrompt?.trim()) { + if ( + !selectedFinalization && + recipe["requiresScanPrompt"] === true && + !prompts.scanPrompt?.trim() + ) { throw new CodexSecurityError( "This scan used additional instructions. The --scan-prompt-file must not be empty.", ); diff --git a/sdk/typescript/src/codex-session.ts b/sdk/typescript/src/codex-session.ts new file mode 100644 index 0000000000..27c21934fc --- /dev/null +++ b/sdk/typescript/src/codex-session.ts @@ -0,0 +1,27 @@ +import { + Codex, + type CodexOptions, + type ThreadOptions, + type TurnOptions, +} from "@openai/codex-sdk"; + +export interface CodexSessionEvent { + readonly type: string; + readonly [key: string]: unknown; +} + +export interface CodexSessionThread { + readonly id: string | null; + runStreamed( + input: string, + options: TurnOptions, + ): Promise<{ events: AsyncGenerator }>; +} + +export interface CodexSessionClient { + startThread(options: ThreadOptions): CodexSessionThread; + resumeThread?(threadId: string, options: ThreadOptions): CodexSessionThread; +} + +export const createCodexClient = (options: CodexOptions): CodexSessionClient => + new Codex(options); diff --git a/sdk/typescript/src/contract.ts b/sdk/typescript/src/contract.ts index 4530a03b3d..19bf126301 100644 --- a/sdk/typescript/src/contract.ts +++ b/sdk/typescript/src/contract.ts @@ -1,22 +1,11 @@ import { isDeepStrictEqual as sameArray } from "node:util"; import { createHash, hash } from "node:crypto"; import { normalizePersistedFindingDetails } from "./finding-normalization.js"; -import { constants, type BigIntStats, type Stats } from "node:fs"; -import { - lstat, - open, - readFile, - realpath, - type FileHandle, -} from "node:fs/promises"; -import { isAbsolute, join, posix, resolve } from "node:path"; +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; import Ajv2020, { type ErrorObject } from "ajv/dist/2020.js"; import { regexes } from "zod"; -import { - ContractValidationError, - abortReason, - errorMessage, -} from "./errors.js"; +import { ContractValidationError, errorMessage } from "./errors.js"; import { isRecord } from "./record.js"; import type { CoverageDocument, @@ -25,12 +14,20 @@ import type { ScanManifest, } from "./models.js"; import { - requirePrivateOutputDirectory, - requireSecureOutputAncestry, -} from "./runtime.js"; + openCheckedScanFile, + portableRelativePath, + requireScanRoot, + safeRelativePath, + throwIfAborted, + verifyScanRoot, + type ScanRoot, +} from "./scan-files.js"; +export { + requireScanFile, + readScanFile, + sameCheckedFileDevice, +} from "./scan-files.js"; import type { NormalizedTarget, ScanMode } from "./targets.js"; -import { isWithin as isContained } from "./trusted-executable.js"; -import { isWindowsUnsafePathComponent } from "./windows-path.js"; const DOCUMENTS = { "scan-manifest.json": "scan-manifest.schema.json", @@ -38,17 +35,6 @@ const DOCUMENTS = { "coverage.json": "coverage.schema.json", } as const; const PRODUCER_NAME = "codex-security-plugin"; -interface CheckedScanFile { - path: string; - metadata: Stats; - parents: Array<{ path: string; metadata: Stats }>; -} - -interface ScanRoot { - path: string; - metadata: Stats; -} - export interface ScanExpectation { repository: string; repositoryRevision: string | null; @@ -354,98 +340,6 @@ function validateCanonicalContract( } } -export async function requireScanFile( - scanDirectory: string, - relativePath: string, - context: string, - signal?: AbortSignal, -): Promise { - return ( - await requireCheckedScanFile(scanDirectory, relativePath, context, signal) - ).path; -} - -export async function readScanFile( - scanDirectory: string, - relativePath: string, - context: string, - signal?: AbortSignal, -): Promise { - const file = await openCheckedScanFile( - scanDirectory, - relativePath, - context, - signal, - ); - try { - return await file.readFile({ signal }); - } finally { - await file.close(); - } -} - -async function requireCheckedScanFile( - scanDirectory: string, - relativePath: string, - context: string, - signal?: AbortSignal, - expectedRoot?: ScanRoot, -): Promise { - const checkedRoot = await requireScanRoot(scanDirectory, signal); - const scanDir = checkedRoot.path; - throwIfAborted(signal); - const safePath = portableRelativePath(relativePath, context); - const parts = safePath.split("/"); - let current = scanDir; - try { - const rootMetadata = checkedRoot.metadata; - if ( - expectedRoot !== undefined && - (scanDir !== expectedRoot.path || - rootMetadata.dev !== expectedRoot.metadata.dev || - rootMetadata.ino !== expectedRoot.metadata.ino) - ) { - throw new Error("scan directory changed while reading"); - } - const parents = [{ path: scanDir, metadata: rootMetadata }]; - throwIfAborted(signal); - for (const part of parts.slice(0, -1)) { - current = join(current, part); - const metadata = await lstat(current); - throwIfAborted(signal); - if (!metadata.isDirectory()) { - throw new Error("unsafe parent"); - } - parents.push({ path: current, metadata }); - } - const path = join(scanDir, ...parts); - const metadata = await lstat(path); - throwIfAborted(signal); - if (!metadata.isFile()) { - throw new ContractValidationError( - `${context}: expected a regular non-symlink file.`, - ); - } - const canonical = await realpath(path); - throwIfAborted(signal); - if (!isContained(scanDir, canonical)) { - throw new Error("outside scan directory"); - } - return { path, metadata, parents }; - } catch (error) { - throwIfAborted(signal); - if (error instanceof ContractValidationError) { - throw error; - } - throw new ContractValidationError( - `${context}: expected a file inside the scan directory.`, - { - cause: error, - }, - ); - } -} - async function validateSeal( scanDir: string, manifest: ScanManifest, @@ -630,125 +524,6 @@ function expectedCoverageMode( return mode === "deep" ? "deep_repository" : "repository"; } -async function requireScanRoot( - scanDirectory: string, - signal?: AbortSignal, -): Promise { - throwIfAborted(signal); - const absolute = resolve(scanDirectory); - try { - const metadata = await lstat(absolute); - throwIfAborted(signal); - const canonical = await realpath(absolute); - throwIfAborted(signal); - const current = await lstat(absolute); - throwIfAborted(signal); - const returned = await lstat(canonical); - throwIfAborted(signal); - if ( - !metadata.isDirectory() || - !current.isDirectory() || - metadata.dev !== current.dev || - metadata.ino !== current.ino || - !returned.isDirectory() || - metadata.dev !== returned.dev || - metadata.ino !== returned.ino - ) { - throw new Error("not a directory"); - } - try { - requirePrivateOutputDirectory(returned, canonical); - await requireSecureOutputAncestry(canonical); - } catch (error) { - throw new ContractValidationError( - error instanceof Error - ? error.message - : "Scan directory must remain private to the current user.", - { cause: error }, - ); - } - return { path: canonical, metadata: returned }; - } catch (error) { - throwIfAborted(signal); - if (error instanceof ContractValidationError) throw error; - throw new ContractValidationError( - "Scan directory must be an existing non-symlink directory.", - { cause: error }, - ); - } -} - -async function verifyScanRoot( - root: ScanRoot, - signal?: AbortSignal, -): Promise { - try { - const current = await lstat(root.path); - throwIfAborted(signal); - if ( - !current.isDirectory() || - current.dev !== root.metadata.dev || - current.ino !== root.metadata.ino - ) { - throw new Error("scan directory changed while reading"); - } - requirePrivateOutputDirectory(current, root.path); - await requireSecureOutputAncestry(root.path); - } catch (error) { - throwIfAborted(signal); - throw new ContractValidationError( - "Scan directory changed while reading the canonical contract.", - { cause: error }, - ); - } -} - -function safeRelativePath( - value: string, - context: string, - allowDot = false, -): string { - const parts = value.split("/"); - if ( - value.length === 0 || - Buffer.from(value, "utf8").toString("utf8") !== value || - (!allowDot && value === ".") || - value.startsWith("/") || - /^[A-Za-z]:/.test(value) || - parts.includes("..") || - value.includes("\\") || - /[\u0000-\u001f]/u.test(value) - ) { - throw new ContractValidationError( - `${context}: expected a safe scan-relative POSIX path.`, - ); - } - const normalized = posix.normalize(value).replace(/\/+$/, ""); - if ( - (!allowDot && normalized === ".") || - normalized.startsWith("../") || - isAbsolute(normalized) - ) { - throw new ContractValidationError( - `${context}: expected a safe scan-relative POSIX path.`, - ); - } - return normalized; -} - -function portableRelativePath(value: string, context: string): string { - const normalized = safeRelativePath(value, context); - if ( - value.trim().length === 0 || - value.split("/").some(isWindowsUnsafePathComponent) - ) { - throw new ContractValidationError( - `${context}: expected a safe scan-relative POSIX path.`, - ); - } - return normalized; -} - export function safeScopePath(value: string): string { return value === "." ? value @@ -936,117 +711,6 @@ async function sha256ScanFile( } } -async function openCheckedScanFile( - scanDir: string, - relativePath: string, - context: string, - signal?: AbortSignal, - expectedRoot?: ScanRoot, -): Promise { - const checked = await requireCheckedScanFile( - scanDir, - relativePath, - context, - signal, - expectedRoot, - ); - let file: FileHandle | undefined; - try { - file = await open( - checked.path, - constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK, - ); - const opened = await file.stat(); - throwIfAborted(signal); - if ( - !opened.isFile() || - opened.ino !== checked.metadata.ino || - !(await sameCheckedFileDevice(file, checked, opened)) - ) { - throw new ContractValidationError( - `${context}: expected the checked regular file.`, - ); - } - for (const parent of checked.parents) { - const current = await lstat(parent.path); - throwIfAborted(signal); - if ( - !current.isDirectory() || - current.dev !== parent.metadata.dev || - current.ino !== parent.metadata.ino - ) { - throw new ContractValidationError( - `${context}: checked parent changed before opening the file.`, - ); - } - } - const current = await lstat(checked.path); - throwIfAborted(signal); - if ( - !current.isFile() || - current.dev !== checked.metadata.dev || - current.ino !== checked.metadata.ino - ) { - throw new ContractValidationError( - `${context}: checked file changed before reading.`, - ); - } - return file; - } catch (error) { - await file?.close(); - throwIfAborted(signal); - if (error instanceof ContractValidationError) throw error; - throw new ContractValidationError( - `${context}: unable to open the checked regular file.`, - { cause: error }, - ); - } -} - -export async function sameCheckedFileDevice( - file: FileHandle, - checked: { path: string; metadata: Pick }, - opened: Pick, - platform: NodeJS.Platform = process.platform, - openReference: (path: string, flags: number) => Promise = open, -): Promise { - if (opened.ino !== checked.metadata.ino) return false; - if (opened.dev === checked.metadata.dev) return true; - if (platform !== "win32") return false; - - const [openedIdentity, checkedIdentity] = await Promise.all([ - file.stat({ bigint: true }), - lstat(checked.path, { bigint: true }), - ]); - if ( - !openedIdentity.isFile() || - !checkedIdentity.isFile() || - openedIdentity.ino !== checkedIdentity.ino - ) { - return false; - } - - const reference = await openReference( - checked.path, - constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK, - ); - try { - const referenceIdentity = await reference.stat({ bigint: true }); - return ( - referenceIdentity.isFile() && - openedIdentity.dev === referenceIdentity.dev && - openedIdentity.ino === referenceIdentity.ino - ); - } finally { - await reference.close(); - } -} - -function throwIfAborted(signal?: AbortSignal): void { - if (!signal?.aborted) return; - throw abortReason(signal); -} - const RFC3339_DATE_TIME = new RegExp( regexes.datetime({ offset: true }).source, "i", diff --git a/sdk/typescript/src/cost-model.ts b/sdk/typescript/src/cost-model.ts index fa2fa63ead..ece49157fe 100644 --- a/sdk/typescript/src/cost-model.ts +++ b/sdk/typescript/src/cost-model.ts @@ -10,6 +10,8 @@ export interface ScanCost { outputTokens: number; /** Short-context baseline retained for compatibility and spending limits. */ estimatedUsd: number; + coverage?: "partial"; + modelCosts?: readonly ScanCost[]; /** Standard token-cost bounds for the observed usage, not a billing total. */ estimatedUsdRange?: { min: number; @@ -174,6 +176,121 @@ export function tokenUsage(value: unknown): ScanTokenUsage | null { export function estimateScanCost( model: string | undefined, usage: unknown, +): ScanCost | null { + if (isRecord(usage) && Array.isArray(usage["modelUsage"])) { + const total = tokenUsage(usage); + if (total === null || usage["modelUsage"].length === 0) return null; + const costs: ScanCost[] = []; + for (const part of usage["modelUsage"]) { + if (!isRecord(part)) return null; + const tokens = tokenUsage(part); + if (tokens === null) return null; + const cost = + typeof part["model"] === "string" + ? estimateModelCost(part["model"], part) + : null; + if (cost === null) { + if (tokens.total_tokens === 0) continue; + return null; + } + costs.push(cost); + } + if (costs.length === 0) return null; + const sum = ( + key: + | "inputTokens" + | "cachedInputTokens" + | "cacheWriteInputTokens" + | "outputTokens" + | "estimatedUsd", + ) => costs.reduce((value, cost) => value + cost[key], 0); + if ( + sum("inputTokens") !== total.input_tokens || + sum("cachedInputTokens") !== total.cached_input_tokens || + sum("cacheWriteInputTokens") !== total.cache_write_input_tokens || + sum("outputTokens") !== total.output_tokens + ) + return null; + return { + model: model ?? costs[0]!.model, + inputTokens: total.input_tokens, + cachedInputTokens: total.cached_input_tokens, + cacheWriteInputTokens: total.cache_write_input_tokens, + ...(total.cache_write_input_tokens_reported === false + ? { cacheWriteInputTokensReported: false } + : {}), + outputTokens: total.output_tokens, + estimatedUsd: sum("estimatedUsd"), + estimatedUsdRange: { + min: sum("estimatedUsd"), + max: costs.some((cost) => cost.estimatedUsdRange?.max == null) + ? null + : costs.reduce( + (value, cost) => value + cost.estimatedUsdRange!.max!, + 0, + ), + context: "unknown", + }, + modelCosts: costs, + ...(costs.length === 1 ? { pricing: costs[0]!.pricing } : {}), + ...(usage["coverage"] === "partial" + ? { coverage: "partial" as const } + : {}), + }; + } + const cost = estimateModelCost(model, usage); + return cost && isRecord(usage) && usage["coverage"] === "partial" + ? { ...cost, coverage: "partial" } + : cost; +} + +// Internal budget enforcement only. The public estimate remains unavailable +// when some attributed usage has no price. +export function estimateScanCostLowerBound( + model: string | undefined, + usage: unknown, +): ScanCost | null { + if (!isRecord(usage) || !Array.isArray(usage["modelUsage"])) return null; + const total = tokenUsage(usage); + if (total === null) return null; + const keys = [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "reasoning_output_tokens", + ] as const; + const observed = Object.fromEntries(keys.map((key) => [key, 0])); + const priced = Object.fromEntries(keys.map((key) => [key, 0])); + const parts: Record[] = []; + let cacheWritesReported = true; + for (const part of usage["modelUsage"]) { + const normalized = tokenUsage(part); + if (!isRecord(part) || normalized === null) return null; + for (const key of keys) observed[key]! += normalized[key]; + if ( + typeof part["model"] !== "string" || + estimateModelCost(part["model"], part) === null + ) + continue; + parts.push(part); + for (const key of keys) priced[key]! += normalized[key]; + if (normalized.cache_write_input_tokens_reported === false) + cacheWritesReported = false; + } + // A malformed partition is not evidence of an enforceable lower bound. + if (keys.some((key) => observed[key] !== total[key])) return null; + return estimateScanCost(model, { + ...priced, + cache_write_input_tokens_reported: cacheWritesReported, + modelUsage: parts, + coverage: "partial", + }); +} + +function estimateModelCost( + model: string | undefined, + usage: unknown, ): ScanCost | null { if (model === undefined) return null; const bedrockPricing = Object.hasOwn(BEDROCK_MODEL_PRICING, model) diff --git a/sdk/typescript/src/cost.ts b/sdk/typescript/src/cost.ts index 5faa6723b0..046c132f86 100644 --- a/sdk/typescript/src/cost.ts +++ b/sdk/typescript/src/cost.ts @@ -1,10 +1,11 @@ import { parseJson } from "./value.js"; -import { open, readdir, stat } from "node:fs/promises"; +import { createHash } from "node:crypto"; +import { open, readdir, realpath, stat } from "node:fs/promises"; import { join } from "node:path"; -import { StringDecoder } from "node:string_decoder"; import { isRecord } from "./record.js"; import { estimateScanCost, + estimateScanCostLowerBound, tokenUsage, type ScanCost, type ScanTokenUsage, @@ -15,10 +16,14 @@ import { type ScanActivity, } from "./scan-activity.js"; import { + attributedScanThreads, + isAttributedScanEvent, isScanArtifactDirectory, + recordedScanCodexHome, sessionOwnsTurn, sessionParentThreadId, sessionStartedAt, + type ScanExecutionAttribution, } from "./scan-sessions.js"; import { scanProgressUpdatesFromText, @@ -47,9 +52,41 @@ function createSessionReasoning(id: string, text = "") { return { id, text, raw: false, activity: null as ScanActivity | null }; } -type SessionUsage = ReturnType & { - events?: Record[]; -}; +interface SessionUsage { + offset: number; + pendingLine: Buffer[]; + unreadable: boolean; + threadId: string | null; + parentThreadId: string | null; + workingDirectory: string | null; + startedAt: number | null; + inheritedUsage: ScanTokenUsage | null; + replaying: boolean; + usage: ScanTokenUsage | null; + counterUsage: ScanTokenUsage | null; + model: string | null; + modelUsage: Map; + counterModelUsage: Map; + currentTurnId: string | null; + previousUsage: ScanTokenUsage | null; + responseIds: Set; + responseUsageObserved: boolean; + responseTokens: number; + expectedResponseTokens: number; + responseEpochUsageObserved: boolean; + incompleteResponseEpoch: boolean; + counterRegressed: boolean; + calls: Map; + activities: { index: number; activity: ScanActivity }[]; + progress: ScanProgress[]; + filesCompleted: number; + filesTotal: number | null; + prose: Set; + reasoning: SessionReasoning | null; + reasoningCount: number; + eventIndex: number; + events?: { index: number; event: Record }[]; +} interface ScanCostTrackerOptions { codexHome: string; @@ -60,6 +97,8 @@ interface ScanCostTrackerOptions { expectedFilesTotal?: number; /** Incomplete costs are measured subtotals, usable for spending limits only. */ onCost?: (cost: Readonly, complete: boolean) => void; + // Only reported when the full public estimate is unavailable. + onCostLowerBound?: (cost: Readonly) => void; onUsage?: (usage: Readonly) => void; onActivity?: (activity: ScanActivity) => void; onProgress?: (progress: ScanProgress) => void; @@ -76,11 +115,10 @@ interface ScanCostSnapshot { const COST_POLL_INTERVAL_MS = 100; const SESSION_READ_SIZE = 64 * 1_024; -function createSessionUsage() { +function createSessionUsage(): SessionUsage { return { offset: 0, - decoder: new StringDecoder("utf8"), - pendingLine: [] as string[], + pendingLine: [], unreadable: false, threadId: null as string | null, parentThreadId: null as string | null, @@ -88,17 +126,29 @@ function createSessionUsage() { startedAt: null as number | null, inheritedUsage: null as ScanTokenUsage | null, replaying: false, - usage: null as ScanTokenUsage | null, - responseUsage: null as ScanTokenUsage | null, - responseIds: new Set(), - calls: new Map(), - activities: [] as ScanActivity[], - progress: [] as ScanProgress[], + usage: null, + counterUsage: null, + model: null, + modelUsage: new Map(), + counterModelUsage: new Map(), + currentTurnId: null, + previousUsage: null, + responseIds: new Set(), + responseUsageObserved: false, + responseTokens: 0, + expectedResponseTokens: 0, + responseEpochUsageObserved: false, + incompleteResponseEpoch: false, + counterRegressed: false, + calls: new Map(), + activities: [], + progress: [], filesCompleted: 0, filesTotal: null as number | null, prose: new Set(), reasoning: null as SessionReasoning | null, reasoningCount: 0, + eventIndex: 0, }; } @@ -109,15 +159,24 @@ export class ScanCostTracker { readonly #workers = new Map(); readonly #workerProgress = new Map(); readonly #reportedProgress = new Set(); + readonly #reportedSessionEvents = new Map>(); + readonly #reportedActivities = new Map>(); #threadId: string | null = null; #observingWorkers = true; #timer: NodeJS.Timeout | null = null; #pending: Promise = Promise.resolve(); #snapshot: ScanCostSnapshot = { usage: null, cost: null }; #lastCost: string | null = null; + #lastCostLowerBound: string | null = null; #lastUsage: string | null = null; #highestFilesCompleted = 0; #expectedFilesTotal: number | undefined; + #attribution: ScanExecutionAttribution | null = null; + #readAttribution: + | (( + force?: boolean, + ) => Promise) + | undefined; public constructor(options: ScanCostTrackerOptions) { this.#options = options; @@ -128,10 +187,25 @@ export class ScanCostTracker { this.#expectedFilesTotal = filesTotal; } + public setAttributionReader( + reader: ( + force?: boolean, + ) => Promise, + ): void { + this.#readAttribution = reader; + } + public recordUsage(usage: unknown, threadId = this.#threadId): void { const normalized = tokenUsage(usage); if (threadId !== null) { - this.#receipts.set(threadId, normalized); + const previous = this.#receipts.get(threadId); + if ( + previous == null || + (normalized !== null && + normalized.total_tokens >= previous.total_tokens) + ) { + this.#receipts.set(threadId, normalized); + } } } @@ -141,6 +215,7 @@ export class ScanCostTracker { if ( this.#options.maxCostUsd === undefined && this.#options.onCost === undefined && + this.#options.onCostLowerBound === undefined && this.#options.onUsage === undefined && this.#options.onActivity === undefined && this.#options.onProgress === undefined && @@ -175,8 +250,12 @@ export class ScanCostTracker { } public async refresh(): Promise { + return this.#refresh(false); + } + + async #refresh(forceAttribution: boolean): Promise { const update = this.#pending.then(async () => { - await this.#readSessions(); + await this.#readSessions(forceAttribution); }); this.#pending = update.catch(() => {}); await update; @@ -188,44 +267,92 @@ export class ScanCostTracker { this.#timer = null; if (fallbackUsage !== undefined) this.recordUsage(fallbackUsage); try { - await this.refresh(); + await this.#refresh(true); } finally { this.#observingWorkers = false; } - if (this.#receipts.size > 0 || this.#snapshot.usage !== null) + if ( + this.#attribution !== null || + this.#receipts.size > 0 || + this.#snapshot.usage !== null + ) return this.#snapshot; const cost = estimateScanCost(this.#options.model, fallbackUsage); this.#snapshot = { usage: fallbackUsage ?? null, cost }; this.#reportUsage(tokenUsage(fallbackUsage)); - this.#reportCost(cost); + this.#reportCost(cost, fallbackUsage); return this.#snapshot; } - async #readSessions(): Promise { + async #readSessions(forceAttribution: boolean): Promise { if (this.#threadId === null) return; const repository = this.#options.onActivity === undefined ? undefined : this.#options.repository; - const unreadable: Array<{ session: SessionUsage; error: unknown }> = []; - for await (const path of sessionFiles( - join(this.#options.codexHome, "sessions"), - )) { - let session = this.#sessions.get(path); - if (session === undefined) { - session = createSessionUsage(); - this.#sessions.set(path, session); + let recordedAttribution = this.#attribution; + if (this.#readAttribution) { + const record = await this.#readAttribution(forceAttribution); + recordedAttribution = record ?? null; + const attribution = record == null || record.legacy ? null : record; + if ( + attribution && + (!this.#attribution || + attribution.workerCodexHome !== this.#attribution.workerCodexHome || + attribution.completedAt !== this.#attribution.completedAt) + ) { + this.#sessions.clear(); } - try { - await readSessionUsage(path, session, repository); - } catch (error) { - if (session.threadId === null) throw error; - unreadable.push({ session, error }); + this.#attribution = attribution; + } + const unreadable: Array<{ session: SessionUsage; error: unknown }> = []; + const homes = new Set([this.#options.codexHome]); + if (this.#options.scanDirectory !== undefined) { + const home = await recordedScanCodexHome( + this.#options.scanDirectory, + recordedAttribution, + ); + if (home !== undefined) homes.add(home); + } + // Recovery restores workers to their recorded home; the SDK parent can + // continue in the current home. Apply the same scan membership to both. + const directories = new Set(); + for (const home of homes) { + for (const source of ["sessions", "archived_sessions"]) { + let directory: string; + try { + directory = await realpath(join(home, source)); + } catch (error) { + if (isMissingFile(error)) continue; + throw error; + } + if (directories.has(directory)) continue; + directories.add(directory); + for await (const path of sessionFiles(directory)) { + let session = this.#sessions.get(path); + if (session === undefined) { + session = createSessionUsage(); + this.#sessions.set(path, session); + } + try { + await readSessionUsage( + path, + session, + repository, + this.#attribution, + ); + } catch (error) { + if (session.threadId === null) throw error; + unreadable.push({ session, error }); + } + } } } - const included = new Set([this.#threadId, ...this.#receipts.keys()]); - if (this.#options.scanDirectory !== undefined) { + const included = this.#attribution + ? attributedScanThreads(this.#sessions.values(), this.#attribution) + : new Set([this.#threadId, ...this.#receipts.keys()]); + if (!this.#attribution && this.#options.scanDirectory !== undefined) { const scanStartedAt = [...this.#sessions.values()].find( (session) => session.threadId === this.#threadId, @@ -251,9 +378,9 @@ export class ScanCostTracker { } } } - let previousSize: number; - do { - previousSize = included.size; + let changed = this.#attribution === null; + while (changed) { + const previousSize = included.size; for (const session of this.#sessions.values()) { if ( session.threadId !== null && @@ -263,12 +390,27 @@ export class ScanCostTracker { included.add(session.threadId); } } - } while (included.size !== previousSize); + changed = included.size !== previousSize; + } for (const { session, error } of unreadable) { if (included.has(session.threadId!)) throw error; } - const usages = new Map(this.#receipts); + let incomplete = false; + const usages = new Map( + [...this.#receipts].filter( + ([threadId]) => + included.has(threadId) && + (!this.#attribution || + this.#attribution.executionThreadIds.includes(threadId)), + ), + ); + if (this.#attribution) { + for (const threadId of included) { + if (!usages.has(threadId)) usages.set(threadId, null); + } + } + const usageSessions = new Map(); for (const [path, tracked] of this.#sessions) { const threadId = tracked.threadId; if (threadId === null || !included.has(threadId)) continue; @@ -280,7 +422,7 @@ export class ScanCostTracker { // Replay only newly associated sessions, including their early events. session = createSessionUsage(); session.events = []; - await readSessionUsage(path, session, repository); + await readSessionUsage(path, session, repository, this.#attribution); this.#sessions.set(path, session); } let worker: number | undefined; @@ -293,7 +435,20 @@ export class ScanCostTracker { this.#options.onWorkerEvent?.({ kind: "observed", worker }); } } - for (const event of session.events?.splice(0) ?? []) { + for (const { index, event } of session.events?.splice(0) ?? []) { + let reported = this.#reportedSessionEvents.get(threadId); + if (reported === undefined) { + reported = new Set(); + this.#reportedSessionEvents.set(threadId, reported); + } + // A physical copy keeps each event's position, including repeated + // identical events. Positions count unfiltered records so attribution + // changes can replay the same log without changing occurrence identity. + const identity = `${index}:${createHash("sha256") + .update(JSON.stringify(event)) + .digest("hex")}`; + if (reported.has(identity)) continue; + reported.add(identity); this.#options.onSessionEvent?.({ threadId, parentThreadId: session.parentThreadId, @@ -302,7 +457,17 @@ export class ScanCostTracker { }); } if (worker !== undefined) { - for (const activity of session.activities.splice(0)) { + for (const { index, activity } of session.activities.splice(0)) { + let reported = this.#reportedActivities.get(threadId); + if (reported === undefined) { + reported = new Set(); + this.#reportedActivities.set(threadId, reported); + } + const identity = `${index}:${createHash("sha256") + .update(JSON.stringify(activity)) + .digest("hex")}`; + if (reported.has(identity)) continue; + reported.add(identity); this.#options.onActivity?.({ ...activity, id: `${threadId}:${activity.id}`, @@ -311,6 +476,31 @@ export class ScanCostTracker { } this.#reportWorkerProgress(session); } + // Choose one copy using both receipts and cumulative lower bounds. + // Equal totals can still precede later corrections in the full log. + const previous = usageSessions.get(threadId); + const total = sessionUsageTotal(session); + const previousTotal = previous ? sessionUsageTotal(previous) : -1; + if ( + previous === undefined || + total > previousTotal || + (total === previousTotal && + (session.eventIndex > previous.eventIndex || + (session.eventIndex === previous.eventIndex && + session.pendingLine.length === 0 && + previous.pendingLine.length > 0))) + ) { + usageSessions.set(threadId, session); + } + } + for (const [threadId, session] of usageSessions) { + if ( + session.counterUsage && + session.counterUsage.total_tokens > + (usages.get(threadId)?.total_tokens ?? -1) + ) { + usages.set(threadId, session.counterUsage); + } const receipt = usages.get(threadId); // A discovered thread without a receipt has unknown usage, not zero. // Retain it so a parent-only subtotal cannot appear to be complete. @@ -324,24 +514,132 @@ export class ScanCostTracker { ) { usages.set(threadId, session.usage); } + if (!usages.has(threadId)) usages.set(threadId, null); + let selected = usages.get(threadId); + const counter = session.counterUsage; + if (selected && counter) + selected = addTokenUsage( + selected, + tokenUsageRemainder(counter, selected), + ); + if (selected && session.usage) { + const remainder = tokenUsageRemainder(selected, session.usage); + if (remainder.total_tokens > 0) { + usages.set(threadId, addTokenUsage(session.usage, remainder)); + if (session.responseUsageObserved) incomplete = true; + } + } + if ( + (session.counterRegressed && !session.responseUsageObserved) || + session.incompleteResponseEpoch || + (session.responseEpochUsageObserved && + session.expectedResponseTokens > session.responseTokens) + ) + incomplete = true; + if (session.pendingLine.length > 0 && !this.#receipts.get(threadId)) + incomplete = true; } let usage: ScanTokenUsage | null = null; - let complete = true; + let missingUsage = false; for (const value of usages.values()) { if (value === null) { - complete = false; + incomplete = true; + if (!this.#attribution) missingUsage = true; continue; } usage = addTokenUsage(usage, value); } - if (!complete) this.#snapshot = { usage: null, cost: null }; - if (usage === null) return; - const cost = estimateScanCost(this.#options.model, usage); - if (complete) { - this.#snapshot = { usage, cost }; - this.#reportUsage(usage); + if (usage === null) { + this.#snapshot = { usage: null, cost: null }; + return; + } + const modelUsage = new Map(); + const liveModelUsage = new Map(); + let observedModel = false; + for (const [threadId, value] of usages) { + if (value === null) continue; + const session = usageSessions.get(threadId); + for (const [model, tokens] of reconcileModelUsage( + value, + session?.modelUsage, + ) ?? []) { + if (model !== null) observedModel = true; + modelUsage.set( + model, + addTokenUsage(modelUsage.get(model) ?? null, tokens), + ); + } + const remainder = session?.usage + ? subtractTokenUsage(value, session.usage) + : value; + if (remainder !== null && remainder.total_tokens > 0) { + const model = + (this.#attribution && threadId !== this.#threadId) || + (session?.modelUsage.size ?? 0) > 0 + ? null + : (session?.model ?? + (threadId === this.#threadId ? this.#options.model : null)); + modelUsage.set( + model, + addTokenUsage(modelUsage.get(model) ?? null, remainder), + ); + } + const live = + (session?.counterUsage?.total_tokens ?? -1) > + (session?.usage?.total_tokens ?? -1); + const liveModels = reconcileModelUsage( + value, + live ? session?.counterModelUsage : session?.modelUsage, + ); + let liveTokens: ScanTokenUsage | null = null; + for (const [model, tokens] of liveModels ?? []) { + liveModelUsage.set( + model, + addTokenUsage(liveModelUsage.get(model) ?? null, tokens), + ); + liveTokens = addTokenUsage(liveTokens, tokens); + } + // Native cumulative totals can exceed the history with a known model. + // Keep that history priced and represent only the unassigned remainder. + const liveRemainder = liveTokens + ? subtractTokenUsage(value, liveTokens) + : value; + if (liveRemainder !== null && liveRemainder.total_tokens > 0) + liveModelUsage.set( + null, + addTokenUsage(liveModelUsage.get(null) ?? null, liveRemainder), + ); } - this.#reportCost(cost, complete); + const reconciled = + observedModel || this.#attribution !== null + ? { + ...usage, + modelUsage: [...modelUsage].map(([model, tokens]) => ({ + model, + ...tokens, + })), + } + : usage; + const measured = incomplete + ? { ...reconciled, coverage: "partial" } + : reconciled; + const cost = estimateScanCost(this.#options.model, measured); + this.#snapshot = missingUsage + ? { usage: null, cost: null } + : { usage: measured, cost }; + if (!missingUsage) this.#reportUsage(tokenUsage(measured)); + this.#reportCost( + cost, + measured, + { + ...usage, + modelUsage: [...liveModelUsage].map(([model, tokens]) => ({ + model, + ...tokens, + })), + }, + missingUsage, + ); } #reportWorkerProgress(session: SessionUsage): void { @@ -364,9 +662,10 @@ export class ScanCostTracker { this.#workerProgress.set(session.threadId, progress.filesCompleted); const filesCompleted = Math.min( expectedFilesTotal ?? Number.MAX_SAFE_INTEGER, - this.#workerProgress - .values() - .reduce((total, reviewed) => total + reviewed, 0), + [...this.#workerProgress.values()].reduce( + (total, reviewed) => total + reviewed, + 0, + ), ); if (filesCompleted < this.#highestFilesCompleted) continue; const update = { @@ -383,12 +682,38 @@ export class ScanCostTracker { } } - #reportCost(cost: ScanCost | null, complete = true): void { - if (cost === null) return; - const signature = JSON.stringify([cost, complete]); + #reportCost( + cost: ScanCost | null, + usage: unknown, + liveUsage?: unknown, + lowerBoundOnly = false, + ): void { + if (cost === null || lowerBoundOnly) { + const receiptCost = cost + ? { ...cost, coverage: "partial" as const } + : estimateScanCostLowerBound(this.#options.model, usage); + const liveCost = estimateScanCostLowerBound( + this.#options.model, + liveUsage, + ); + const lowerBound = + liveCost && liveCost.estimatedUsd > (receiptCost?.estimatedUsd ?? -1) + ? liveCost + : receiptCost; + if (lowerBound === null) return; + const signature = JSON.stringify(lowerBound); + if (signature === this.#lastCostLowerBound) return; + this.#lastCostLowerBound = signature; + this.#options.onCostLowerBound?.(lowerBound); + this.#lastCost = JSON.stringify([lowerBound, false]); + this.#options.onCost?.(lowerBound, false); + return; + } + const signature = JSON.stringify([cost, true]); if (signature === this.#lastCost) return; this.#lastCost = signature; - this.#options.onCost?.(cost, complete); + this.#lastCostLowerBound = null; + this.#options.onCost?.(cost, true); } #reportUsage(usage: ScanTokenUsage | null): void { @@ -429,6 +754,7 @@ async function readSessionUsage( path: string, session: SessionUsage, repository?: string, + attribution: ScanExecutionAttribution | null = null, ): Promise { if (session.unreadable) return; let file; @@ -451,14 +777,12 @@ async function readSessionUsage( if (bytesRead === 0) return; session.offset += bytesRead; try { - const lines = session.decoder - .write(buffer.subarray(0, bytesRead)) - .split("\n"); - session.pendingLine.push(lines[0]!); - for (const line of lines.slice(1)) { - readSessionEvent(session.pendingLine.join(""), session, repository); - session.pendingLine = [line]; - } + readSessionChunk( + buffer.subarray(0, bytesRead), + session, + repository, + attribution, + ); } catch (error) { session.unreadable = true; session.pendingLine = []; @@ -470,19 +794,61 @@ async function readSessionUsage( } } +function readSessionChunk( + contents: Buffer, + session: SessionUsage, + repository?: string, + attribution: ScanExecutionAttribution | null = null, +): void { + let lineStart = 0; + while (lineStart < contents.length) { + const newline = contents.indexOf(0x0a, lineStart); + const lineEnd = newline === -1 ? contents.length : newline; + const fragment = contents.subarray(lineStart, lineEnd); + + if (newline === -1) { + if (fragment.length > 0) { + session.pendingLine.push(Buffer.from(fragment)); + } + return; + } + + if (session.pendingLine.length === 0) { + readSessionEvent( + fragment.toString("utf8"), + session, + repository, + attribution, + ); + } else { + if (fragment.length > 0) session.pendingLine.push(Buffer.from(fragment)); + readSessionEvent( + Buffer.concat(session.pendingLine).toString("utf8"), + session, + repository, + attribution, + ); + session.pendingLine = []; + } + lineStart = newline + 1; + } +} + function readSessionEvent( line: string, session: SessionUsage, repository?: string, + attribution: ScanExecutionAttribution | null = null, ): void { if (line.length === 0) return; const event = parseJson(() => line); if (!isRecord(event) || !isRecord(event["payload"])) return; const payload = event["payload"]; + const index = session.eventIndex++; if (event["type"] === "session_meta") { if (session.threadId !== null) { session.replaying = payload["id"] !== session.threadId; - if (!session.replaying) session.events?.push(event); + if (!session.replaying) session.events?.push({ index, event }); return; } if (typeof payload["id"] === "string") { @@ -491,9 +857,10 @@ function readSessionEvent( if (typeof payload["cwd"] === "string") { session.workingDirectory = payload["cwd"]; } + if (typeof payload["model"] === "string") session.model = payload["model"]; session.startedAt = sessionStartedAt(payload["timestamp"]); session.parentThreadId = sessionParentThreadId(payload); - session.events?.push(event); + session.events?.push({ index, event }); return; } if (session.replaying) { @@ -507,39 +874,107 @@ function readSessionEvent( sessionOwnsTurn(session, payload) ) { session.replaying = false; - session.events?.push(event); + session.events?.push({ index, event }); } return; } - session.events?.push(event); + if (event["type"] === "compacted") { + // Later counter epochs cannot account for missing earlier scan receipts. + if ( + session.responseEpochUsageObserved && + session.expectedResponseTokens > session.responseTokens + ) + session.incompleteResponseEpoch = true; + session.responseTokens = 0; + session.expectedResponseTokens = 0; + session.responseEpochUsageObserved = false; + } + if ( + (event["type"] === "turn_context" || payload["type"] === "task_started") && + typeof payload["turn_id"] === "string" + ) { + session.currentTurnId = payload["turn_id"]; + } + if ( + event["type"] === "turn_context" && + typeof payload["model"] === "string" + ) { + session.model = payload["model"]; + } if (event["type"] === "token_usage_record") { const responseId = payload["response_id"]; + const usage = tokenUsage(payload["usage"]); if ( - payload["thread_id"] !== session.threadId || typeof responseId !== "string" || + usage === null || + (typeof payload["thread_id"] === "string" && + payload["thread_id"] !== session.threadId) || session.responseIds.has(responseId) - ) { + ) return; - } - const usage = tokenUsage(payload["usage"]); - if (usage === null) return; session.responseIds.add(responseId); - session.responseUsage = addTokenUsage(session.responseUsage, usage); - // Native thread totals include compaction and any earlier turns. They are - // scoped to thread_id, unlike inherited legacy token_count snapshots. - const total = tokenUsage(payload["thread_token_usage"]); - const recorded = - total !== null && total.total_tokens >= session.responseUsage.total_tokens - ? total - : session.responseUsage; + const cumulative = tokenUsage(payload["thread_token_usage"]); + // Keep native resumed totals above an observed legacy counter floor. if ( - session.usage === null || - recorded.total_tokens >= session.usage.total_tokens - ) { - session.usage = recorded; + attribution === null && + session.previousUsage !== null && + cumulative !== null && + cumulative.total_tokens >= usage.total_tokens && + cumulative.total_tokens > (session.counterUsage?.total_tokens ?? -1) + ) + session.counterUsage = cumulative; + if (cumulative) + session.expectedResponseTokens = Math.max( + session.expectedResponseTokens, + cumulative.total_tokens, + ); + session.responseTokens += usage.total_tokens; + const turnId = + typeof payload["turn_id"] === "string" + ? payload["turn_id"] + : session.currentTurnId; + if ( + attribution && + !isAttributedScanEvent( + attribution, + session.threadId!, + turnId, + event["timestamp"], + ) + ) + return; + session.responseEpochUsageObserved = true; + if (!session.responseUsageObserved) { + // Exact receipts include compaction and survive counter resets. Keep the + // legacy counter as an independent lower bound, never add it to receipts. + session.responseUsageObserved = true; + session.usage = null; + session.modelUsage.clear(); } + const model = + typeof payload["model"] === "string" ? payload["model"] : session.model; + session.usage = addTokenUsage(session.usage, usage); + session.modelUsage.set( + model, + addTokenUsage(session.modelUsage.get(model) ?? null, usage), + ); + session.events?.push({ index, event }); return; } + const attributable = + attribution === null || + isAttributedScanEvent( + attribution, + session.threadId!, + session.currentTurnId, + event["timestamp"], + ); + if (attributable) session.events?.push({ index, event }); + if ( + !attributable && + !(event["type"] === "event_msg" && payload["type"] === "token_count") + ) + return; if (event["type"] === "response_item") { session.progress.push(...sessionProgressUpdates(payload)); if (repository === undefined) return; @@ -606,7 +1041,7 @@ function readSessionEvent( if (activity.status === "running") { session.calls.set(activity.id, activity); } - session.activities.push(activity); + session.activities.push({ index: session.eventIndex - 1, activity }); return; } if ( @@ -617,8 +1052,11 @@ function readSessionEvent( const call = session.calls.get(payload["call_id"]); if (call !== undefined) { session.activities.push({ - ...call, - status: payload["status"] === "failed" ? "failed" : "completed", + index: session.eventIndex - 1, + activity: { + ...call, + status: payload["status"] === "failed" ? "failed" : "completed", + }, }); session.calls.delete(call.id); } @@ -651,7 +1089,7 @@ function readSessionEvent( !session.prose.has(`${activity.kind}:${activity.description}`) ) { session.prose.add(`${activity.kind}:${activity.description}`); - session.activities.push(activity); + session.activities.push({ index: session.eventIndex - 1, activity }); } return; } @@ -668,16 +1106,54 @@ function readSessionEvent( session.inheritedUsage === null ? usage : subtractTokenUsage(usage, session.inheritedUsage); - // Older runtimes only emit token_count. In newer rollouts these snapshots can - // lag compaction receipts, so do not replace a more complete native total. - if ( - ownUsage !== null && - (session.usage === null || - ownUsage.total_tokens > session.usage.total_tokens || - (session.responseUsage === null && - ownUsage.total_tokens === session.usage.total_tokens)) - ) { - session.usage = ownUsage; + if (ownUsage !== null) { + const hadPreviousUsage = session.previousUsage !== null; + const delta = + session.previousUsage === null + ? ownUsage + : subtractTokenUsage(ownUsage, session.previousUsage); + if ( + session.previousUsage !== null && + ownUsage.total_tokens < session.previousUsage.total_tokens + ) { + session.counterRegressed = true; + return; + } + session.previousUsage = ownUsage; + if ( + attribution && + !isAttributedScanEvent( + attribution, + session.threadId!, + session.currentTurnId, + event["timestamp"], + ) + ) + return; + if (delta !== null) { + const model = + !hadPreviousUsage && session.responseUsageObserved + ? null + : session.model; + session.counterModelUsage.set( + model, + addTokenUsage(session.counterModelUsage.get(model) ?? null, delta), + ); + if (!session.responseUsageObserved) + session.modelUsage.set( + session.model, + addTokenUsage(session.modelUsage.get(session.model) ?? null, delta), + ); + } + session.counterUsage = + attribution && delta !== null + ? addTokenUsage(session.counterUsage, delta) + : session.responseUsageObserved && + session.counterUsage !== null && + session.counterUsage.total_tokens >= ownUsage.total_tokens + ? session.counterUsage + : ownUsage; + if (!session.responseUsageObserved) session.usage = session.counterUsage; } } @@ -743,7 +1219,7 @@ function recordReasoningActivity( } reasoning.activity = activity; session.prose.add(`${activity.kind}:${activity.description}`); - session.activities.push(activity); + session.activities.push({ index: session.eventIndex - 1, activity }); } function sessionProgressUpdates( @@ -792,6 +1268,23 @@ function addTokenUsage( }; } +function reconcileModelUsage( + usage: ScanTokenUsage, + models: ReadonlyMap | undefined, +): ReadonlyMap | undefined { + if (models?.size !== 1) return models; + const [model, observed] = models.entries().next().value!; + if ( + model === null || + observed.input_tokens !== usage.input_tokens || + observed.output_tokens !== usage.output_tokens + ) + return models; + // A final receipt can classify existing input as cache reads or writes. + // When all tokens have one observed model, the classification belongs to it. + return new Map([[model, usage]]); +} + function subtractTokenUsage( usage: ScanTokenUsage, inherited: ScanTokenUsage, @@ -815,3 +1308,49 @@ function subtractTokenUsage( function isMissingFile(error: unknown): boolean { return isRecord(error) && error["code"] === "ENOENT"; } + +function sessionUsageTotal(session: SessionUsage): number { + if (session.usage === null) return session.counterUsage?.total_tokens ?? -1; + if (session.counterUsage === null) return session.usage.total_tokens; + return ( + session.usage.total_tokens + + tokenUsageRemainder(session.counterUsage, session.usage).total_tokens + ); +} + +function tokenUsageRemainder( + usage: ScanTokenUsage, + inherited: ScanTokenUsage, +): ScanTokenUsage { + const input = Math.max(0, usage.input_tokens - inherited.input_tokens); + const cached = Math.min( + input, + Math.max(0, usage.cached_input_tokens - inherited.cached_input_tokens), + ); + const writes = Math.min( + input - cached, + Math.max( + 0, + usage.cache_write_input_tokens - inherited.cache_write_input_tokens, + ), + ); + const output = Math.max(0, usage.output_tokens - inherited.output_tokens); + return { + input_tokens: input, + cached_input_tokens: cached, + cache_write_input_tokens: writes, + output_tokens: output, + reasoning_output_tokens: Math.min( + output, + Math.max( + 0, + usage.reasoning_output_tokens - inherited.reasoning_output_tokens, + ), + ), + total_tokens: input + output, + ...(usage.cache_write_input_tokens_reported === false || + inherited.cache_write_input_tokens_reported === false + ? { cache_write_input_tokens_reported: false } + : {}), + }; +} diff --git a/sdk/typescript/src/deep-scan-finalization.ts b/sdk/typescript/src/deep-scan-finalization.ts new file mode 100644 index 0000000000..e18bf9f583 --- /dev/null +++ b/sdk/typescript/src/deep-scan-finalization.ts @@ -0,0 +1,26 @@ +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; + +type WorkbenchRunner = (args: string[]) => Promise>; + +/** Load the same publisher used by native Deep from the installed plugin. */ +export async function resumeSelectedDeepScan(input: { + scanId: string; + threadId: string; + pluginRoot: string; + runWorkbench: WorkbenchRunner; + signal: AbortSignal; +}): Promise { + const helper = ( + await import(pathToFileURL(join(input.pluginRoot, "mcp/helpers.mjs")).href) + ).default as { + resumeSelectedDeepScan: (input: { + scanId: string; + threadId: string; + pluginRoot: string; + runWorkbench: WorkbenchRunner; + signal: AbortSignal; + }) => Promise; + }; + await helper.resumeSelectedDeepScan(input); +} diff --git a/sdk/typescript/src/preflight-config.ts b/sdk/typescript/src/preflight-config.ts new file mode 100644 index 0000000000..0f8724606d --- /dev/null +++ b/sdk/typescript/src/preflight-config.ts @@ -0,0 +1,182 @@ +import { isSafeNonNegativeInteger as safeInteger } from "./value.js"; +import { isRecord } from "./record.js"; +import { isAbsolute } from "node:path"; +import { + EXTERNAL_CODEX_PROVIDERS, + isExternalModelProvider, + resolveCodexProfile, + scanModelProvider, + type JsonObject, +} from "./config.js"; + +export function scanPreflightCodexConfig(config: JsonObject): JsonObject { + const safeString = (value: unknown): value is string => + typeof value === "string" && + value.length > 0 && + !/[\u0000-\u001f\u007f]/u.test(value); + const safeProfileName = (value: unknown): value is string => + safeString(value) && /^[A-Za-z0-9_-]+$/u.test(value); + const capabilityFeatures = (value: unknown): JsonObject => { + if (!isRecord(value)) return {}; + const result: JsonObject = {}; + for (const key of [ + "goals", + "cli_daybreak", + "multi_agent", + "enable_fanout", + "api_key_cyber_access_programs", + "api_key_model_discovery", + ]) { + if (typeof value[key] === "boolean") result[key] = value[key]; + } + const multiAgent = value["multi_agent_v2"]; + if (typeof multiAgent === "boolean") { + result["multi_agent_v2"] = multiAgent; + } else if (isRecord(multiAgent)) { + const sanitized: JsonObject = {}; + if (typeof multiAgent["enabled"] === "boolean") { + sanitized["enabled"] = multiAgent["enabled"]; + } + const capacity = multiAgent["max_concurrent_threads_per_session"]; + if (safeInteger(capacity)) { + sanitized["max_concurrent_threads_per_session"] = capacity; + } + if (Object.keys(sanitized).length > 0) { + result["multi_agent_v2"] = sanitized; + } + } + return result; + }; + const executionConfig = (source: JsonObject): JsonObject => { + const result: JsonObject = {}; + if (typeof source["daybreak"] === "boolean") { + result["daybreak"] = source["daybreak"]; + } + for (const key of [ + "model", + "model_reasoning_effort", + "model_reasoning_summary", + "model_provider", + "service_tier", + ]) { + const value = source[key]; + if (safeString(value)) result[key] = value; + } + const features = capabilityFeatures(source["features"]); + if (Object.keys(features).length > 0) result["features"] = features; + const skills = source["skills"]; + if (isRecord(skills) && Array.isArray(skills["config"])) { + result["skills"] = { + config: skills["config"].flatMap((entry) => + isRecord(entry) && + safeString(entry["path"]) && + typeof entry["enabled"] === "boolean" + ? [{ path: entry["path"], enabled: entry["enabled"] }] + : [], + ), + }; + } + const agents = source["agents"]; + if (isRecord(agents)) { + const sanitized: JsonObject = {}; + for (const key of ["max_threads", "max_depth"]) { + const value = agents[key]; + if (safeInteger(value)) sanitized[key] = value; + } + if (Object.keys(sanitized).length > 0) result["agents"] = sanitized; + } + const multiagent = source["multiagent_config"]; + if (isRecord(multiagent) && safeInteger(multiagent["max_concurrency"])) { + result["multiagent_config"] = { + max_concurrency: multiagent["max_concurrency"], + }; + } + return result; + }; + const resolved = resolveCodexProfile(config); + const result = executionConfig( + safeProfileName(config["profile"]) ? config : resolved, + ); + // Keep effective worker settings even when preflight filters the profile name. + for (const key of [ + "model_provider", + "model_reasoning_summary", + "service_tier", + ]) { + const value = resolved[key]; + if (safeString(value)) result[key] = value; + } + if (typeof resolved["daybreak"] === "boolean") { + result["daybreak"] = resolved["daybreak"]; + } + const resolvedFeatures = capabilityFeatures(resolved["features"]); + for (const key of [ + "cli_daybreak", + "api_key_cyber_access_programs", + "api_key_model_discovery", + ]) { + if (resolvedFeatures[key] !== undefined) { + result["features"] = { + ...(isRecord(result["features"]) ? result["features"] : {}), + [key]: resolvedFeatures[key], + }; + } + } + const selectedProfile = safeProfileName(config["profile"]) + ? config["profile"] + : undefined; + if (selectedProfile !== undefined) { + result["profile"] = selectedProfile; + } + const profiles = config["profiles"]; + if (isRecord(profiles)) { + const sanitized: JsonObject = {}; + for (const [name, profile] of Object.entries(profiles)) { + if (!safeProfileName(name) || !isRecord(profile)) continue; + const projected = executionConfig(profile as JsonObject); + if (Object.keys(projected).length === 0) continue; + sanitized[name] = projected; + } + if (Object.keys(sanitized).length > 0) result["profiles"] = sanitized; + } + const modelProvider = scanModelProvider(result); + if (isExternalModelProvider(modelProvider)) { + result["model_providers"] = { + [modelProvider]: { ...EXTERNAL_CODEX_PROVIDERS[modelProvider] }, + }; + } else if (modelProvider === "amazon-bedrock") { + const providers = resolved["model_providers"]; + const provider = isRecord(providers) ? providers[modelProvider] : undefined; + const aws = isRecord(provider) ? provider["aws"] : undefined; + if (isRecord(aws)) { + const sanitized: JsonObject = {}; + for (const key of ["region", "profile"]) { + const value = aws[key]; + if (safeString(value)) sanitized[key] = value; + } + if (Object.keys(sanitized).length > 0) { + result["model_providers"] = { + [modelProvider]: { aws: sanitized }, + }; + } + } + } + const rootMarkers = config["project_root_markers"]; + if (Array.isArray(rootMarkers)) { + result["project_root_markers"] = rootMarkers.filter(safeString); + } + const projects = config["projects"]; + if (isRecord(projects)) { + const sanitized: JsonObject = {}; + for (const [path, project] of Object.entries(projects)) { + if (!safeString(path) || !isAbsolute(path) || !isRecord(project)) { + continue; + } + const trust = project["trust_level"]; + if (trust !== "trusted" && trust !== "untrusted") continue; + sanitized[path] = { trust_level: trust }; + } + if (Object.keys(sanitized).length > 0) result["projects"] = sanitized; + } + return result; +} diff --git a/sdk/typescript/src/reasoning-summary.ts b/sdk/typescript/src/reasoning-summary.ts new file mode 100644 index 0000000000..e5babaabad --- /dev/null +++ b/sdk/typescript/src/reasoning-summary.ts @@ -0,0 +1,75 @@ +import { + inlineToml, + resolveCodexProfile, + scanModelConfiguration, + structuredCodexConfig, + type JsonObject, +} from "./config.js"; +import { scanPreflightCodexConfig } from "./preflight-config.js"; +import { + runCodexCommand, + type CodexCommand, + type ProcessEnvironment, +} from "./runtime.js"; + +/** Capture a future owner's selected model default; never infer historical settings. */ +export async function captureOriginalReasoningSummary(options: { + config: JsonObject; + nativeProfile?: string; + command: CodexCommand; + cwd: string; + environment: ProcessEnvironment; + signal: AbortSignal; +}): Promise { + // Keep explicit selections and invalid values with their existing native validator. + if ( + options.config["model_reasoning_summary"] !== undefined || + scanPreflightCodexConfig(options.config)["model_reasoning_summary"] !== + undefined || + resolveCodexProfile(options.config)["model_reasoning_summary"] !== undefined + ) + return undefined; + const { model } = scanModelConfiguration(options.config); + // The same per-session overrides protect the lookup from concurrent home edits. + const config = structuredCodexConfig(options.config); + const args = [ + "debug", + "models", + ...(options.nativeProfile === undefined + ? [] + : ["--profile", options.nativeProfile]), + ...Object.entries(config).flatMap(([key, value]) => [ + "--config", + `${key}=${inlineToml(value)}`, + ]), + ]; + const result = await runCodexCommand( + options.command, + args, + options.environment, + undefined, + options.signal, + options.cwd, + ); + // Older native executables and models absent from their catalog provide no + // recoverable value. Preserve omission instead of choosing another default. + if (!result.success) return undefined; + let catalog: unknown; + try { + catalog = JSON.parse(result.stdout); + } catch { + return undefined; + } + if (!isRecord(catalog) || !Array.isArray(catalog["models"])) return undefined; + const selected = catalog["models"].find( + (entry: unknown) => isRecord(entry) && entry["slug"] === model, + ); + return isRecord(selected) && + typeof selected["default_reasoning_summary"] === "string" + ? selected["default_reasoning_summary"] + : undefined; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/sdk/typescript/src/runtime.ts b/sdk/typescript/src/runtime.ts index 45b0e35183..c4532648af 100644 --- a/sdk/typescript/src/runtime.ts +++ b/sdk/typescript/src/runtime.ts @@ -85,6 +85,17 @@ import { windowsUnsafePathComponent, } from "./windows-path.js"; +import { + nodeErrorCode, + requirePrivateOutputDirectory, + requireSecureOutputAncestry, +} from "./scan-files.js"; +export { + requirePrivateOutputDirectory, + requireSecureOutputAncestry, + requireTrustedOutputAncestor, +} from "./scan-files.js"; + const execFile = promisify(execFileCallback); export type StartCodex = ( @@ -1700,6 +1711,7 @@ export async function runWorkbench( ) : input, signal, + undefined, archiveHandshake, // Match native argv's UTF-8 encoding for the private argument frame. framedArguments @@ -2157,96 +2169,6 @@ export async function validatePreparedOutputDir( return canonical; } -export function requirePrivateOutputDirectory( - metadata: Pick, - path: string, - effectiveUid = process.geteuid?.(), -): void { - if (process.platform === "win32") return; - if ((metadata.mode & 0o077) !== 0) { - throw new OutputDirectoryError( - `Scan output directory must not be accessible to other users (chmod 700): ${path}`, - ); - } - if (effectiveUid !== undefined && metadata.uid !== effectiveUid) { - throw new OutputDirectoryError( - `Scan output directory must be owned by the current user: ${path}`, - ); - } -} - -/** Reject shared parents whose owner can rename or replace private scan output. */ -export async function requireSecureOutputAncestry( - path: string, - effectiveUid = process.geteuid?.(), -): Promise { - if (process.platform === "win32") return; - let current = dirname(resolve(path)); - while (true) { - try { - current = await realpath(current); - break; - } catch (error) { - if (nodeErrorCode(error) !== "ENOENT") { - throw new OutputDirectoryError( - `Unable to inspect scan output parent directory: ${current}`, - { cause: error }, - ); - } - const parent = dirname(current); - if (parent === current) { - throw new OutputDirectoryError( - `Unable to inspect scan output parent directory: ${current}`, - { cause: error }, - ); - } - current = parent; - } - } - while (true) { - let metadata: Stats; - try { - metadata = await lstat(current); - } catch (error) { - throw new OutputDirectoryError( - `Unable to inspect scan output parent directory: ${current}`, - { cause: error }, - ); - } - if (!metadata.isDirectory()) { - throw new OutputDirectoryError( - `Scan output parent must be a non-symlink directory: ${current}`, - ); - } - requireTrustedOutputAncestor(metadata, current, effectiveUid); - const parent = dirname(current); - if (parent === current) return; - current = parent; - } -} - -export function requireTrustedOutputAncestor( - metadata: Pick, - path: string, - effectiveUid = process.geteuid?.(), -): void { - if ( - effectiveUid !== undefined && - metadata.uid !== 0 && - metadata.uid !== effectiveUid - ) { - throw new OutputDirectoryError( - `Scan output parent must have a trusted owner: ${path}`, - ); - } - if ((metadata.mode & 0o022) === 0) return; - if ((metadata.mode & 0o1000) === 0) { - throw new OutputDirectoryError( - `Scan output parent must not be group- or world-writable without the sticky bit: ${path}`, - ); - } -} - async function removeEmptyDirectories( path: string, root: string, @@ -3212,6 +3134,7 @@ export async function runCodexCommand( environment: ProcessEnvironment, input?: string | Uint8Array, signal?: AbortSignal, + cwd?: string, archiveHandshake = false, stdinPrefix?: string, ): Promise { @@ -3227,6 +3150,7 @@ export async function runCodexCommand( stdio: ["pipe", "pipe", "pipe"], windowsHide: true, signal: commandSignal, + ...(cwd === undefined ? {} : { cwd }), }, ); let stdout = ""; @@ -3586,12 +3510,6 @@ function processErrorDetail(error: unknown): string { return String(error) || "unknown error"; } -function nodeErrorCode(error: unknown): string | undefined { - return isRecord(error) && typeof error["code"] === "string" - ? error["code"] - : undefined; -} - function throwIfSignalAborted(signal?: AbortSignal): void { if (signal?.aborted) throw abortReason(signal); } diff --git a/sdk/typescript/src/scan-files.ts b/sdk/typescript/src/scan-files.ts new file mode 100644 index 0000000000..255c4bc1a7 --- /dev/null +++ b/sdk/typescript/src/scan-files.ts @@ -0,0 +1,440 @@ +import { constants, type BigIntStats, type Stats } from "node:fs"; +import { lstat, open, realpath, type FileHandle } from "node:fs/promises"; +import { dirname, isAbsolute, join, posix, resolve } from "node:path"; +import { + ContractValidationError, + OutputDirectoryError, + abortReason, +} from "./errors.js"; +import { isRecord } from "./record.js"; +import { isWithin as isContained } from "./trusted-executable.js"; +import { isWindowsUnsafePathComponent } from "./windows-path.js"; + +interface CheckedScanFile { + path: string; + metadata: Stats; + parents: Array<{ path: string; metadata: Stats }>; +} + +export interface ScanRoot { + path: string; + metadata: Stats; +} + +export async function requireScanFile( + scanDirectory: string, + relativePath: string, + context: string, + signal?: AbortSignal, +): Promise { + return ( + await requireCheckedScanFile(scanDirectory, relativePath, context, signal) + ).path; +} + +export async function readScanFile( + scanDirectory: string, + relativePath: string, + context: string, + signal?: AbortSignal, +): Promise { + const file = await openCheckedScanFile( + scanDirectory, + relativePath, + context, + signal, + ); + try { + return await file.readFile({ signal }); + } finally { + await file.close(); + } +} + +async function requireCheckedScanFile( + scanDirectory: string, + relativePath: string, + context: string, + signal?: AbortSignal, + expectedRoot?: ScanRoot, +): Promise { + const checkedRoot = await requireScanRoot(scanDirectory, signal); + const scanDir = checkedRoot.path; + throwIfAborted(signal); + const safePath = portableRelativePath(relativePath, context); + const parts = safePath.split("/"); + let current = scanDir; + try { + const rootMetadata = checkedRoot.metadata; + if ( + expectedRoot !== undefined && + (scanDir !== expectedRoot.path || + rootMetadata.dev !== expectedRoot.metadata.dev || + rootMetadata.ino !== expectedRoot.metadata.ino) + ) { + throw new Error("scan directory changed while reading"); + } + const parents = [{ path: scanDir, metadata: rootMetadata }]; + throwIfAborted(signal); + for (const part of parts.slice(0, -1)) { + current = join(current, part); + const metadata = await lstat(current); + throwIfAborted(signal); + if (!metadata.isDirectory()) { + throw new Error("unsafe parent"); + } + parents.push({ path: current, metadata }); + } + const path = join(scanDir, ...parts); + const metadata = await lstat(path); + throwIfAborted(signal); + if (!metadata.isFile()) { + throw new ContractValidationError( + `${context}: expected a regular non-symlink file.`, + ); + } + const canonical = await realpath(path); + throwIfAborted(signal); + if (!isContained(scanDir, canonical)) { + throw new Error("outside scan directory"); + } + return { path, metadata, parents }; + } catch (error) { + throwIfAborted(signal); + if (error instanceof ContractValidationError) { + throw error; + } + throw new ContractValidationError( + `${context}: expected a file inside the scan directory.`, + { + cause: error, + }, + ); + } +} + +export async function requireScanRoot( + scanDirectory: string, + signal?: AbortSignal, +): Promise { + throwIfAborted(signal); + const absolute = resolve(scanDirectory); + try { + const metadata = await lstat(absolute); + throwIfAborted(signal); + const canonical = await realpath(absolute); + throwIfAborted(signal); + const current = await lstat(absolute); + throwIfAborted(signal); + const returned = await lstat(canonical); + throwIfAborted(signal); + if ( + !metadata.isDirectory() || + !current.isDirectory() || + metadata.dev !== current.dev || + metadata.ino !== current.ino || + !returned.isDirectory() || + metadata.dev !== returned.dev || + metadata.ino !== returned.ino + ) { + throw new Error("not a directory"); + } + try { + requirePrivateOutputDirectory(returned, canonical); + await requireSecureOutputAncestry(canonical); + } catch (error) { + throw new ContractValidationError( + error instanceof Error + ? error.message + : "Scan directory must remain private to the current user.", + { cause: error }, + ); + } + return { path: canonical, metadata: returned }; + } catch (error) { + throwIfAborted(signal); + if (error instanceof ContractValidationError) throw error; + throw new ContractValidationError( + "Scan directory must be an existing non-symlink directory.", + { cause: error }, + ); + } +} + +export async function verifyScanRoot( + root: ScanRoot, + signal?: AbortSignal, +): Promise { + try { + const current = await lstat(root.path); + throwIfAborted(signal); + if ( + !current.isDirectory() || + current.dev !== root.metadata.dev || + current.ino !== root.metadata.ino + ) { + throw new Error("scan directory changed while reading"); + } + requirePrivateOutputDirectory(current, root.path); + await requireSecureOutputAncestry(root.path); + } catch (error) { + throwIfAborted(signal); + throw new ContractValidationError( + "Scan directory changed while reading the canonical contract.", + { cause: error }, + ); + } +} + +export function safeRelativePath( + value: string, + context: string, + allowDot = false, +): string { + const parts = value.split("/"); + if ( + value.length === 0 || + Buffer.from(value, "utf8").toString("utf8") !== value || + (!allowDot && value === ".") || + value.startsWith("/") || + /^[A-Za-z]:/.test(value) || + parts.includes("..") || + value.includes("\\") || + /[\u0000-\u001f]/u.test(value) + ) { + throw new ContractValidationError( + `${context}: expected a safe scan-relative POSIX path.`, + ); + } + const normalized = posix.normalize(value).replace(/\/+$/, ""); + if ( + (!allowDot && normalized === ".") || + normalized.startsWith("../") || + isAbsolute(normalized) + ) { + throw new ContractValidationError( + `${context}: expected a safe scan-relative POSIX path.`, + ); + } + return normalized; +} + +export function portableRelativePath(value: string, context: string): string { + const normalized = safeRelativePath(value, context); + if ( + value.trim().length === 0 || + value.split("/").some(isWindowsUnsafePathComponent) + ) { + throw new ContractValidationError( + `${context}: expected a safe scan-relative POSIX path.`, + ); + } + return normalized; +} + +export async function openCheckedScanFile( + scanDir: string, + relativePath: string, + context: string, + signal?: AbortSignal, + expectedRoot?: ScanRoot, +): Promise { + const checked = await requireCheckedScanFile( + scanDir, + relativePath, + context, + signal, + expectedRoot, + ); + let file: FileHandle | undefined; + try { + file = await open( + checked.path, + constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK, + ); + const opened = await file.stat(); + throwIfAborted(signal); + if ( + !opened.isFile() || + opened.ino !== checked.metadata.ino || + !(await sameCheckedFileDevice(file, checked, opened)) + ) { + throw new ContractValidationError( + `${context}: expected the checked regular file.`, + ); + } + for (const parent of checked.parents) { + const current = await lstat(parent.path); + throwIfAborted(signal); + if ( + !current.isDirectory() || + current.dev !== parent.metadata.dev || + current.ino !== parent.metadata.ino + ) { + throw new ContractValidationError( + `${context}: checked parent changed before opening the file.`, + ); + } + } + const current = await lstat(checked.path); + throwIfAborted(signal); + if ( + !current.isFile() || + current.dev !== checked.metadata.dev || + current.ino !== checked.metadata.ino + ) { + throw new ContractValidationError( + `${context}: checked file changed before reading.`, + ); + } + return file; + } catch (error) { + await file?.close(); + throwIfAborted(signal); + if (error instanceof ContractValidationError) throw error; + throw new ContractValidationError( + `${context}: unable to open the checked regular file.`, + { cause: error }, + ); + } +} + +export async function sameCheckedFileDevice( + file: FileHandle, + checked: { path: string; metadata: Pick }, + opened: Pick, + platform: NodeJS.Platform = process.platform, + openReference: (path: string, flags: number) => Promise = open, +): Promise { + if (opened.ino !== checked.metadata.ino) return false; + if (opened.dev === checked.metadata.dev) return true; + if (platform !== "win32") return false; + + const [openedIdentity, checkedIdentity] = await Promise.all([ + file.stat({ bigint: true }), + lstat(checked.path, { bigint: true }), + ]); + if ( + !openedIdentity.isFile() || + !checkedIdentity.isFile() || + openedIdentity.ino !== checkedIdentity.ino + ) { + return false; + } + + const reference = await openReference( + checked.path, + constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK, + ); + try { + const referenceIdentity = await reference.stat({ bigint: true }); + return ( + referenceIdentity.isFile() && + openedIdentity.dev === referenceIdentity.dev && + openedIdentity.ino === referenceIdentity.ino + ); + } finally { + await reference.close(); + } +} + +export function throwIfAborted(signal?: AbortSignal): void { + if (!signal?.aborted) return; + throw abortReason(signal); +} + +export function requirePrivateOutputDirectory( + metadata: Pick, + path: string, + effectiveUid = process.geteuid?.(), +): void { + if (process.platform === "win32") return; + if ((metadata.mode & 0o077) !== 0) { + throw new OutputDirectoryError( + `Scan output directory must not be accessible to other users (chmod 700): ${path}`, + ); + } + if (effectiveUid !== undefined && metadata.uid !== effectiveUid) { + throw new OutputDirectoryError( + `Scan output directory must be owned by the current user: ${path}`, + ); + } +} + +/** Reject shared parents whose owner can rename or replace private scan output. */ +export async function requireSecureOutputAncestry( + path: string, + effectiveUid = process.geteuid?.(), +): Promise { + if (process.platform === "win32") return; + let current = dirname(resolve(path)); + while (true) { + try { + current = await realpath(current); + break; + } catch (error) { + if (nodeErrorCode(error) !== "ENOENT") { + throw new OutputDirectoryError( + `Unable to inspect scan output parent directory: ${current}`, + { cause: error }, + ); + } + const parent = dirname(current); + if (parent === current) { + throw new OutputDirectoryError( + `Unable to inspect scan output parent directory: ${current}`, + { cause: error }, + ); + } + current = parent; + } + } + while (true) { + let metadata: Stats; + try { + metadata = await lstat(current); + } catch (error) { + throw new OutputDirectoryError( + `Unable to inspect scan output parent directory: ${current}`, + { cause: error }, + ); + } + if (!metadata.isDirectory()) { + throw new OutputDirectoryError( + `Scan output parent must be a non-symlink directory: ${current}`, + ); + } + requireTrustedOutputAncestor(metadata, current, effectiveUid); + const parent = dirname(current); + if (parent === current) return; + current = parent; + } +} + +export function requireTrustedOutputAncestor( + metadata: Pick, + path: string, + effectiveUid = process.geteuid?.(), +): void { + if ( + effectiveUid !== undefined && + metadata.uid !== 0 && + metadata.uid !== effectiveUid + ) { + throw new OutputDirectoryError( + `Scan output parent must have a trusted owner: ${path}`, + ); + } + if ((metadata.mode & 0o022) === 0) return; + if ((metadata.mode & 0o1000) === 0) { + throw new OutputDirectoryError( + `Scan output parent must not be group- or world-writable without the sticky bit: ${path}`, + ); + } +} + +export function nodeErrorCode(error: unknown): string | undefined { + return isRecord(error) && typeof error["code"] === "string" + ? error["code"] + : undefined; +} diff --git a/sdk/typescript/src/scan-logs.ts b/sdk/typescript/src/scan-logs.ts index 33a284760a..47f1627469 100644 --- a/sdk/typescript/src/scan-logs.ts +++ b/sdk/typescript/src/scan-logs.ts @@ -9,10 +9,14 @@ import { sessionFiles } from "./cost.js"; import { CodexSecurityError } from "./errors.js"; import type { JsonObject } from "./config.js"; import { + attributedScanThreads, + isAttributedScanEvent, isScanArtifactDirectory, + recordedScanCodexHome, sessionOwnsTurn, sessionParentThreadId, sessionStartedAt, + type ScanExecutionAttribution, } from "./scan-sessions.js"; interface ScanLogOptions { @@ -24,6 +28,7 @@ interface ScanLogOptions { scanDirectory?: string; completedAt?: string | null; allowMissingRoot?: boolean; + executionAttribution?: ScanExecutionAttribution | null; } export type ScanLogSource = JsonObject & { @@ -31,6 +36,7 @@ export type ScanLogSource = JsonObject & { continuationThreadId?: string; threadIds?: string[]; executionThreadIds?: string[]; + executionAttribution?: ScanExecutionAttribution | null; mode?: string; scanDir?: string; progress?: { status?: string; updatedAt?: string }; @@ -52,6 +58,7 @@ export function readSavedScanLogs( threadId: threadId ?? scan.threadIds?.[0], threadIds: scan.threadIds, executionThreadIds: scan.executionThreadIds ?? [], + executionAttribution: scan.executionAttribution, codexHome, allowMissingRoot: options.allowMissingRoot, scanDirectory: scan.mode === "deep" ? scan.scanDir : undefined, @@ -111,6 +118,14 @@ export async function readScanLogs(options: ScanLogOptions) { ? [options.codexHome] : options.codexHome, ); + if (options.scanDirectory !== undefined) { + const home = await recordedScanCodexHome( + options.scanDirectory, + options.executionAttribution, + ); + if (home !== undefined) homes.add(home); + } + // Recovered workers retain their original home; the parent can use the current home. for (const directory of ["sessions", "archived_sessions"]) { for (const home of homes) { for await (const session of scanSessions(home, directory)) { @@ -128,15 +143,24 @@ export async function readScanLogs(options: ScanLogOptions) { ); } - const included = new Set([ - ...(options.threadId ? [options.threadId] : []), - ...(options.threadIds ?? []), - ...(options.executionThreadIds ?? []), - ]); + const attribution = options.executionAttribution?.legacy + ? null + : options.executionAttribution; + const included = attribution + ? attributedScanThreads( + Array.from(logs.values(), ([session]) => session), + attribution, + ) + : new Set([ + ...(options.threadId ? [options.threadId] : []), + ...(options.threadIds ?? []), + ...(options.executionThreadIds ?? []), + ]); // A Desktop owner can contain other work. Include its log without treating // the whole conversation tree as part of this scan. const traversed = new Set(options.executionThreadIds ?? included); - for (const parentId of traversed) { + const pending = attribution ? [] : traversed; + for (const parentId of pending) { const parent = logs.get(parentId)?.[0]; for (const [session] of logs.values()) { if ( @@ -158,29 +182,20 @@ export async function readScanLogs(options: ScanLogOptions) { if (copies === undefined) continue; let session = copies[0]; for (const copy of copies.slice(1)) { - if (await extendsSessionLog(session.path, copy.path)) session = copy; + if ( + await extendsSessionLog(session.path, copy.path, session, attribution) + ) + session = copy; } sessions.push(session); } const events: Record[] = []; for (const session of sessions) { - let replaying = false; - for await (const event of sessionEvents(session.path)) { - const payload = event["payload"]; - if (event["type"] === "session_meta" && isRecord(payload)) { - replaying = payload["id"] !== session.threadId; - } - if (replaying) { - if ( - event["type"] !== "event_msg" || - !isRecord(payload) || - payload["type"] !== "task_started" || - !sessionOwnsTurn(session, payload) - ) { - continue; - } - replaying = false; - } + for await (const event of attributedSessionEvents( + session.path, + session, + attribution, + )) { events.push({ threadId: session.threadId, event }); } } @@ -197,6 +212,56 @@ export async function readScanLogs(options: ScanLogOptions) { }; } +async function* attributedSessionEvents( + path: string, + session: SessionLog, + attribution: ScanExecutionAttribution | null | undefined, +): AsyncGenerator> { + let replaying = false; + let turnId: string | null = null; + for await (const event of sessionEvents(path)) { + const payload = event["payload"]; + if ( + isRecord(payload) && + (event["type"] === "turn_context" || + payload["type"] === "task_started") && + typeof payload["turn_id"] === "string" + ) { + turnId = payload["turn_id"]; + } + if (event["type"] === "session_meta" && isRecord(payload)) { + replaying = payload["id"] !== session.threadId; + } + if (replaying) { + if ( + event["type"] !== "event_msg" || + !isRecord(payload) || + payload["type"] !== "task_started" || + !sessionOwnsTurn(session, payload) + ) { + continue; + } + replaying = false; + } + if ( + !attribution || + event["type"] === "session_meta" || + isAttributedScanEvent( + attribution, + session.threadId, + event["type"] === "token_usage_record" && + isRecord(payload) && + typeof payload["turn_id"] === "string" + ? payload["turn_id"] + : turnId, + event["timestamp"], + ) + ) { + yield event; + } + } +} + function belongsToScan( session: SessionLog, root: SessionLog, @@ -246,10 +311,16 @@ function belongsToScan( async function extendsSessionLog( previousPath: string, path: string, + session: SessionLog, + attribution: ScanExecutionAttribution | null | undefined, ): Promise { - const events = sessionEvents(path); + const selectedEvents = (source: string) => + attribution + ? attributedSessionEvents(source, session, attribution) + : sessionEvents(source); + const events = selectedEvents(path); try { - for await (const previous of sessionEvents(previousPath)) { + for await (const previous of selectedEvents(previousPath)) { const next = await events.next(); if (next.done || !isDeepStrictEqual(previous, next.value)) return false; } diff --git a/sdk/typescript/src/scan-sessions.ts b/sdk/typescript/src/scan-sessions.ts index b74dab0ea2..ae94a08eb6 100644 --- a/sdk/typescript/src/scan-sessions.ts +++ b/sdk/typescript/src/scan-sessions.ts @@ -1,6 +1,119 @@ +import { realpath, stat } from "node:fs/promises"; import { isAbsolute, join, relative, sep } from "node:path"; import { isRecord } from "./record.js"; +export async function recordedScanCodexHome( + _scanDirectory: string, + attribution?: ScanExecutionAttribution | null, +): Promise { + const home = attribution?.workerCodexHome; + if (home == null) return undefined; + try { + return await realpath(home); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; + throw error; + } +} + +export interface ScanExecutionAttribution { + formatVersion: 1; + legacy?: true; + workerCodexHome?: string | null; + executionThreadIds: string[]; + owner: { + threadId: string | null; + turnId: string | null; + startedAt: string; + dedicated?: boolean; + }; + startedAt: string; + completedAt: string | null; +} + +/** Reuse attribution until SQLite changes; token-log polling remains independent. */ +export function cachedScanAttributionReader( + stateDirectory: string, + read: () => Promise, +): (force?: boolean) => Promise { + const database = join(stateDirectory, "workbench.sqlite3"); + let cachedKey: string | null = null; + let cached: ScanExecutionAttribution | null | undefined; + return async (force = false) => { + const [main, wal] = await Promise.all([ + databaseSignature(database), + databaseSignature(`${database}-wal`), + ]); + const key = main === null ? null : `${main};${wal}`; + if (!force && key !== null && key === cachedKey) return cached; + const value = await read(); + // Save the pre-read signature so a concurrent transaction is read next time. + cachedKey = key; + cached = value; + return value; + }; +} + +async function databaseSignature(path: string): Promise { + try { + const metadata = await stat(path, { bigint: true }); + return `${metadata.dev}:${metadata.ino}:${metadata.size}:${metadata.mtimeNs}`; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } +} + +export function attributedScanThreads( + sessions: Iterable<{ + threadId: string | null; + parentThreadId: string | null; + }>, + attribution: ScanExecutionAttribution, +): Set { + const included = new Set(attribution.executionThreadIds); + const pending = [...included]; + const all = [...sessions]; + for (const parent of pending) { + for (const session of all) { + if ( + session.threadId !== null && + session.parentThreadId === parent && + !included.has(session.threadId) + ) { + included.add(session.threadId); + pending.push(session.threadId); + } + } + } + if (attribution.owner.threadId) included.add(attribution.owner.threadId); + return included; +} + +export function isAttributedScanEvent( + attribution: ScanExecutionAttribution, + threadId: string, + turnId: string | null, + timestamp: unknown, +): boolean { + const time = sessionStartedAt(timestamp); + if ( + time === null || + time < Date.parse(attribution.startedAt) || + (attribution.completedAt !== null && + time > Date.parse(attribution.completedAt)) + ) + return false; + if ( + threadId !== attribution.owner.threadId || + attribution.executionThreadIds.includes(threadId) + ) + return true; + return ( + attribution.owner.turnId !== null && turnId === attribution.owner.turnId + ); +} + export function sessionStartedAt(timestamp: unknown): number | null { const startedAt = typeof timestamp === "string" ? Date.parse(timestamp) : Number.NaN; diff --git a/sdk/typescript/src/selected-scan-recovery.ts b/sdk/typescript/src/selected-scan-recovery.ts new file mode 100644 index 0000000000..a7ba7dc25e --- /dev/null +++ b/sdk/typescript/src/selected-scan-recovery.ts @@ -0,0 +1,378 @@ +import { mkdtemp, realpath } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, relative, resolve, sep } from "node:path"; +import type { ScanOptions } from "./api.js"; +import type { CodexSecurityConfig, JsonObject } from "./config.js"; +import { DEFAULT_CODEX_CONFIG, scanModelConfiguration } from "./config.js"; +import type { ScanExpectation } from "./contract.js"; +import { ScanCostTracker, type ScanCost } from "./cost.js"; +import { resumeSelectedDeepScan } from "./deep-scan-finalization.js"; +import { + CodexSecurityError, + ScanCostLimitExceededError, + errorMessage, +} from "./errors.js"; +import { isRecord } from "./record.js"; +import { + expandHome, + cleanupSdkDirectory, + canonicalConfigPath, + codexSecurityCredentialHome, + codexSecurityStateDirectory, + pluginMetadata, + requireOutputOutsideRepository, + resolvePluginPath, + resolvePluginPython, + runWorkbench, + validateOutputDir, + type ProcessEnvironment, + type WorkbenchCommandOptions, +} from "./runtime.js"; +import { type ScanExecutionAttribution } from "./scan-sessions.js"; +import { + DiffTarget, + enclosingGitWorktreeRoot, + resolveRepositoryPath, + type NormalizedTarget, +} from "./targets.js"; +import type { TurnResultMetadata } from "./result.js"; + +/** Recover publication only when a saved selection outlived its startup inputs. */ +export async function recoverSelectedScanStartup(input: { + repository: string; + normalizedTarget?: NormalizedTarget; + options: ScanOptions; + config: CodexSecurityConfig; + environment: ProcessEnvironment; + signal: AbortSignal; + startupError: unknown; + workbench: typeof runWorkbench; + python: typeof resolvePluginPython; + warn: (message: string, details?: { kind: "target_changed" }) => void; + registered: ( + scan: { + scanId: string; + scanDir: string; + startedAt?: string; + }, + command: WorkbenchCommandOptions, + threadId: string, + ) => void; + cost: (cost: Readonly) => void; + collect: (context: { + scanDir: string; + threadId: string; + pluginRoot: string; + python: string; + expectation: ScanExpectation; + turnResult: TurnResultMetadata; + }) => Promise; +}): Promise { + const { options, signal } = input; + if ( + options.resumeScanId === undefined || + options.mode !== "deep" || + !options.outputDir || + options.archiveExisting || + options.parentScanId !== undefined || + options.workflowId !== undefined || + options.mock + ) + return null; + signal.throwIfAborted(); + const repository = resolveRepositoryPath(input.repository); + const boundRepository = await realpath(repository).catch( + (error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return repository; + throw error; + }, + ); + const protectedRoot = + (await enclosingGitWorktreeRoot(boundRepository, signal)) ?? + boundRepository; + const scanDir = await validateOutputDir(options.outputDir, true); + if (scanDir === null) return null; + requireOutputOutsideRepository(protectedRoot, scanDir); + const temporaryRoot = await realpath(tmpdir()); + requireOutputOutsideRepository(protectedRoot, temporaryRoot, "temporary"); + const workspace = await mkdtemp( + join(temporaryRoot, "codex-security-recovery-"), + ); + try { + const pluginRoot = await resolvePluginPath( + input.config.pluginPath, + workspace, + signal, + ); + const plugin = await pluginMetadata(pluginRoot); + const python = await input.python({ + configuredPath: input.config.pythonPath, + environment: input.environment, + protectedRoot, + signal, + }); + const stateDirectory = codexSecurityStateDirectory(input.environment); + requireOutputOutsideRepository( + protectedRoot, + await canonicalConfigPath(stateDirectory), + ); + const command: WorkbenchCommandOptions = { + python, + pluginRoot, + protectedRoot, + signal, + environment: { + ...input.environment, + CODEX_SECURITY_STATE_DIR: stateDirectory, + }, + }; + const saved = await input.workbench(command, [ + "get-cli-scan-resume", + "--scan-id", + options.resumeScanId, + ]); + if (saved["selectedFinalization"] !== true) return null; + const recipe = saved["recipe"]; + const target = isRecord(recipe) ? recipe["target"] : null; + const threadId = saved["threadId"]; + if ( + saved["scanId"] !== options.resumeScanId || + saved["scanDir"] !== scanDir || + !isRecord(recipe) || + recipe["repository"] !== boundRepository || + !isRecord(target) || + typeof threadId !== "string" || + !threadId || + typeof saved["targetRevision"] !== "string" + ) { + throw new CodexSecurityError( + "The workbench returned mismatched scan resume context.", + ); + } + // Match the caller's saved scope without consulting files that publication no longer reads. + const requested = options.target ?? "repository"; + const paths = Array.isArray(requested) + ? [ + ...new Set( + requested.map( + (path) => + relative( + boundRepository, + resolve(boundRepository, expandHome(path)), + ) + .split(sep) + .join("/") || ".", + ), + ), + ] + : []; + const matches = + input.normalizedTarget !== undefined + ? JSON.stringify(input.normalizedTarget) === JSON.stringify(target) + : requested === "repository" + ? target["kind"] === "repository" + : requested instanceof DiffTarget + ? target["kind"] === requested.kind && + [target["base"], target["baseRef"]].includes(requested.base) && + (requested.kind === "working_tree" || + [target["head"], target["headRef"]].includes(requested.head)) + : Array.isArray(requested) && + target["kind"] === "paths" && + JSON.stringify(paths) === JSON.stringify(target["paths"]); + if (!matches) + throw new CodexSecurityError( + "The workbench returned mismatched scan resume context.", + ); + if ( + options.expectedPluginVersion !== undefined && + options.expectedPluginVersion !== plugin.version + ) + throw new CodexSecurityError( + "The selected plugin version does not match the expected plugin version.", + ); + const deep = await input.workbench(command, [ + "get-deep-scan", + "--scan-id", + options.resumeScanId, + "--thread-id", + threadId, + ]); + const run = deep["deepScan"]; + if ( + !isRecord(run) || + !isRecord(run["finalizationInput"]) || + !["running", "succeeded"].includes(String(run["status"])) + ) + throw new CodexSecurityError( + "The saved Deep Scan selection is unavailable.", + ); + const model = scanModelConfiguration({ + ...DEFAULT_CODEX_CONFIG, + ...(isRecord(recipe["config"]) ? (recipe["config"] as JsonObject) : {}), + }).model; + input.registered( + { + scanId: options.resumeScanId, + scanDir, + ...(typeof saved["startedAt"] === "string" + ? { startedAt: saved["startedAt"] } + : {}), + }, + command, + threadId, + ); + let lowerBound: Readonly | null = null; + const tracker = new ScanCostTracker({ + codexHome: codexSecurityCredentialHome(input.environment), + model, + scanDirectory: scanDir, + ...(options.maxCostUsd === undefined + ? {} + : { + onCostLowerBound: (cost: Readonly) => { + lowerBound = cost; + }, + }), + }); + tracker.setAttributionReader(async () => { + const context = await input.workbench(command, [ + "get-scan", + "--scan-id", + options.resumeScanId!, + ]); + const scan = context["scan"]; + return isRecord(scan) && isRecord(scan["executionAttribution"]) + ? (scan["executionAttribution"] as unknown as ScanExecutionAttribution) + : undefined; + }); + tracker.start(threadId); + const snapshot = await tracker.stop().catch((error: unknown) => { + if (options.maxCostUsd !== undefined) throw error; + input.warn(`Could not track scan cost: ${errorMessage(error)}`); + return { usage: null, cost: null }; + }); + signal.throwIfAborted(); + if (snapshot.cost !== null) input.cost(snapshot.cost); + await resumeSelectedDeepScan({ + scanId: options.resumeScanId, + threadId, + pluginRoot, + signal, + runWorkbench: (args) => input.workbench(command, args), + }); + const budgetCost = snapshot.cost ?? lowerBound; + const budgetError = + options.maxCostUsd !== undefined && + budgetCost !== null && + budgetCost.estimatedUsd > options.maxCostUsd + ? new ScanCostLimitExceededError( + options.maxCostUsd, + budgetCost, + scanDir, + ) + : null; + const completionArgs = + budgetError === null + ? [ + "complete-scan", + "--scan-id", + options.resumeScanId, + ...(snapshot.cost === null + ? [] + : ["--cost-json", JSON.stringify(snapshot.cost)]), + ] + : [ + "complete-budget-exhausted-scan", + "--scan-id", + options.resumeScanId, + "--cost-json", + JSON.stringify(snapshot.cost ?? { lowerBound: budgetCost }), + `--message=${budgetError.message}`, + ]; + const preparation = + budgetError === null + ? await input.workbench(command, [ + "prepare-scan-completion", + "--scan-id", + options.resumeScanId, + ]) + : undefined; + const completion = await input + .workbench(command, completionArgs) + .catch(async (error: unknown) => { + const context = await input.workbench(command, [ + "get-scan", + "--scan-id", + options.resumeScanId!, + ]); + const scan = context["scan"]; + if ( + !isRecord(scan) || + !isRecord(scan["progress"]) || + scan["progress"]["status"] !== "complete" + ) + throw error; + return input.workbench(command, [ + "complete-scan", + "--scan-id", + options.resumeScanId!, + ]); + }); + const targetWarnings = new Set([ + ...(Array.isArray(preparation?.["targetWarnings"]) + ? preparation["targetWarnings"] + : []), + ...(Array.isArray(completion["targetWarnings"]) + ? completion["targetWarnings"] + : []), + ]); + const completed = completion["scan"]; + if (isRecord(completed) && Array.isArray(completed["warnings"])) { + for (const warning of completed["warnings"]) + if (typeof warning === "string") + input.warn( + warning, + targetWarnings.has(warning) + ? { kind: "target_changed" } + : undefined, + ); + } + if ( + options.maxCostUsd !== undefined && + (snapshot.cost === null || snapshot.cost.coverage === "partial") + ) + input.warn( + "Scan completed, but its cost limit could not be verified because model pricing or token usage is unavailable.", + ); + if (options.postScanPrompt?.trim() || options.postScanPromptFile) + input.warn( + `Could not run post-scan instructions: ${errorMessage(input.startupError)}`, + ); + return await input.collect({ + scanDir, + threadId, + pluginRoot, + python, + expectation: { + repository: recipe["repository"] as string, + repositoryRevision: + saved["targetRevision"] === "unversioned" + ? null + : saved["targetRevision"], + target: target as unknown as NormalizedTarget, + mode: "deep", + pluginVersion: + typeof saved["sealedProducerVersion"] === "string" + ? saved["sealedProducerVersion"] + : plugin.version, + }, + turnResult: { status: "completed", model, usage: snapshot.usage }, + }); + } finally { + await cleanupSdkDirectory(workspace).catch((error: unknown) => + input.warn( + `Could not clean up scan recovery files: ${errorMessage(error)}`, + ), + ); + } +} diff --git a/sdk/typescript/tests-ts/api-audit-admission.test.ts b/sdk/typescript/tests-ts/api-audit-admission.test.ts index d25ba02a22..d2a724ff24 100644 --- a/sdk/typescript/tests-ts/api-audit-admission.test.ts +++ b/sdk/typescript/tests-ts/api-audit-admission.test.ts @@ -3,12 +3,17 @@ import { join } from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { afterEach, expect, test } from "bun:test"; import { build } from "esbuild"; -import { createApiTestFixtures } from "./support/temporary-directories.js"; import { runScanEvents } from "../src/api.js"; import { loadContract } from "../src/contract.js"; +import { sha256 } from "./support/finding-identity.js"; +import { runWorkbench } from "../src/runtime.js"; +import { TestClient } from "./support/api-client.js"; +import { preparedRuntime } from "./support/api-events.js"; +import { runNodePython } from "./support/python-probe.js"; +import type { ScanDraftInput } from "../src/accepted-audit.js"; import { PLUGIN_ROOT, copyCompletedScan } from "./plugin-root.js"; +import { createApiTestFixtures } from "./support/temporary-directories.js"; import { completedEvents } from "./support/api-events.js"; -import { sha256 } from "./support/finding-identity.js"; const { temporaryDirectory, cleanup } = createApiTestFixtures(); afterEach(cleanup); @@ -19,6 +24,7 @@ const bundle = await build({ new URL("../../../plugins/codex-security/mcp-app/", import.meta.url), ), contents: `export * from "./src/artifact-scan-draft.ts"; + export { recordCodexSecurityScanDraft } from "./tests/scan-draft-writer-fixture.ts"; export * from "./src/deep-scan/artifacts.ts"; export * from "./src/deep-scan/artifact-validation.ts"; export * from "./src/deep-scan/worker-runner.ts";`, @@ -33,21 +39,13 @@ const bundlePath = join(await temporaryDirectory(), "deep-admission.mjs"); await writeFile(bundlePath, bundle.outputFiles[0]!.contents); const { createDeepScanArtifacts, - recordCodexSecurityScanDraftViaWorkbench, parseCanonicalScanDraft, - validateDiscoveryArtifacts, + recordCodexSecurityScanDraft, + recordCodexSecurityScanDraftViaWorkbench, + readDiscoveryAuditDraft, DeepScanWorkerRunner, } = await import(pathToFileURL(bundlePath).href); -interface DraftFixture { - scanId: string; - complete?: boolean; - scope?: Record; - threatModel?: Record; - findings: Record[]; - coverage: Record; -} - const scanId = "811aef98-3709-4c2d-8b7a-742977521865"; type Mutation = | "missing-findings" @@ -61,6 +59,7 @@ const cases: { complete?: boolean; mutation?: Mutation; accepted: boolean; + canonicalAccepted?: boolean; }[] = [ { name: "scope notes", complete: true, accepted: true }, { @@ -75,7 +74,12 @@ const cases: { mutation: "legacy-details", accepted: false, }, - { name: "legacy surface ID", mutation: "legacy-surface", accepted: false }, + { + name: "legacy surface ID", + mutation: "legacy-surface", + accepted: false, + canonicalAccepted: true, + }, { name: "unfinished checkpoint", complete: false, accepted: false }, { name: "missing findings", mutation: "missing-findings", accepted: false }, { @@ -101,7 +105,7 @@ for (const scenario of cases) { mkdir(standardRoot, { mode: 0o700 }), mkdir(deepRoot, { mode: 0o700 }), ]); - const semantic: DraftFixture = { + const semantic: ScanDraftInput = { scanId, ...(scenario.complete === undefined ? {} @@ -135,7 +139,7 @@ for (const scenario of cases) { deferred: [{ reason: "Deployment controls remain unverified." }], }, }; - await recordCodexSecurityScanDraftViaWorkbench( + await recordCodexSecurityScanDraft( { root: standardRoot, repoRoot: repository, @@ -156,20 +160,6 @@ for (const scenario of cases) { }, }, semantic, - async (args: string[]) => { - const draft = JSON.parse( - await readFile(args[args.indexOf("--draft-path") + 1]!, "utf8"), - ); - await Promise.all( - [ - ["manifest", "scan-manifest.json"], - ["findings", "findings.json"], - ["coverage", "coverage.json"], - ].map(([key, name]) => - writeFile(join(standardRoot, name!), JSON.stringify(draft[key!])), - ), - ); - }, ); const submitted = mutateDraft(semantic, scenario.mutation); const findings = { @@ -245,12 +235,19 @@ for (const scenario of cases) { "discovery-1", ); expect(executions).toBe(1); - if (scenario.accepted) { + if (scenario.canonicalAccepted ?? scenario.accepted) { expect(standard.error).toBe(standard.finalization); expect(standard.finalizations).toBe(1); + expect(standard.drafts).toHaveLength(1); + } else { + expect(standard.error).toBeInstanceOf(Error); + expect(standard.error).not.toBe(standard.finalization); + expect(standard.finalizations).toBe(0); + } + if (scenario.accepted) { expect(deepResult.status).toBe("succeeded"); expect(acceptedPaths).toEqual([deepResult.worker.resultPath]); - const deepDraft: DraftFixture = await validateDiscoveryArtifacts( + const deepDraft: ScanDraftInput = await readDiscoveryAuditDraft( artifacts, deepResult.worker.resultPath, scanId, @@ -268,9 +265,6 @@ for (const scenario of cases) { expect(deepDraft.scope).toEqual(canonicalDraft.scope); expect(deepDraft.threatModel).toEqual(canonicalDraft.threatModel); } else { - expect(standard.error).toBeInstanceOf(Error); - expect(standard.error).not.toBe(standard.finalization); - expect(standard.finalizations).toBe(0); expect(deepResult.status).toBe("failed"); expect(acceptedPaths).toHaveLength(0); } @@ -294,8 +288,77 @@ test("Standard admission preserves existing canonical scan IDs", async () => { expect(manifest.scan.id).toBe("scan_example_001"); expect(standard.error).toBe(standard.finalization); expect(standard.finalizations).toBe(1); + expect(standard.drafts).toHaveLength(1); + expect(standard.drafts[0]!.scanId).toBe(manifest.scan.id); }); +test.each(["http", "HTTP", "HTTP API"])( + "Standard admission preserves file-authored canonical surface ID %s", + async (surfaceId) => { + const root = await temporaryDirectory(); + const repository = join(root, "repository"); + await mkdir(repository); + const scanDir = await copyCompletedScan(root); + const coveragePath = join(scanDir, "coverage.json"); + const coverage = JSON.parse(await readFile(coveragePath, "utf8")); + coverage.surfaces[0].id = surfaceId; + await writeFile(coveragePath, JSON.stringify(coverage)); + const standard = await observeStandardAdmission(repository, scanDir); + expect(standard.error).toBe(standard.finalization); + expect(standard.finalizations).toBe(1); + expect( + (standard.drafts[0]!.coverage["surfaces"] as { id: string }[])[0]!.id, + ).toBe(surfaceId); + expect(JSON.parse(await readFile(coveragePath, "utf8"))).toEqual(coverage); + }, +); + +test.each(["current", "legacy"])( + "Standard admission retains selected %s plugin helper compatibility", + async (kind) => { + const root = await temporaryDirectory(); + const repository = join(root, "repository"); + await mkdir(repository); + const scanDir = await copyCompletedScan(root); + const pluginRoot = join(root, "selected-plugin"); + await mkdir(join(pluginRoot, "mcp"), { recursive: true }); + await writeFile( + join(pluginRoot, "mcp", "helpers.mjs"), + kind === "legacy" + ? "process.exitCode = 2;\n" + : `export { default } from ${JSON.stringify(pathToFileURL(join(PLUGIN_ROOT, "mcp", "helpers.mjs")).href)};\n`, + ); + const finalization = new Error("Finalization receives the admitted scan."); + let finalizations = 0; + const previousExitCode = process.exitCode; + try { + const error = await runScanEvents({ + thread: { id: "standard-thread" }, + events: completedEvents("standard-thread"), + signal: new AbortController().signal, + scanDir, + pluginRoot, + expectation: { + repository, + repositoryRevision: null, + target: { kind: "repository", paths: [] }, + mode: "standard", + pluginVersion: "0.1.95", + }, + onFinalize: async () => { + finalizations++; + throw finalization; + }, + }).catch((error: unknown) => error); + expect(error).toBe(finalization); + expect(finalizations).toBe(1); + expect(process.exitCode).toBe(previousExitCode); + } finally { + process.exitCode = previousExitCode; + } + }, +); + test.each(["scan-manifest.json", "findings.json", "coverage.json"])( "Standard admission preserves BOM-prefixed %s bytes", async (name) => { @@ -389,22 +452,102 @@ test("canonical admission still rejects reversed typed evidence bounds", async ( expect(outcome.finalizations).toBe(0); }); +test.each(["standard", "deep"] as const)( + "managed %s preparation without accepted authority rejects historical findings", + async (mode) => { + const scanDir = await copyCompletedScan(await temporaryDirectory()); + const file = join(scanDir, "findings.json"); + const findings = JSON.parse(await readFile(file, "utf8")); + findings.findings[0].root_cause = { summary: "Unaccepted legacy detail." }; + const original = JSON.stringify(findings); + await writeFile(file, original); + let preparations = 0; + await expect( + runScanEvents({ + thread: { id: "synthetic-thread" }, + events: completedEvents("synthetic-thread"), + signal: new AbortController().signal, + scanDir, + pluginRoot: PLUGIN_ROOT, + expectation: { + repository: "/repository", + repositoryRevision: "deadbeef", + target: { kind: "repository", paths: [] }, + mode, + pluginVersion: "0.1.0", + }, + workbenchValidated: true, + acceptedFindings: () => false, + onFinalize: async () => { + preparations++; + return null; + }, + }), + ).rejects.toThrow("root_cause"); + expect(preparations).toBe(1); + expect(await readFile(file, "utf8")).toBe(original); + }, +); + +test("recovered admission does not repeat successful managed finalization", async () => { + const scanDir = await copyCompletedScan(await temporaryDirectory()); + const file = join(scanDir, "findings.json"); + const saved = await readFile(file); + await writeFile(file, "interrupted synthetic artifact"); + const finalizedUsage = { input_tokens: 13, output_tokens: 5 }; + let finalizations = 0; + let recoveries = 0; + const result = await runScanEvents({ + thread: { id: "standard-thread" }, + events: completedEvents("standard-thread"), + signal: new AbortController().signal, + scanDir, + pluginRoot: PLUGIN_ROOT, + expectation: { + repository: "/repository", + repositoryRevision: "deadbeef", + target: { kind: "repository", paths: [] }, + mode: "standard", + pluginVersion: "0.1.0", + }, + workbenchValidated: true, + onFinalize: async () => { + finalizations++; + return finalizedUsage; + }, + recoverCompletion: async () => { + recoveries++; + await writeFile(file, saved); + return { + threadId: "standard-thread", + status: "completed", + finalResponse: "Recovered synthetic scan.", + usage: null, + lastStreamError: null, + }; + }, + }); + expect(recoveries).toBe(1); + expect(finalizations).toBe(1); + expect(result.turnResult.usage).toEqual(finalizedUsage); +}); + async function observeStandardAdmission( repository: string, scanDir: string, scanId?: string, ) { + const drafts: ScanDraftInput[] = []; const finalization = new Error("The enclosing finalizer owns the next step."); let finalizations = 0; - const thread = { - id: "standard-thread", - async runStreamed() { - return { events: completedEvents("standard-thread") }; - }, - }; const error = await runScanEvents({ scanId, - thread, + thread: { + id: "standard-thread", + async runStreamed() { + return { events: completedEvents("standard-thread") }; + }, + }, events: completedEvents("standard-thread"), signal: new AbortController().signal, scanDir, @@ -418,6 +561,24 @@ async function observeStandardAdmission( }, onFinalize: async () => { finalizations++; + const [manifest, findings, coverage] = await Promise.all( + ["scan-manifest.json", "findings.json", "coverage.json"].map( + async (name) => + JSON.parse( + new TextDecoder("utf-8", { fatal: true }).decode( + await readFile(join(scanDir, name)), + ), + ), + ), + ); + drafts.push( + parseCanonicalScanDraft({ + scanId: scanId ?? manifest.scan.id, + manifest, + findings, + coverage, + }), + ); throw finalization; }, }).catch((error: unknown) => error); @@ -425,13 +586,17 @@ async function observeStandardAdmission( error, finalization, finalizations, + drafts, }; } -function mutateDraft(input: DraftFixture, mutation?: Mutation): DraftFixture { +function mutateDraft( + input: ScanDraftInput, + mutation?: Mutation, +): ScanDraftInput { const draft = structuredClone(input); if (mutation === "missing-findings") - return { ...draft, findings: undefined } as unknown as DraftFixture; + return { ...draft, findings: undefined } as unknown as ScanDraftInput; if (mutation === "wrong-scan") draft.scanId = "553a0c18-dcdf-4a3b-8e39-2751a8187bce"; if (mutation === "inverted-lines") { @@ -451,3 +616,207 @@ function mutateDraft(input: DraftFixture, mutation?: Mutation): DraftFixture { }; return draft; } + +for (const scenario of ["published", "interrupted", "unfinished"] as const) { + test(`Standard admission reconciles its committed terminal checkpoint: ${scenario}`, async () => { + const root = await temporaryDirectory(); + const repository = join(root, "repository"); + const home = join(root, "codex-home"); + const scanDir = join(root, "scan"); + await mkdir(repository); + await writeFile(join(repository, "extract.py"), "# Synthetic source\n"); + await mkdir(home); + await mkdir(scanDir, { mode: 0o700 }); + const environment = { + ...process.env, + CODEX_HOME: home, + CODEX_SECURITY_STATE_DIR: join(root, "state"), + }; + const python = process.env["PYTHON"] ?? Bun.which("python3")!; + let registration: Record; + let savedOptions: Parameters[0]; + const commands: string[] = []; + const runtime = preparedRuntime(home); + runtime.plugin.version = JSON.parse( + await readFile(join(PLUGIN_ROOT, ".codex-plugin", "plugin.json"), "utf8"), + ).version; + const client = new TestClient( + {}, + { + environment, + prepareRuntime: async () => ({ ...runtime, environment }), + resolvePluginPython: async () => python, + prepareOutputDir: async () => scanDir, + runWorkbench: async (options, args, input) => { + savedOptions = options; + commands.push(args[0]!); + const value = await runWorkbench(options, args, input); + if (args[0] === "register-cli-scan") registration = value; + return value; + }, + createCodex: () => ({ + startThread: () => ({ + id: null, + async runStreamed() { + const id = String(registration!["scanId"]); + const context = { + root: scanDir, + repoRoot: repository, + layout: "scan", + scanId: id, + mode: "standard", + status: "running", + scope: ".", + targetContract: registration!["contract"], + }; + const draft: ScanDraftInput = { + scanId: id, + complete: false, + scope: { summary: "Archive extraction." }, + threatModel: { + summary: "An untrusted caller supplies archive entries.", + }, + findings: [ + { + ruleId: "path-traversal.archive", + title: "Unsafe archive extraction", + summary: "An archive entry reaches a filesystem write.", + severity: { level: "high" }, + confidence: { level: "high", rationale: "Source review." }, + taxonomy: { category: "path-traversal", cwe: ["CWE-22"] }, + locations: [ + { path: "extract.py", startLine: 1, endLine: 1 }, + ], + remediation: + "Validate the resolved output path before writing.", + provenance: { + source: "local_plugin", + candidateId: "archive-entry", + }, + }, + ], + coverage: { + completeness: "partial", + surfaces: [ + { + id: "archive", + label: "Archive extraction", + disposition: "reported", + }, + ], + explicitExclusions: [], + deferred: [ + { + id: "deployment", + reason: "Deployment controls remain unverified.", + }, + ], + }, + }; + const publish = (args: readonly string[]) => + runWorkbench(savedOptions!, args); + await recordCodexSecurityScanDraftViaWorkbench( + context, + draft, + publish, + ); + if (scenario !== "unfinished") { + const terminal = { ...draft, complete: true }; + if (scenario === "interrupted") { + await expect( + recordCodexSecurityScanDraftViaWorkbench( + context, + terminal, + async (args: readonly string[]) => { + const result = runNodePython( + python, + [ + "-c", + [ + "import sys", + "sys.path.insert(0, sys.argv[1])", + "import workbench_db as db", + "original = db.saved_results.write_scan_local_bytes", + "def fail_manifest(root, relative, contents):", + " if relative == 'scan-manifest.json': raise OSError('Synthetic interrupted canonical manifest publication')", + " return original(root, relative, contents)", + "db.saved_results.write_scan_local_bytes = fail_manifest", + "sys.argv = ['workbench_db.py', *sys.argv[2:]]", + "db.main()", + ].join("\n"), + join(PLUGIN_ROOT, "scripts"), + ...args, + ], + { + env: { + ...process.env, + ...savedOptions!.environment, + }, + }, + ); + expect(result.status).toBe(1); + expect(result.stderr).toContain( + "Synthetic interrupted canonical manifest publication", + ); + throw new Error( + "Synthetic interrupted canonical manifest publication", + ); + }, + ), + ).rejects.toThrow( + "Synthetic interrupted canonical manifest publication", + ); + const head = JSON.parse( + await readFile( + join(scanDir, "checkpoint-head.json"), + "utf8", + ), + ); + const checkpoint = JSON.parse( + await readFile( + join(scanDir, "checkpoints", head.checkpoint), + "utf8", + ), + ); + expect(checkpoint.complete).not.toBe(false); + expect( + JSON.parse( + await readFile( + join(scanDir, "scan-manifest.json"), + "utf8", + ), + ).scan.complete, + ).toBe(false); + } else + await recordCodexSecurityScanDraftViaWorkbench( + context, + terminal, + publish, + ); + } + return { events: completedEvents("standard-checkpoint-thread") }; + }, + }), + }), + }, + ); + try { + const pending = client.run(repository); + if (scenario === "unfinished") { + await expect(pending).rejects.toThrow(); + expect(commands).not.toContain("complete-scan"); + } else { + const result = await pending; + expect(result.findings.findings).toHaveLength(1); + expect(result.findings.findings[0]!.title).toBe( + "Unsafe archive extraction", + ); + expect(commands).toContain("prepare-scan-completion"); + expect(commands).toContain("complete-scan"); + expect(commands).not.toContain("fail-scan"); + } + } finally { + await client.close(); + } + }); +} diff --git a/sdk/typescript/tests-ts/api-policy.test.ts b/sdk/typescript/tests-ts/api-policy.test.ts index 5b6f30489b..36adb74068 100644 --- a/sdk/typescript/tests-ts/api-policy.test.ts +++ b/sdk/typescript/tests-ts/api-policy.test.ts @@ -1806,6 +1806,93 @@ describe("CodexSecurity policy API", () => { await f.security.close(); }); + test.each(["architecture", "threat_model"] as const)( + "enforces priced policy usage despite an unpriced remainder in %s", + async (crossingStage) => { + const costs: number[] = []; + const f = await setup({ + config: { codexOverrides: { model: "gpt-5.6-sol" } }, + stream: async function* (stage, signal) { + if (stage !== crossingStage) { + yield* events(stage); + return; + } + const directory = join(f.root, "codex-home", "sessions"); + await mkdir(directory, { recursive: true }); + const thread = `policy-${stage}`; + const input = stage === "architecture" ? 1_200 : 1_000; + for (const [id, model, parent, inputTokens] of [ + [thread, "gpt-5.6-sol", undefined, input], + ["unpriced-policy-worker", "synthetic-unpriced-model", thread, 100], + ] as const) { + await writeFile( + join(directory, `${id}.jsonl`), + [ + JSON.stringify({ + type: "session_meta", + payload: { + id, + ...(parent === undefined + ? {} + : { parent_thread_id: parent }), + }, + }), + JSON.stringify({ type: "turn_context", payload: { model } }), + JSON.stringify({ + type: "event_msg", + payload: { + type: "token_count", + info: { + total_token_usage: { + input_tokens: inputTokens, + output_tokens: 0, + }, + }, + }, + }), + "", + ].join("\n"), + ); + } + yield { type: "thread.started", thread_id: thread }; + await new Promise((resolve) => { + if (signal.aborted) resolve(); + else + signal.addEventListener("abort", () => resolve(), { once: true }); + }); + throw signal.reason; + }, + }); + const keepAlive = setTimeout(() => {}, 10_000); + try { + await expect( + f.security.generatePolicy(f.repository, { + outputDir: f.outputDir, + maxCostUsd: 0.0045, + signal: AbortSignal.timeout(5_000), + onCost: (cost) => costs.push(cost.estimatedUsd), + }), + ).rejects.toThrow("exceeded its $0.0045 cost limit"); + expect(f.threads).toHaveLength( + crossingStage === "architecture" ? 1 : 2, + ); + if (crossingStage === "architecture") expect(costs).toEqual([]); + else { + expect(costs.length).toBeGreaterThan(0); + for (const cost of costs) expect(cost).toBeCloseTo(0.0006, 12); + } + if (crossingStage === "threat_model") { + expect( + await readFile(join(f.outputDir, "project-spec.md"), "utf8"), + ).toContain("src/service.ts:1"); + } + } finally { + clearTimeout(keepAlive); + await f.security.close(); + } + }, + ); + test("enforces one cost budget across stages and preserves completed evidence", async () => { const f = await setup(); await expect( diff --git a/sdk/typescript/tests-ts/api-preflight-config.test.ts b/sdk/typescript/tests-ts/api-preflight-config.test.ts index 7d1d19392b..f37bd159cc 100644 --- a/sdk/typescript/tests-ts/api-preflight-config.test.ts +++ b/sdk/typescript/tests-ts/api-preflight-config.test.ts @@ -15,6 +15,8 @@ import { type JsonObject, } from "../src/config.js"; import { TestClient } from "./support/api-client.js"; +import { preparedRuntime } from "./support/api-events.js"; +import { parse as parseToml } from "smol-toml"; import { createApiTestFixtures } from "./support/temporary-directories.js"; const { temporaryDirectory, cleanup } = createApiTestFixtures( @@ -630,7 +632,7 @@ describe("CodexSecurity preflight configuration", () => { }); expect(config).toEqual({ - model_provider: "openai", + model_provider: provider, profile: "selected", profiles: { selected: { model, model_provider: provider } }, model_providers: { [provider]: providerConfig }, @@ -680,7 +682,7 @@ describe("CodexSecurity preflight configuration", () => { }); expect(config).toEqual({ - model_provider: "openai", + model_provider: "amazon-bedrock", profile: "bedrock", profiles: { bedrock: { @@ -699,3 +701,63 @@ describe("CodexSecurity preflight configuration", () => { }, ); }); + +test.each(["cloud.production", "selected", undefined] as const)( + "saved Deep runtime retains the resolved provider for profile %s", + async (profile) => { + const root = await temporaryDirectory(); + const repository = join(root, "repository"); + const home = join(root, "home"); + const scanDir = join(root, "scan"); + const configPath = join(home, "config-preflight.toml"); + const deepScanConfigPath = join(home, "deep-scan.toml"); + await Promise.all([ + mkdir(repository), + mkdir(home), + mkdir(scanDir, { mode: 0o700 }), + ]); + const expected = profile === undefined ? "openai" : "openrouter"; + const client = new TestClient( + { + codexOverrides: { + model_provider: "openai", + model_reasoning_summary: "none", + ...(profile === undefined ? {} : { profile }), + profiles: { + [profile ?? "unselected"]: { model_provider: "openrouter" }, + }, + }, + }, + { + environment: { + OPENAI_API_KEY: "synthetic-parent-key", + OPENROUTER_API_KEY: "synthetic-provider-key", + }, + prepareRuntime: async () => ({ + ...preparedRuntime(home), + configPath, + deepScanConfigPath, + }), + resolvePluginPython: async () => "/managed/python", + prepareOutputDir: async () => scanDir, + repositoryRevision: async () => "deadbeef", + createCodex: () => { + throw new Error("Synthetic parent boundary reached"); + }, + }, + ); + try { + await expect(client.run(repository, { mode: "deep" })).rejects.toThrow( + "Synthetic parent boundary reached", + ); + const saved = parseToml(await readFile(configPath, "utf8")) as JsonObject; + expect(scanModelProvider(saved)).toBe(expected); + const worker = parseToml(await readFile(deepScanConfigPath, "utf8")); + expect((worker["worker_runtime"] as JsonObject)["model_provider"]).toBe( + expected, + ); + } finally { + await client.close(); + } + }, +); diff --git a/sdk/typescript/tests-ts/api-usage-finalization.test.ts b/sdk/typescript/tests-ts/api-usage-finalization.test.ts index 685cbfa811..8cfc0d122d 100644 --- a/sdk/typescript/tests-ts/api-usage-finalization.test.ts +++ b/sdk/typescript/tests-ts/api-usage-finalization.test.ts @@ -216,6 +216,9 @@ test.each([ } if (args[0] === "prepare-scan-completion") throw originalFailure; if (args[0] === "get-scan") { + if (scenario === "resumed" && !turnExited) { + return { scan: { progress: { status: "running" } } }; + } expect(turnExited).toBe(true); if (scenario !== "post-turn-error") expect(options.signal).toBeUndefined(); diff --git a/sdk/typescript/tests-ts/api.test.ts b/sdk/typescript/tests-ts/api.test.ts index a83d5fb315..23d81e0f84 100644 --- a/sdk/typescript/tests-ts/api.test.ts +++ b/sdk/typescript/tests-ts/api.test.ts @@ -1908,7 +1908,7 @@ describe("CodexSecurity orchestration", () => { expect(result.cost).toMatchObject({ model: "openai.gpt-5.6-luna" }); expect(savedRecipe).toMatchObject({ config: { - model_provider: "openai", + model_provider: "amazon-bedrock", profile: "bedrock", profiles: { bedrock: { @@ -3356,7 +3356,7 @@ describe("CodexSecurity orchestration", () => { ).toBeUndefined(); expect(prompt).toContain("The SDK has already registered this scan."); expect(prompt).toContain("never call a scan-start or completion tool"); - expect(prompt).toContain("do not finalize or seal them"); + expect(prompt).toContain("leave finalization to the SDK"); expect(prompt).toContain( "This Standard scan authorizes its independent baseline auditor and focused investigators", ); @@ -6082,6 +6082,252 @@ describe("CodexSecurity orchestration", () => { }, ); + test.each([ + ["standard", false], + ["deep", false], + ["standard", true], + ["deep", true], + ] as const)( + "enforces priced usage with an unpriced remainder (%s, raised limit: %p)", + async (mode, raised) => { + const root = await temporaryDirectory(); + const repository = join(root, "repository"); + const codexHome = join(root, "codex-home"); + const scanDir = join(root, "scan"); + await Promise.all([ + mkdir(repository), + mkdir(codexHome), + mkdir(scanDir, { mode: 0o700 }), + ]); + const commands: Array = []; + const costs: number[] = []; + let turns = 0; + let releaseIncrease!: () => void; + const increased = new Promise((resolve) => { + releaseIncrease = resolve; + }); + const knownUsage = { + input_tokens: raised ? 2_500 : 1_250, + cached_input_tokens: 200, + output_tokens: 30, + }; + const expectedCost = estimateScanCost("gpt-5.6-sol", knownUsage)!; + const writePricedUsage = async (usage: Record) => { + const path = await writeUsageSession(codexHome, "scan-thread", usage); + const lines = (await readFile(path, "utf8")).split("\n"); + lines.splice( + 1, + 0, + JSON.stringify({ + type: "turn_context", + payload: { model: "gpt-5.6-sol" }, + }), + ); + await writeFile(path, lines.join("\n")); + }; + const writeUnpricedUsage = async () => { + const path = await writeUsageSession( + codexHome, + "unpriced-worker", + { input_tokens: 100, output_tokens: 10 }, + { parent: "scan-thread" }, + ); + const lines = (await readFile(path, "utf8")).split("\n"); + lines.splice( + 1, + 0, + JSON.stringify({ + type: "turn_context", + payload: { model: "synthetic-unpriced-model" }, + }), + ); + await writeFile(path, lines.join("\n")); + }; + const client = new TestClient( + {}, + { + environment: {}, + prepareRuntime: async () => preparedRuntime(codexHome), + resolvePluginPython: async () => "/managed/python", + prepareOutputDir: async () => scanDir, + repositoryRevision: async () => "deadbeef", + runWorkbench: async (_options, args, input) => { + commands.push(args); + if (args[0] === "get-scan") + return { scan: { id: "scan_example_001" } }; + if (args[0] === "complete-budget-exhausted-scan") + throw new Error("Synthetic canonical output is not ready"); + return mockWorkbench(args, input); + }, + createCodex: () => ({ + startThread: () => ({ + id: null, + async runStreamed( + _input: string, + options: { signal: AbortSignal }, + ) { + turns++; + async function* events(): AsyncGenerator { + if (raised) { + await writePricedUsage({ + input_tokens: 800, + output_tokens: 0, + }); + } else { + await writePricedUsage(knownUsage); + await writeUnpricedUsage(); + } + yield { type: "thread.started", thread_id: "scan-thread" }; + if (raised) { + await increased; + await writeUnpricedUsage(); + const path = join( + codexHome, + "sessions", + "2026", + "07", + "26", + "rollout-scan-thread.jsonl", + ); + await appendFile( + path, + JSON.stringify({ + type: "event_msg", + payload: { + type: "token_count", + info: { total_token_usage: knownUsage }, + }, + }) + "\n", + ); + } + await new Promise((resolve) => { + if (options.signal.aborted) resolve(); + else + options.signal.addEventListener( + "abort", + () => resolve(), + { once: true }, + ); + }); + throw new DOMException("aborted", "AbortError"); + } + return { events: events() }; + }, + }), + }), + }, + ); + const keepAlive = setTimeout(() => {}, 10_000); + try { + const failure = await client + .run(repository, { + mode, + maxCostUsd: 0.004, + signal: AbortSignal.timeout(5_000), + postScanPrompt: "No model work after the budget stop.", + ...(raised ? { onBudgetApproaching: () => 0.008 } : {}), + onCost: (cost, limit) => { + costs.push(cost.estimatedUsd); + if (limit === 0.008) releaseIncrease(); + }, + }) + .catch((error: unknown) => error); + expect(failure).toBeInstanceOf(ScanCostLimitExceededError); + expect(failure).toMatchObject({ + maxCostUsd: raised ? 0.008 : 0.004, + cost: { + estimatedUsd: expectedCost.estimatedUsd, + inputTokens: knownUsage.input_tokens, + coverage: "partial", + }, + }); + expect(costs.every((cost) => raised && cost === 0.0032)).toBe(true); + expect(turns).toBe(1); + expect( + commands.some((args) => args[0] === "complete-budget-exhausted-scan"), + ).toBe(mode === "deep"); + if (raised) + expect( + commands + .filter((args) => args[0] === "set-scan-cost-limit") + .map((args) => args.at(-1)), + ).toEqual(["0.008"]); + } finally { + clearTimeout(keepAlive); + await client.close(); + } + }, + ); + + test("enforces known Standard cost while a delegated child has only metadata", async () => { + const root = await temporaryDirectory(); + const repository = join(root, "repository"); + const codexHome = join(root, "codex-home"); + const scanDir = join(root, "scan"); + await Promise.all([ + mkdir(repository), + mkdir(codexHome), + mkdir(scanDir, { mode: 0o700 }), + ]); + const commands: string[] = []; + const runStreamed = mock( + async (_input: string, options: { signal: AbortSignal }) => { + async function* events(): AsyncGenerator { + await writeUsageSession(codexHome, "scan-thread", { + input_tokens: 1_250, + cached_input_tokens: 200, + output_tokens: 30, + }); + const child = await writeUsageSession( + codexHome, + "worker-thread", + {}, + { + parent: "scan-thread", + parentField: "source", + }, + ); + // Native session metadata is persisted before the first usage record. + const metadata = (await readFile(child, "utf8")).split("\n")[0]!; + await writeFile(child, metadata + "\n"); + yield { type: "thread.started", thread_id: "scan-thread" }; + await (options.signal.aborted + ? undefined + : once(options.signal, "abort")); + throw new DOMException("aborted", "AbortError"); + } + return { events: events() }; + }, + ); + const client = TestClient.withDependencies({ + ...scanRuntimeDependencies(codexHome, scanDir), + runWorkbench: async (_options, args, input) => { + commands.push(args[0]!); + return mockWorkbench(args, input); + }, + createCodex: codexFactory(runStreamed), + }); + const keepEventLoopAlive = setTimeout(() => {}, 10_000); + try { + const failure = await client + .run(repository, { + mode: "standard", + maxCostUsd: 0.004, + signal: AbortSignal.timeout(5_000), + }) + .catch((error: unknown) => error); + expect(failure).toBeInstanceOf(ScanCostLimitExceededError); + expect(failure).toMatchObject({ + cost: { estimatedUsd: 0.00488, coverage: "partial" }, + }); + expect(commands).not.toContain("complete-scan"); + expect(runStreamed).toHaveBeenCalledTimes(1); + } finally { + clearTimeout(keepEventLoopAlive); + await client.close(); + } + }); + test("stops and records a scan as soon as its live cost exceeds the limit", async () => { const { repository, codexHome, scanDir } = await scanDirectories(); const commands: Array = []; @@ -6205,6 +6451,10 @@ describe("CodexSecurity orchestration", () => { input?: string, ): Promise => { commands.push(args); + if (args[0] === "get-scan") { + // Older workbench readers return a scan without execution attribution. + return { scan: { id: "scan_example_001" } }; + } if (args[0] !== "complete-budget-exhausted-scan") { return mockWorkbench(args, input); } @@ -6561,7 +6811,7 @@ describe("CodexSecurity orchestration", () => { "--scan-id", expect.any(String), ]); - expect(commands[2]).toMatchObject([ + expect(commands.find((args) => args[0] === "fail-scan")).toMatchObject([ "fail-scan", "--scan-id", expect.stringMatching(/^[0-9a-f-]{36}$/), @@ -7787,7 +8037,8 @@ describe("CodexSecurity orchestration", () => { ); expect(prompt()).not.toContain("$codex-security:"); expect(prompt()).not.toContain("record_codex_security_scan_draft"); - expect(prompt()).toContain("complete_codex_security_scan"); + expect(prompt()).not.toContain("complete_codex_security_scan"); + expect(prompt()).toContain("Leave finalization to the SDK after this turn"); expect(prompt()).not.toContain("do not finalize or seal them"); expect(prompt()).toContain( 'start_codex_security_deep_scan with {"scanId":"scan_example_001"}', @@ -8660,9 +8911,14 @@ if ([basename(process.argv[1]), ...process.argv.slice(2)].join(" ") !== "login s expect(scanSignal?.aborted).toBe(false); }); - test.each(["standard", "deep"] as const)( - "isolates concurrent managed %s sessions at the Codex child boundary", - async (mode) => { + test.each([ + ["standard", false, false], + ["deep", false, false], + ["deep", false, true], + ["deep", true, true], + ] as const)( + "isolates concurrent managed %s sessions at the Codex child boundary (capture=%p, program=%p)", + async (mode, captureSummary, selectedProgram) => { const clients: TestClient[] = []; try { const outcomes = await Promise.allSettled( @@ -8678,12 +8934,17 @@ if ([basename(process.argv[1]), ...process.argv.slice(2)].join(" ") !== "login s mkdir(codexHome), mkdir(scanDir, { mode: 0o700 }), ]); + const model = `fixture-${name}-model`; + const summary = name === "first" ? "none" : "concise"; + const configPath = join(codexHome, "config-preflight.toml"); + let recipeConfig: JsonObject | undefined; await writeFile( preload, [ 'import { appendFileSync } from "node:fs";', 'let prompt = ""; for await (const chunk of process.stdin) prompt += chunk;', - `appendFileSync(${JSON.stringify(marker)}, JSON.stringify({args:process.argv, executable:process.execPath, home:process.env.CODEX_HOME, key:process.env.CODEX_API_KEY, value:process.env.FIXTURE_SCAN_VALUE, prompt}) + "\\n");`, + `appendFileSync(${JSON.stringify(marker)}, JSON.stringify({args:process.argv, executable:process.execPath, cwd:process.cwd(), home:process.env.CODEX_HOME, key:process.env.CODEX_API_KEY, value:process.env.FIXTURE_SCAN_VALUE, prompt}) + "\\n");`, + `if (process.argv.includes("models")) { console.log(JSON.stringify({models:[{slug:${JSON.stringify(model)},default_reasoning_summary:${JSON.stringify(summary)}}]})); process.exit(0); }`, `console.log(JSON.stringify({type:"thread.started",thread_id:${JSON.stringify(`fixture-${name}-thread`)}}));`, 'console.log(JSON.stringify({type:"item.completed",item:{type:"agent_message",text:"scan complete"}}));', 'console.log(JSON.stringify({type:"turn.completed",usage:null}));', @@ -8691,7 +8952,6 @@ if ([basename(process.argv[1]), ...process.argv.slice(2)].join(" ") !== "login s ].join("\n"), ); const fake = nodeCodex(preload); - const model = `fixture-${name}-model`; const provider = `fixture-${name}-provider`; const client = new TestClient( { @@ -8699,8 +8959,11 @@ if ([basename(process.argv[1]), ...process.argv.slice(2)].join(" ") !== "login s model, model_provider: provider, model_reasoning_effort: "ultra", - model_reasoning_summary: - name === "first" ? "none" : "concise", + // JavaScript callers can omit the merged default with undefined. + model_reasoning_summary: captureSummary + ? (undefined as unknown as string) + : summary, + service_tier: name === "first" ? "flex" : "fast", features: { multi_agent_v2: { max_concurrent_threads_per_session: 4 }, }, @@ -8713,8 +8976,8 @@ if ([basename(process.argv[1]), ...process.argv.slice(2)].join(" ") !== "login s }, prepareRuntime: async () => ({ ...preparedRuntime(codexHome), - configPath: join(codexHome, "preflight.toml"), - deepScanConfigPath: join(codexHome, "deep.toml"), + configPath, + deepScanConfigPath: join(codexHome, "deep-scan.toml"), environment: { ...fake.environment, FIXTURE_SCAN_VALUE: name, @@ -8723,6 +8986,11 @@ if ([basename(process.argv[1]), ...process.argv.slice(2)].join(" ") !== "login s resolvePluginPython: async () => "/managed/python", prepareOutputDir: async () => scanDir, repositoryRevision: async () => "deadbeef", + runWorkbench: async (_options, args, input) => { + if (args[0] === "register-cli-scan") + recipeConfig = JSON.parse(input!).recipe.config; + return mockWorkbench(args, input); + }, createCodex: (options: CodexOptions) => { const codex = new Codex(options); return { @@ -8775,16 +9043,43 @@ if ([basename(process.argv[1]), ...process.argv.slice(2)].join(" ") !== "login s ); clients.push(client); const postScanPrompt = "Summarize the completed synthetic scan."; + const program = name === "first" ? "daybreak_blue" : "daybreak_red"; const result = await client.run(repository, { mode, postScanPrompt, + ...(selectedProgram ? { cyberAccessProgram: program } : {}), }); expect(result.threadId).toBe(`fixture-${name}-thread`); expect(result.turnResult.usage).toBeNull(); - const children = (await readFile(marker, "utf8")) + const invocations = (await readFile(marker, "utf8")) .trim() .split("\n") .map((line) => JSON.parse(line)); + const lookups = invocations.filter((child) => + child.args.includes("models"), + ); + const children = invocations.filter( + (child) => !child.args.includes("models"), + ); + expect(recipeConfig?.["model_reasoning_summary"]).toBe(summary); + expect(lookups).toHaveLength(captureSummary ? 1 : 0); + for (const lookup of lookups) { + expect(await realpath(lookup.cwd)).toBe(await realpath(scanDir)); + expect(lookup.home).toBe(codexHome); + expect(lookup.key).toBe(`synthetic-${name}-key`); + expect(lookup.value).toBe(name); + expect(lookup.args).toContain(`model=${JSON.stringify(model)}`); + } + const preflight = await readFile(configPath, "utf8"); + expect(parseToml(preflight)["model_reasoning_summary"]).toBe( + summary, + ); + expect(preflight).not.toContain(`synthetic-${name}-key`); + if (selectedProgram) + expect(parseToml(preflight)).toMatchObject({ + codex_security: { cyber_access_program: program }, + }); + else expect(parseToml(preflight)["codex_security"]).toBeUndefined(); expect(children).toHaveLength(2); expect(children[1].prompt).toBe(postScanPrompt); expect(children[1].args).toContain("resume"); @@ -8798,6 +9093,14 @@ if ([basename(process.argv[1]), ...process.argv.slice(2)].join(" ") !== "login s expect(child.home).toBe(codexHome); expect(child.key).toBe(`synthetic-${name}-key`); expect(child.value).toBe(name); + if (selectedProgram) { + const programIndex = child.args.indexOf( + "--cyber-access-program", + ); + expect( + child.args.slice(programIndex, programIndex + 2), + ).toEqual(["--cyber-access-program", program]); + } else expect(child.args).not.toContain("--cyber-access-program"); expect(child.args).toContain(`model=${JSON.stringify(model)}`); expect(child.args).toContain( `model_provider=${JSON.stringify(provider)}`, @@ -8806,6 +9109,9 @@ if ([basename(process.argv[1]), ...process.argv.slice(2)].join(" ") !== "login s expect(child.args).toContain( `model_reasoning_summary=${JSON.stringify(name === "first" ? "none" : "concise")}`, ); + expect(child.args).toContain( + `service_tier=${JSON.stringify(name === "first" ? "flex" : "fast")}`, + ); expect(child.args).toContain( "features.multi_agent_v2.max_concurrent_threads_per_session=4", ); diff --git a/sdk/typescript/tests-ts/build-plugin.test.ts b/sdk/typescript/tests-ts/build-plugin.test.ts index c5f6e351e8..3b43a89525 100644 --- a/sdk/typescript/tests-ts/build-plugin.test.ts +++ b/sdk/typescript/tests-ts/build-plugin.test.ts @@ -15,6 +15,7 @@ import { fileURLToPath, pathToFileURL } from "node:url"; import { brotliDecompressSync } from "node:zlib"; import { promisify } from "node:util"; import { afterEach, describe, expect, test } from "bun:test"; +import { transform } from "esbuild"; import { buildBundledPlugin } from "../scripts/build-plugin.mjs"; import { assertGeneratedPluginUntracked } from "../scripts/check-plugin-source.mjs"; @@ -64,6 +65,286 @@ async function snapshot(root: string) { afterEach(temporaryDirectories.cleanup); describe("bundled plugin build", () => { + test("bundles the native policy proof with only SDK dependencies", async () => { + const root = await temporaryDirectory(); + const plugin = join(root, "plugins", "codex-security"); + const native = join(plugin, "native"); + const sdk = join(root, "sdk", "typescript"); + const source = new URL("../../../plugins/codex-security/", import.meta.url); + await mkdir(native, { recursive: true }); + await mkdir(sdk, { recursive: true }); + await cp(new URL("../src", import.meta.url), join(sdk, "src"), { + recursive: true, + }); + await symlink( + fileURLToPath(new URL("../node_modules", import.meta.url)), + join(sdk, "node_modules"), + process.platform === "win32" ? "junction" : "dir", + ); + for (const name of ["schemas", "shared", "mcp-app/src"]) { + await mkdir(join(plugin, name, ".."), { recursive: true }); + await cp(new URL(name, source), join(plugin, name), { recursive: true }); + } + await copyFile( + new URL("mcp-app/helpers-main.ts", source), + join(plugin, "mcp-app", "helpers-main.ts"), + ); + for (const name of [ + "binding", + "platform", + "windows-binding", + "windows-flags", + "windows-files", + "proof-policy-windows", + ]) { + const compiled = await transform( + await readFile(new URL(`native/${name}.mts`, source), "utf8"), + { loader: "ts", format: "esm", target: "node20" }, + ); + await writeFile(join(native, `${name}.mjs`), compiled.code); + } + const { nativeTarget } = await import( + pathToFileURL(join(native, "platform.mjs")).href + ); + const binary = process.platform === "win32" ? "windows.node" : "unix.node"; + // The build copies this artifact; this portable test does not load native code. + await writeFixture( + native, + `dist/${nativeTarget}/${binary}`, + "native fixture", + ); + await expect( + stat(join(plugin, "mcp-app", "node_modules")), + ).rejects.toMatchObject({ + code: "ENOENT", + }); + await execFileAsync( + "node", + [join(native, "proof-policy-windows.mjs"), "build"], + { + cwd: native, + env: { ...process.env, NODE_PATH: "" }, + }, + ); + const proof = join(native, "dist", nativeTarget, "policy-proof"); + expect( + await readFile( + join(proof, "native", nativeTarget, "windows.node"), + "utf8", + ), + ).toBe("native fixture"); + const helper = join(root, "helpers.cjs"); + await copyFile(join(proof, "helpers.cjs"), helper); + await execFileAsync("node", [ + "--eval", + "require('node:fs').unlinkSync(process.argv[1])", + join(sdk, "node_modules"), + ]); + await expect(stat(join(sdk, "node_modules"))).rejects.toMatchObject({ + code: "ENOENT", + }); + const result = await execFileAsync( + "node", + [ + "--eval", + ` + const assert = require("node:assert/strict"); + const helper = require(process.argv.pop()); + const input = { + scanId: "synthetic-scan", + manifest: { scan: {} }, + findings: { findings: [] }, + coverage: { + surfaces: [], explicitExclusions: [], deferred: [], + }, + }; + assert.equal(helper.parseCanonicalScanDraft(input).scanId, input.scanId); + assert.throws(() => helper.parseCanonicalScanDraft({ + ...input, coverage: { ...input.coverage, surfaces: "invalid" }, + })); + console.log("Bundled parser accepted valid input and rejected invalid coverage."); + `, + helper, + ], + { cwd: root, env: { ...process.env, NODE_PATH: "" } }, + ); + expect(result.stdout).toBe( + "Bundled parser accepted valid input and rejected invalid coverage.\n", + ); + expect(result.stderr).toBe(""); + }); + + test("builds the MCP runtime with only MCP dependencies and no npm launcher", async () => { + const root = await temporaryDirectory(); + const plugin = join(root, "plugins", "codex-security"); + const mcp = join(plugin, "mcp-app"); + const sdk = join(root, "sdk", "typescript"); + const source = new URL("../../../plugins/codex-security/", import.meta.url); + await mkdir(mcp, { recursive: true }); + await mkdir(sdk, { recursive: true }); + for (const name of [ + "package.json", + "tsconfig.json", + "main.ts", + "deep-scan-main.ts", + "artifact-writer-main.ts", + "helpers-main.ts", + "server.ts", + "src", + "scripts", + "templates", + "tests/support/temporary-directories.ts", + "tests/support/json.ts", + ]) { + await cp(new URL(`mcp-app/${name}`, source), join(mcp, name), { + recursive: true, + }); + } + for (const name of [ + "schemas", + "shared", + "native/prebuilt", + "plugin-files.json", + "scripts/reserved_artifact_paths.json", + "scripts/codex_profile.mjs", + "scripts/codex_profile.d.mts", + "scripts/codex_session.mjs", + "scripts/codex_session.d.mts", + ]) { + await mkdir(join(plugin, name, ".."), { recursive: true }); + await cp(new URL(name, source), join(plugin, name), { recursive: true }); + } + for (const name of await readdir(new URL("native/", source))) { + if (/\.(?:mjs|mts)$/.test(name)) { + await copyFile( + new URL(`native/${name}`, source), + join(plugin, "native", name), + ); + } + } + for (const name of ["src", "package.json", "tsconfig.json"]) { + await cp(new URL(`../${name}`, import.meta.url), join(sdk, name), { + recursive: true, + }); + } + await symlink( + fileURLToPath(new URL("mcp-app/node_modules", source)), + join(mcp, "node_modules"), + process.platform === "win32" ? "junction" : "dir", + ); + await expect(stat(join(sdk, "node_modules"))).rejects.toMatchObject({ + code: "ENOENT", + }); + const bin = join(root, "bin"); + const launcher = process.platform === "win32" ? "npm.cmd" : "npm"; + await writeFixture( + bin, + launcher, + process.platform === "win32" ? "@exit /b 91\r\n" : "#!/bin/sh\nexit 91\n", + ); + if (process.platform !== "win32") await chmod(join(bin, launcher), 0o755); + + const destination = join(root, "mcp"); + await execFileAsync( + "node", + [join(mcp, "scripts", "build_mcp_app.mjs"), "--output", destination], + { + env: { + ...process.env, + NODE_PATH: "", + PATH: [bin, process.env["PATH"]].filter(Boolean).join(delimiter), + }, + }, + ); + + await execFileAsync("node", [ + "--eval", + "require('node:fs').unlinkSync(process.argv[1])", + join(mcp, "node_modules"), + ]); + await expect(stat(join(mcp, "node_modules"))).rejects.toMatchObject({ + code: "ENOENT", + }); + const contract = JSON.parse( + await readFile( + new URL( + "../../../plugins/codex-security/plugin-files.json", + import.meta.url, + ), + "utf8", + ), + ) as { shippedExact: string[] }; + expect(await files(destination)).toEqual( + contract.shippedExact + .filter((path) => path.startsWith("mcp/")) + .map((path) => path.slice(4)) + .sort(), + ); + const helper = await execFileAsync("node", [ + join(destination, "helpers.mjs"), + "resolve-security-md", + "--repo", + root, + "--list", + ]); + expect(helper.stdout).toBe("[]\n"); + expect(helper.stderr).toBe(""); + + const repository = await temporaryDirectory(); + const policy = "Preserve this synthetic inherited security policy."; + await writeFixture( + repository, + "SECURITY.md", + `# Synthetic policy\n${policy}\n`, + ); + const alias = join(await temporaryDirectory(), "plugin link"); + await symlink( + root, + alias, + process.platform === "win32" ? "junction" : "dir", + ); + await mkdir(join(root, "scripts"), { recursive: true }); + await copyFile( + new URL("scripts/launch_codex_security_mcp", source), + join(root, "scripts", "launch_codex_security_mcp"), + ); + const node = ( + await execFileAsync("node", ["--print", "process.execPath"]) + ).stdout.trim(); + for (const pluginPath of [root, alias]) { + const linkedHelper = join(pluginPath, "mcp", "helpers.mjs"); + const list = await execFileAsync( + process.platform === "win32" ? node : "/bin/sh", + [ + ...(process.platform === "win32" + ? [linkedHelper] + : [ + join(pluginPath, "scripts", "launch_codex_security_mcp"), + "--helper", + ]), + "resolve-security-md", + "--repo", + repository, + "--list", + ], + { env: { ...process.env, CODEX_MCP_NODE_PATH: node, NODE_PATH: "" } }, + ); + expect(list.stdout).toBe('["SECURITY.md"]\n'); + expect(list.stderr).toBe(""); + const guidance = await execFileAsync(node, [ + linkedHelper, + "resolve-security-md", + "--repo", + repository, + "--scope", + repository, + "--out", + "-", + ]); + expect(guidance.stdout).toContain(policy); + expect(guidance.stderr).toBe(""); + } + }); test("ships the Cloud connection and skill unchanged", async () => { for (const path of [ ".app.json", @@ -104,7 +385,17 @@ describe("bundled plugin build", () => { const plugin = fileURLToPath( new URL("../../../plugins/codex-security/", import.meta.url), ); - const source = join(root, "plugin"); + const source = join(root, "plugins", "codex-security"); + const sdk = join(root, "sdk", "typescript"); + await mkdir(sdk, { recursive: true }); + for (const name of ["src", "package.json", "tsconfig.json"]) { + await cp(new URL(`../${name}`, import.meta.url), join(sdk, name), { + recursive: true, + }); + } + await expect(stat(join(sdk, "node_modules"))).rejects.toMatchObject({ + code: "ENOENT", + }); await cp(join(plugin, "mcp-app"), join(source, "mcp-app"), { recursive: true, filter: (path) => diff --git a/sdk/typescript/tests-ts/cli-scan-logs.test.ts b/sdk/typescript/tests-ts/cli-scan-logs.test.ts index 817d711cc0..393493f14a 100644 --- a/sdk/typescript/tests-ts/cli-scan-logs.test.ts +++ b/sdk/typescript/tests-ts/cli-scan-logs.test.ts @@ -1,4 +1,6 @@ import { writeJsonLines } from "./support/json.js"; +import { throwing } from "./support/errors.js"; +import { createCliTest } from "./support/cli-run.js"; import { execFile } from "node:child_process"; import { mkdir, mkdtemp, realpath, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; @@ -13,8 +15,6 @@ import { scanLogsJson } from "../src/cli-scan-logs-json.js"; import { readSavedScanLogs } from "../src/scan-logs.js"; import { VERSION } from "../src/version.js"; import { capture, dependencies } from "./cli-fixtures.js"; -import { throwing } from "./support/errors.js"; -import { createCliTest } from "./support/cli-run.js"; async function fixture(desktop = false, archived = false, continuation = true) { const state = await realpath(await mkdtemp(join(tmpdir(), "saved-logs-"))); @@ -72,6 +72,128 @@ function withoutDuration(text: string) { return value; } +async function attributedFixture(attributedOwner = false) { + const state = await realpath(await mkdtemp(join(tmpdir(), "saved-logs-"))); + const home = join(state, "codex-home"); + await mkdir(join(home, "sessions"), { recursive: true }); + const events: Record[] = [ + { type: "session_meta", payload: { id: "thread-1" } }, + { + type: "event_msg", + payload: { + message: 'full values: \" \\ \n \u0000 😀', + nested: [null, false, 2], + }, + }, + ]; + const timestamp = "2026-08-11T12:01:00.000Z"; + if (attributedOwner) { + events.splice(1, 0, { + type: "turn_context", + timestamp, + payload: { turn_id: "scan-turn" }, + }); + Object.assign(events.at(-1)!, { timestamp }); + } + await writeFile( + join(home, "sessions", "rollout.jsonl"), + events.map((event) => JSON.stringify(event)).join("\n"), + ); + const originalHome = join(state, "original-home"); + const scanDir = join(state, "scan"); + const settingsDirectory = join(scanDir, "artifacts", "deep_discovery"); + await mkdir(settingsDirectory, { recursive: true }); + await mkdir(join(originalHome, "sessions"), { recursive: true }); + let ownerEvents = events; + if (attributedOwner) { + const repeated = { + type: "event_msg", + timestamp, + payload: { message: "repeated scan occurrence" }, + }; + ownerEvents = [ + ...events, + repeated, + repeated, + { + type: "event_msg", + timestamp, + payload: { message: "recorded non-usage suffix" }, + }, + ]; + await writeFile( + join(home, "sessions", "rollout.jsonl"), + [ + ...events, + { type: "turn_context", timestamp, payload: { turn_id: "other-turn" } }, + { + type: "event_msg", + timestamp, + payload: { message: "unrelated first-copy suffix" }, + }, + ] + .map((event) => JSON.stringify(event)) + .join("\n"), + ); + await writeFile( + join(originalHome, "sessions", "owner.jsonl"), + ownerEvents.map((event) => JSON.stringify(event)).join("\n"), + ); + } + await writeFile( + join(settingsDirectory, "execution-settings.json"), + JSON.stringify({ + version: 1, + settings: { + codexHome: originalHome, + codexPath: join(originalHome, "codex"), + }, + }), + ); + await writeFile( + join(home, "sessions", "worker.jsonl"), + JSON.stringify({ type: "session_meta", payload: { id: "worker" } }) + "\n", + ); + await writeFile( + join(originalHome, "sessions", "worker.jsonl"), + [ + { type: "session_meta", payload: { id: "worker" } }, + { type: "event_msg", payload: { message: "recorded worker suffix" } }, + ] + .map((event) => JSON.stringify(event) + "\n") + .join(""), + ); + const scan = { + scanId: "scan-1", + continuationThreadId: "thread-1", + mode: "deep", + scanDir, + executionThreadIds: ["worker"], + executionAttribution: { + formatVersion: 1 as const, + ...(attributedOwner ? {} : { legacy: true as const }), + workerCodexHome: originalHome, + executionThreadIds: ["worker"], + owner: { + threadId: "thread-1", + turnId: attributedOwner ? "scan-turn" : null, + startedAt: timestamp, + }, + startedAt: timestamp, + completedAt: timestamp, + }, + }; + const logs = await readSavedScanLogs(scan, [home, originalHome]); + const deps = dependencies({ + environment: { CODEX_SECURITY_STATE_DIR: state, CODEX_HOME: home }, + onWorkbench: () => ({ scan }), + }); + deps.createSecurity = () => { + throw new Error("Reading logs must not start Codex"); + }; + return { state, logs, deps, ownerEvents }; +} + describe("saved logs JSON output", () => { let dataHome: string; let previousDataHome: string | undefined; @@ -88,6 +210,63 @@ describe("saved logs JSON output", () => { await rm(dataHome, { recursive: true, force: true }); }); + test("selects the recorded owner suffix after scan attribution through the saved logs command", async () => { + const f = await attributedFixture(true); + try { + const stdout = capture(); + expect( + await main( + ["scans", "logs", "scan-1", "--json"], + stdout.stream, + capture().stream, + f.deps, + ), + ).toBe(0); + const result = JSON.parse(stdout.text()); + expect( + result.sessions.map(({ threadId }: { threadId: string }) => threadId), + ).toEqual(["worker", "thread-1"]); + expect( + result.events + .filter( + ({ threadId }: { threadId: string }) => threadId === "thread-1", + ) + .map(({ event }: { event: unknown }) => event), + ).toEqual(f.ownerEvents); + } finally { + await rm(f.state, { recursive: true, force: true }); + } + }); + + test("loads the same-thread recorded worker suffix through the saved logs command", async () => { + const f = await attributedFixture(); + try { + const stdout = capture(); + expect( + await main( + ["scans", "logs", "scan-1", "--json"], + stdout.stream, + capture().stream, + f.deps, + ), + ).toBe(0); + expect( + JSON.parse(stdout.text()).sessions.map( + ({ threadId }: { threadId: string }) => threadId, + ), + ).toEqual(["thread-1", "worker"]); + expect(JSON.parse(stdout.text()).events).toContainEqual({ + threadId: "worker", + event: { + type: "event_msg", + payload: { message: "recorded worker suffix" }, + }, + }); + } finally { + await rm(f.state, { recursive: true, force: true }); + } + }); + test.each([ [false, true], [true, true], @@ -129,7 +308,9 @@ describe("saved logs JSON output", () => { test("preserves the stale installed-skills CTA after saved logs", async () => { const f = await fixture(); + const previousDataHome = process.env["XDG_DATA_HOME"]; try { + const dataHome = join(f.state, "data"); const skillPath = join(f.state, "skills", "codex-security-scans"); await mkdir(join(dataHome, "incur"), { recursive: true }); await mkdir(skillPath, { recursive: true }); @@ -145,16 +326,22 @@ describe("saved logs JSON output", () => { paths: [skillPath], }), ); + process.env["XDG_DATA_HOME"] = dataHome; for (const args of [ ["--json"], ["--format", "json"], ["--format=json"], ]) { - const { stdout, stderr, runCli } = createCliTest(main); - - expect(await runCli(["scans", "logs", "scan-1", ...args], f.deps)).toBe( - 0, - ); + const stdout = capture(); + const stderr = capture(); + expect( + await main( + ["scans", "logs", "scan-1", ...args], + stdout.stream, + stderr.stream, + f.deps, + ), + ).toBe(0); const expected = await referenceOutput(["--json"], f.logs); expect(Object.keys(JSON.parse(expected))).toEqual([ "scanId", @@ -167,6 +354,8 @@ describe("saved logs JSON output", () => { expect(stderr.text()).toBe(""); } } finally { + if (previousDataHome === undefined) delete process.env["XDG_DATA_HOME"]; + else process.env["XDG_DATA_HOME"] = previousDataHome; await rm(f.state, { recursive: true, force: true }); } }); @@ -200,11 +389,16 @@ describe("saved logs JSON output", () => { )("preserves Incur output for %j", async (args) => { const f = await fixture(); try { - const { stdout, stderr, runCli } = createCliTest(main); - - expect(await runCli(["scans", "logs", "scan-1", ...args], f.deps)).toBe( - 0, - ); + const stdout = capture(); + const stderr = capture(); + expect( + await main( + ["scans", "logs", "scan-1", ...args], + stdout.stream, + stderr.stream, + f.deps, + ), + ).toBe(0); const expected = await referenceOutput( args.flatMap((arg) => arg === "--format=json" ? ["--format", "json"] : [arg], @@ -238,27 +432,42 @@ describe("saved logs JSON output", () => { ["--json", "--filter-output"], ].map((args) => [args]), )("preserves invalid-option failure for %j", async (args) => { - const { stdout, stderr, runCli } = createCliTest(main); - + const stdout = capture(); + const stderr = capture(); expect( - await runCli(["scans", "logs", "scan-1", ...args], dependencies()), + await main( + ["scans", "logs", "scan-1", ...args], + stdout.stream, + stderr.stream, + dependencies(), + ), ).toBe(2); expect(stdout.text()).toBe(""); expect(stderr.text()).not.toBe(""); }); - test("formats empty session and event lists", async () => { - const logs = { - scanId: "scan-1", - threadId: "thread-1", - sessions: [], - events: [], - }; - const chunks = []; - for await (const chunk of scanLogsJson(logs)) chunks.push(chunk); - expect(Buffer.concat(chunks).toString()).toBe( - `${Formatter.format(logs, "json")}\n`, - ); + test("preserves JSON conversion and empty arrays", async () => { + const f = await fixture(); + try { + f.logs.events.push({ + threadId: "thread-1", + event: { + bigint: 42n, + missing: undefined, + values: [NaN, Infinity, -0], + }, + }); + for (const logs of [f.logs, { ...f.logs, sessions: [], events: [] }]) { + const chunks = []; + for await (const chunk of scanLogsJson(logs)) + chunks.push(Buffer.from(chunk)); + expect(Buffer.concat(chunks).toString()).toBe( + `${Formatter.format(logs, "json")}\n`, + ); + } + } finally { + await rm(f.state, { recursive: true, force: true }); + } }); test("waits for output backpressure and leaves the stream open", async () => { diff --git a/sdk/typescript/tests-ts/cli.test.ts b/sdk/typescript/tests-ts/cli.test.ts index b92046f93f..95ffa710b0 100644 --- a/sdk/typescript/tests-ts/cli.test.ts +++ b/sdk/typescript/tests-ts/cli.test.ts @@ -3530,7 +3530,7 @@ describe("CLI", () => { ), ).toBe(0); expect(rerunConfig?.codexOverrides).toMatchObject({ - model_provider: "openai", + model_provider: provider, profile: "selected", profiles: { selected: { model, model_provider: provider } }, model_providers: { [provider]: providerConfig }, diff --git a/sdk/typescript/tests-ts/codex-process-cancellation.test.ts b/sdk/typescript/tests-ts/codex-process-cancellation.test.ts index d49377bbc9..038bee725b 100644 --- a/sdk/typescript/tests-ts/codex-process-cancellation.test.ts +++ b/sdk/typescript/tests-ts/codex-process-cancellation.test.ts @@ -79,6 +79,7 @@ for (const mode of [ }, undefined, controller.signal, + undefined, mode.startsWith("archive-"), ); } finally { diff --git a/sdk/typescript/tests-ts/cost.test.ts b/sdk/typescript/tests-ts/cost.test.ts index f69281ba0c..755f014dde 100644 --- a/sdk/typescript/tests-ts/cost.test.ts +++ b/sdk/typescript/tests-ts/cost.test.ts @@ -1,7 +1,17 @@ +import { Database } from "bun:sqlite"; import { jsonLines } from "./support/json.js"; import { runNodePython } from "./support/python-probe.js"; import * as filesystem from "node:fs/promises"; -import { appendFile, writeFile } from "node:fs/promises"; +import { + appendFile, + cp, + mkdir, + readFile, + rename, + unlink, + symlink, + writeFile, +} from "node:fs/promises"; import * as fs from "node:fs/promises"; import { join, parse, sep } from "node:path"; import { Codex } from "@openai/codex-sdk"; @@ -10,12 +20,21 @@ import { estimateScanCost, ScanCostTracker, type ScanSessionEvent, + type ScanCost, type ScanWorkerEvent, } from "../src/cost.js"; import type { ScanActivity } from "../src/scan-activity.js"; -import { formatTokenUsage, tokenUsage } from "../src/cost-model.js"; +import { + estimateScanCostLowerBound, + formatTokenUsage, + tokenUsage, +} from "../src/cost-model.js"; import { readScanLogs } from "../src/scan-logs.js"; -import { sessionParentThreadId } from "../src/scan-sessions.js"; +import { + cachedScanAttributionReader, + sessionParentThreadId, + type ScanExecutionAttribution, +} from "../src/scan-sessions.js"; import type { ScanProgress } from "../src/worker-progress.js"; import { PLUGIN_ROOT as BUNDLED_PLUGIN_ROOT } from "./plugin-root.js"; import { @@ -785,6 +804,155 @@ describe("live scan cost tracking", () => { }, ); + test.each([false, true])( + "reconciles final cache categories for an observed model (missing child: %p)", + async (missingChild) => { + const home = await codexHome(); + const usage = { + input_tokens: 1_000_000, + cached_input_tokens: 0, + output_tokens: 10_000, + }; + const finalUsage = { ...usage, cache_write_input_tokens: 120_000 }; + const parent = await writeSession(home, "scan-thread", usage); + const records = (await readFile(parent, "utf8")).trimEnd().split("\n"); + records.splice( + 1, + 0, + JSON.stringify({ + type: "turn_context", + payload: { model: "gpt-6-astra", turn_id: "scan-turn" }, + }), + ); + await writeFile(parent, `${records.join("\n")}\n`); + if (missingChild) { + await writeFile( + join(home, "sessions", "missing-worker.jsonl"), + `${JSON.stringify({ + type: "session_meta", + payload: { id: "missing-worker", parent_thread_id: "scan-thread" }, + })}\n`, + ); + } + const initialCost = estimateScanCost("gpt-6-astra", usage)!; + const correctedCost = estimateScanCost("gpt-6-astra", finalUsage)!; + const budget = + (initialCost.estimatedUsd + correctedCost.estimatedUsd) / 2; + expect(initialCost.estimatedUsd).toBeLessThan(budget); + expect(correctedCost.estimatedUsd).toBeGreaterThan(budget); + const published: Readonly[] = []; + const lowerBounds: Readonly[] = []; + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-6-astra", + maxCostUsd: budget, + onCost: (cost, complete) => { + if (complete) published.push(cost); + }, + onCostLowerBound: (cost) => lowerBounds.push(cost), + }); + try { + tracker.start("scan-thread"); + await tracker.refresh(); + const before = missingChild ? lowerBounds : published; + expect(before.at(-1)?.estimatedUsd).toBe(initialCost.estimatedUsd); + const completed = await tracker.stop(finalUsage); + const emitted = missingChild ? lowerBounds : published; + expect(emitted.at(-1)).toMatchObject({ + inputTokens: usage.input_tokens, + outputTokens: usage.output_tokens, + cacheWriteInputTokens: finalUsage.cache_write_input_tokens, + estimatedUsd: correctedCost.estimatedUsd, + }); + expect(emitted.at(-1)?.estimatedUsd).toBeGreaterThan(budget); + if (missingChild) { + expect(completed.cost).toBeNull(); + expect(completed.usage).toBeNull(); + expect(published).toHaveLength(0); + } else { + expect(completed.cost?.modelCosts).toHaveLength(1); + expect(completed.cost?.modelCosts?.[0]).toMatchObject({ + model: "gpt-6-astra", + cacheWriteInputTokens: finalUsage.cache_write_input_tokens, + estimatedUsd: correctedCost.estimatedUsd, + }); + } + const count = emitted.length; + expect(await tracker.refresh()).toEqual(completed); + expect(emitted).toHaveLength(count); + } finally { + await tracker.stop(); + } + }, + ); + + test("does not assign a final cache correction across observed models", async () => { + const home = await codexHome(); + const parent = await writeSession(home, "scan-thread", { + input_tokens: 100, + output_tokens: 10, + }); + const records = (await readFile(parent, "utf8")).trimEnd().split("\n"); + records.splice( + 1, + 0, + JSON.stringify({ + type: "turn_context", + payload: { model: "gpt-5.6-sol", turn_id: "first-turn" }, + }), + ); + records.push( + JSON.stringify({ + type: "turn_context", + payload: { model: "gpt-6-astra", turn_id: "second-turn" }, + }), + JSON.stringify({ + type: "event_msg", + payload: { + type: "token_count", + info: { total_token_usage: { input_tokens: 200, output_tokens: 20 } }, + }, + }), + ); + await writeFile(parent, `${records.join("\n")}\n`); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-6-astra", + }); + try { + tracker.start("scan-thread"); + const running = await tracker.refresh(); + expect(running.cost?.modelCosts?.map((cost) => cost.model)).toEqual([ + "gpt-5.6-sol", + "gpt-6-astra", + ]); + const completed = await tracker.stop({ + input_tokens: 200, + output_tokens: 20, + cache_write_input_tokens: 20, + }); + expect(completed.cost).toBeNull(); + expect(completed.usage).toMatchObject({ + input_tokens: 200, + cache_write_input_tokens: 20, + modelUsage: [ + { + model: "gpt-5.6-sol", + input_tokens: 100, + cache_write_input_tokens: 0, + }, + { + model: "gpt-6-astra", + input_tokens: 100, + cache_write_input_tokens: 0, + }, + ], + }); + } finally { + await tracker.stop(); + } + }); + test("retains reported write charges when another worker omits cache writes", async () => { const home = await codexHome(); await writeSession(home, "scan-thread", { @@ -858,23 +1026,26 @@ describe("live scan cost tracking", () => { const tracker = costTracker(home, { maxCostUsd: 1, }); - const refresh = tracker.refresh.bind(tracker); - tracker.refresh = async () => { + const forcedReads: boolean[] = []; + tracker.setAttributionReader(async (force) => { + forcedReads.push(force === true); await new Promise((resolve) => releases.push(resolve)); - return refresh(); - }; + return null; + }); tracker.start("scan-thread"); await new Promise((resolve) => setTimeout(resolve, 350)); - expect(releases).toHaveLength(1); + expect(forcedReads).toEqual([false]); const stopped = tracker.stop(); - expect(releases).toHaveLength(2); + expect(releases).toHaveLength(1); releases[0]!(); + await waitFor(() => releases.length === 2); + expect(forcedReads).toEqual([false, true]); releases[1]!(); expect((await stopped).cost?.inputTokens).toBe(100); - expect(releases).toHaveLength(2); + expect(forcedReads).toEqual([false, true]); }); test("retries one coalesced poll after a failed refresh", async () => { @@ -1026,6 +1197,115 @@ describe("live scan cost tracking", () => { expect(resumed).toEqual([{ kind: "observed", worker: 1 }]); }); + test("retains unknown Standard usage while reporting known cost and later child usage", async () => { + const home = await codexHome(); + await writeSession(home, "scan-thread", { + input_tokens: 1_250, + cached_input_tokens: 200, + output_tokens: 30, + }); + const worker = await writeSession( + home, + "worker-thread", + {}, + { + parent: "scan-thread", + }, + ); + await writeFile( + worker, + (await readFile(worker, "utf8")).split("\n")[0]! + "\n", + ); + const publicCosts: Readonly[] = []; + const lowerBounds: Readonly[] = []; + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + onCost: (cost, complete) => { + if (complete) publicCosts.push(cost); + }, + onCostLowerBound: (cost) => lowerBounds.push(cost), + }); + tracker.start("scan-thread"); + try { + expect(await tracker.refresh()).toEqual({ usage: null, cost: null }); + expect(publicCosts).toEqual([]); + expect(lowerBounds.at(-1)).toMatchObject({ + inputTokens: 1_250, + estimatedUsd: 0.00488, + coverage: "partial", + }); + await writeSession( + home, + "worker-thread", + { + input_tokens: 100, + output_tokens: 0, + }, + { parent: "scan-thread" }, + ); + expect(await tracker.refresh()).toMatchObject({ + usage: { input_tokens: 1_350 }, + cost: { inputTokens: 1_350, estimatedUsd: 0.00528 }, + }); + expect(publicCosts.at(-1)?.estimatedUsd).toBe(0.00528); + } finally { + await tracker.stop(); + } + }); + + test("prices known Standard receipts by their recorded model while child usage is missing", async () => { + const home = await codexHome(); + const parent = await writeSession(home, "scan-thread", {}); + await appendFile( + parent, + jsonLines([ + { + type: "token_usage_record", + payload: { + thread_id: "scan-thread", + response_id: "known-response", + model: "gpt-5.6-luna", + usage: { input_tokens: 1_000, output_tokens: 0 }, + }, + }, + ]) + "\n", + ); + const child = await writeSession( + home, + "worker-thread", + {}, + { parent: "scan-thread" }, + ); + await writeFile( + child, + (await readFile(child, "utf8")).split("\n")[0]! + "\n", + ); + const lowerBounds: Readonly[] = []; + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + onCostLowerBound: (cost) => lowerBounds.push(cost), + }); + tracker.start("scan-thread"); + try { + expect(await tracker.refresh()).toEqual({ usage: null, cost: null }); + expect(lowerBounds).toHaveLength(1); + expect(lowerBounds[0]).toMatchObject({ + estimatedUsd: estimateScanCost("gpt-5.6-luna", { + input_tokens: 1_000, + output_tokens: 0, + })!.estimatedUsd, + coverage: "partial", + modelCosts: [{ model: "gpt-5.6-luna", inputTokens: 1_000 }], + }); + await tracker.refresh(); + expect(lowerBounds).toHaveLength(1); + } finally { + await tracker.stop(); + } + }); + test("counts the scan and delegated workers without including other scans", async () => { const home = await codexHome(); const parent = await writeSession(home, "scan-thread", { @@ -2733,3 +3013,1447 @@ describe("live scan cost tracking", () => { }, ); }); + +describe("recorded Deep worker homes", () => { + test.each([ + ["recorded=false archived=false", false, false], + ["recorded=false archived=true", false, true], + ["recorded=true archived=false", true, false], + ["recorded=true archived=true", true, true], + ] as const)( + "retains archived worker usage: %s", + async (_label, recorded, archived) => { + const home = await codexHome(); + const workerHome = recorded ? await codexHome() : home; + const at = "2026-09-01T00:00:02Z"; + await writeSession(home, "owner", {}); + const worker = await writeSession(workerHome, "worker", {}); + await appendFile( + worker, + jsonLines([ + { + type: "turn_context", + timestamp: at, + payload: { turn_id: "worker-turn", model: "gpt-5.6-sol" }, + }, + { + type: "token_usage_record", + timestamp: at, + payload: { + thread_id: "worker", + turn_id: "worker-turn", + response_id: "worker-response", + model: "gpt-5.6-sol", + usage: { input_tokens: 1_000, output_tokens: 0 }, + }, + }, + ]) + "\n", + ); + if (archived) { + const directory = join(workerHome, "archived_sessions"); + await mkdir(directory, { recursive: true }); + await rename(worker, join(directory, "worker.jsonl")); + } + const costs: Readonly[] = []; + const tracker = new ScanCostTracker({ + codexHome: home, + scanDirectory: join(home, "scan"), + model: "gpt-5.6-sol", + maxCostUsd: 0.003, + onCost: (cost) => costs.push(cost), + onCostLowerBound: (cost) => costs.push(cost), + }); + tracker.setAttributionReader(async () => ({ + formatVersion: 1, + workerCodexHome: workerHome, + executionThreadIds: ["worker"], + owner: { threadId: "owner", turnId: "owner-turn", startedAt: at }, + startedAt: at, + completedAt: null, + })); + tracker.start("owner"); + try { + await tracker.refresh(); + expect(costs.at(-1)?.inputTokens).toBe(1_000); + expect(costs.at(-1)?.estimatedUsd).toBeCloseTo(0.004, 10); + expect(costs.at(-1)!.estimatedUsd).toBeGreaterThan(0.003); + } finally { + await tracker.stop(); + } + }, + ); + + test("only enforces a priced subtotal from a valid attributed usage partition", () => { + const known = { + model: "gpt-5.6-sol", + input_tokens: 1_000, + output_tokens: 0, + }; + const unknown = { model: null, input_tokens: 100, output_tokens: 0 }; + const usage = { + input_tokens: 1_100, + output_tokens: 0, + modelUsage: [known, unknown], + }; + expect(estimateScanCostLowerBound("gpt-5.6-sol", usage)?.estimatedUsd).toBe( + 0.004, + ); + expect(estimateScanCost("gpt-5.6-sol", usage)).toBeNull(); + for (const invalid of [ + { ...usage, input_tokens: 999 }, + { ...usage, modelUsage: [known, known, unknown] }, + { ...usage, modelUsage: [known, { ...unknown, input_tokens: -1 }] }, + { ...usage, modelUsage: [{ ...known, model: null }, unknown] }, + ]) + expect(estimateScanCostLowerBound("gpt-5.6-sol", invalid)).toBeNull(); + }); + + test.each(["gpt-5.6-sol", "synthetic-unpriced-model"])( + "prices live worker counters after an earlier response receipt with model %s", + async (currentModel) => { + const home = await codexHome(); + const at = "2026-09-01T00:00:02Z"; + await writeSession(home, "owner", {}); + const worker = await writeSession(home, "worker", {}); + const receipt = (id: string, model: string, input: number) => ({ + type: "token_usage_record", + timestamp: at, + payload: { + thread_id: "worker", + turn_id: "worker-turn", + response_id: id, + model, + usage: { input_tokens: input, output_tokens: 0 }, + }, + }); + await appendFile( + worker, + jsonLines([ + { + type: "turn_context", + timestamp: at, + payload: { turn_id: "worker-turn", model: "gpt-5.6-sol" }, + }, + { + type: "event_msg", + timestamp: at, + payload: { + type: "token_count", + info: { + total_token_usage: { + input_tokens: 100, + output_tokens: 0, + }, + }, + }, + }, + receipt("earlier", "gpt-5.6-sol", 100), + ]) + "\n", + ); + const costUpdates: Readonly[] = []; + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + maxCostUsd: 0.002, + onCost: (cost) => costUpdates.push(cost), + onCostLowerBound: (cost) => costUpdates.push(cost), + }); + tracker.setAttributionReader(async () => ({ + formatVersion: 1, + executionThreadIds: ["worker"], + owner: { threadId: "owner", turnId: "turn", startedAt: at }, + startedAt: at, + completedAt: null, + })); + tracker.start("owner"); + try { + await tracker.refresh(); + expect(costUpdates.at(-1)?.estimatedUsd).toBeCloseTo(0.0004, 10); + await appendFile( + worker, + jsonLines([ + { + type: "turn_context", + timestamp: at, + payload: { turn_id: "worker-turn", model: currentModel }, + }, + { + type: "event_msg", + timestamp: at, + payload: { + type: "token_count", + info: { + total_token_usage: { + input_tokens: 1_100, + output_tokens: 0, + }, + }, + }, + }, + ]) + "\n", + ); + const running = await tracker.refresh(); + expect(tokenUsage(running.usage)?.input_tokens).toBe(1_100); + expect(running.cost).toBeNull(); + expect(costUpdates.at(-1)?.estimatedUsd).toBeCloseTo( + currentModel === "gpt-5.6-sol" ? 0.0044 : 0.0004, + 10, + ); + if (currentModel === "gpt-5.6-sol") + expect(costUpdates.at(-1)!.estimatedUsd).toBeGreaterThan(0.002); + const completedReceipt = receipt("current", currentModel, 1_000); + await appendFile( + worker, + jsonLines([completedReceipt, completedReceipt]) + "\n", + ); + const completed = await tracker.refresh(); + expect(tokenUsage(completed.usage)?.input_tokens).toBe(1_100); + expect(costUpdates.at(-1)?.estimatedUsd).toBeCloseTo( + currentModel === "gpt-5.6-sol" ? 0.0044 : 0.0004, + 10, + ); + } finally { + await tracker.stop(); + } + }, + ); + + test.each([null, "synthetic-unpriced-model"])( + "reports an internal priced lower bound with model %p without inventing a total", + async (unknownModel) => { + const home = await codexHome(); + const at = "2026-09-01T00:00:02Z"; + const known = await writeSession(home, "owner", {}); + await appendFile( + known, + JSON.stringify({ + type: "token_usage_record", + timestamp: at, + payload: { + thread_id: "owner", + turn_id: "turn", + response_id: "known-response", + model: "gpt-5.6-sol", + usage: { input_tokens: 1_000, output_tokens: 0 }, + }, + }) + "\n", + ); + const unknown = await writeSession(home, "worker", {}); + await appendFile( + unknown, + JSON.stringify({ + type: "turn_context", + timestamp: at, + payload: { + turn_id: "worker-turn", + ...(unknownModel === null ? {} : { model: unknownModel }), + }, + }) + + "\n" + + JSON.stringify({ + type: "event_msg", + timestamp: at, + payload: { + type: "token_count", + info: { + total_token_usage: { input_tokens: 100, output_tokens: 0 }, + }, + }, + }) + + "\n", + ); + const lowerBounds: Readonly[] = []; + const publicCosts: Readonly[] = []; + const options = { + codexHome: home, + model: "gpt-5.6-sol", + maxCostUsd: 0.003, + onCost: (cost: Readonly, complete: boolean) => { + if (complete) publicCosts.push(cost); + }, + onCostLowerBound: (cost: Readonly) => lowerBounds.push(cost), + }; + const tracker = new ScanCostTracker(options); + tracker.setAttributionReader(async () => ({ + formatVersion: 1, + executionThreadIds: ["worker"], + owner: { threadId: "owner", turnId: "turn", startedAt: at }, + startedAt: at, + completedAt: null, + })); + tracker.start("owner"); + try { + const snapshot = await tracker.refresh(); + expect(tokenUsage(snapshot.usage)?.input_tokens).toBe(1_100); + expect(snapshot.cost).toBeNull(); + expect(publicCosts).toEqual([]); + expect(lowerBounds).toHaveLength(1); + expect(lowerBounds[0]).toMatchObject({ + inputTokens: 1_000, + estimatedUsd: 0.004, + coverage: "partial", + }); + expect(lowerBounds[0]!.estimatedUsd).toBeGreaterThan( + options.maxCostUsd, + ); + await tracker.refresh(); + expect(lowerBounds).toHaveLength(1); + } finally { + await tracker.stop(); + } + }, + ); + + test.each(["identical", "prefix-first", "prefix-last"] as const)( + "forwards each event occurrence once from copied logs: %s", + async (copy) => { + const home = await codexHome(); + const recordedHome = await codexHome(); + const scanDirectory = join(home, "scan"); + const settings = join(scanDirectory, "artifacts", "deep_discovery"); + await mkdir(settings, { recursive: true }); + await writeFile( + join(settings, "execution-settings.json"), + JSON.stringify({ + version: 1, + settings: { codexHome: recordedHome }, + }), + ); + await mkdir(join(home, "sessions")); + await mkdir(join(recordedHome, "sessions")); + const first = join(home, "sessions", "worker.jsonl"); + const second = join(recordedHome, "sessions", "worker-copy.jsonl"); + const repeated = { + timestamp: "2026-09-01T00:00:02Z", + type: "event_msg", + payload: { type: "agent_message", message: "Reviewing source." }, + }; + const expected = [ + { + timestamp: "2026-09-01T00:00:00Z", + type: "session_meta", + payload: { id: "worker", model: "gpt-5.6-sol" }, + }, + repeated, + repeated, + { + timestamp: "2026-09-01T00:00:03Z", + type: "token_usage_record", + payload: { + thread_id: "worker", + turn_id: "turn", + response_id: "response", + model: "gpt-5.6-sol", + usage: { input_tokens: 100, output_tokens: 0 }, + }, + }, + ]; + const contents = expected.map((event) => JSON.stringify(event) + "\n"); + await writeFile( + first, + contents.slice(0, copy === "prefix-first" ? 2 : 4).join(""), + ); + await writeFile( + second, + contents.slice(0, copy === "prefix-last" ? 2 : 4).join(""), + ); + const events: ScanSessionEvent[] = []; + const options = { + codexHome: home, + scanDirectory, + model: "gpt-5.6-sol", + onSessionEvent: (event: ScanSessionEvent) => events.push(event), + }; + const recordedOwner = async () => ({ + formatVersion: 1 as const, + legacy: true as const, + workerCodexHome: recordedHome, + executionThreadIds: [], + owner: { + threadId: "worker", + turnId: null, + startedAt: "2026-09-01T00:00:00Z", + }, + startedAt: "2026-09-01T00:00:00Z", + completedAt: null, + }); + const tracker = new ScanCostTracker(options); + tracker.setAttributionReader(recordedOwner); + tracker.start("worker"); + try { + expect((await tracker.refresh()).cost?.inputTokens).toBe(100); + expect(events.map((event) => event.event)).toEqual(expected); + await tracker.refresh(); + expect(events).toHaveLength(expected.length); + // Both logs catch up, then a genuine repeated occurrence is copied later. + await writeFile(first, contents.join("")); + await writeFile(second, contents.join("")); + await appendFile(first, JSON.stringify(repeated) + "\n"); + await tracker.refresh(); + expect(events.map((event) => event.event)).toEqual([ + ...expected, + repeated, + ]); + await appendFile(second, JSON.stringify(repeated) + "\n"); + await tracker.stop(); + expect(events.map((event) => event.event)).toEqual([ + ...expected, + repeated, + ]); + // Re-reading after the owner interval becomes available filters early + // events, but must not renumber the surviving source occurrences. + tracker.setAttributionReader(async () => ({ + formatVersion: 1, + workerCodexHome: recordedHome, + executionThreadIds: ["worker"], + owner: { + threadId: "worker", + turnId: "turn", + startedAt: "2026-09-01T00:00:03Z", + }, + startedAt: "2026-09-01T00:00:03Z", + completedAt: "2026-09-01T00:00:04Z", + })); + await tracker.refresh(); + expect(events.map((event) => event.event)).toEqual([ + ...expected, + repeated, + ]); + events.length = 0; + const reconstructed = new ScanCostTracker(options); + reconstructed.setAttributionReader(recordedOwner); + reconstructed.start("worker"); + await reconstructed.stop(); + expect(events.map((event) => event.event)).toEqual([ + ...expected, + repeated, + ]); + } finally { + await tracker.stop(); + } + }, + ); + + test("keeps resumed worker usage and current parent usage isolated per scan", async () => { + const currentHome = await codexHome(); + const firstHome = await codexHome(); + const secondHome = await codexHome(); + const at = "2026-09-01T00:00:02Z"; + const trackers: ScanCostTracker[] = []; + const fixture = async (home: string, id: string, count: number) => { + const path = await writeSession(home, id, {}); + await appendFile( + path, + [ + { + type: "turn_context", + timestamp: at, + payload: { turn_id: "scan-turn", model: "gpt-5.6-sol" }, + }, + { + type: "token_usage_record", + timestamp: at, + payload: { + thread_id: id, + turn_id: "scan-turn", + response_id: `${id}-response`, + model: "gpt-5.6-sol", + usage: { input_tokens: count, output_tokens: 0 }, + }, + }, + ] + .map((event) => JSON.stringify(event)) + .join("\n") + "\n", + ); + return path; + }; + try { + const cases = [ + { id: "one", home: firstHome, parent: 10, discovery: 20, reducer: 30 }, + { id: "two", home: secondHome, parent: 11, discovery: 21, reducer: 31 }, + ]; + for (const row of cases) { + const scanDirectory = join(currentHome, "scans", row.id); + const settingsDirectory = join( + scanDirectory, + "artifacts", + "deep_discovery", + ); + await mkdir(settingsDirectory, { recursive: true }); + await writeFile( + join(settingsDirectory, "execution-settings.json"), + JSON.stringify({ + version: 1, + settings: { codexHome: row.home }, + }), + ); + await fixture(currentHome, `${row.id}-parent`, row.parent); + const discovery = await fixture( + row.home, + `${row.id}-discovery`, + row.discovery, + ); + await fixture(row.home, `${row.id}-reducer`, row.reducer); + await fixture(row.home, `${row.id}-unrelated`, 10_000); + // Repeated receipt identity after reconnect must remain one charge. + const duplicate = (await readFile(discovery, "utf8")) + .trim() + .split("\n") + .at(-1)!; + await appendFile(discovery, duplicate + "\n"); + const attribution = { + formatVersion: 1 as const, + workerCodexHome: row.home, + executionThreadIds: [`${row.id}-discovery`, `${row.id}-reducer`], + owner: { + threadId: `${row.id}-parent`, + turnId: "scan-turn", + startedAt: at, + }, + startedAt: at, + completedAt: null, + }; + const tracker = new ScanCostTracker({ + codexHome: currentHome, + scanDirectory, + model: "gpt-5.6-sol", + maxCostUsd: 0.0002, + }); + tracker.setAttributionReader(async () => attribution); + tracker.start(`${row.id}-parent`); + trackers.push(tracker); + } + const initial = await Promise.all( + trackers.map((tracker) => tracker.refresh()), + ); + expect( + initial.map((snapshot) => tokenUsage(snapshot.usage)?.input_tokens), + ).toEqual([60, 63]); + expect(initial.map((snapshot) => snapshot.cost?.inputTokens)).toEqual([ + 60, 63, + ]); + expect(initial[0]!.cost!.estimatedUsd).toBeGreaterThan(0.0002); + const firstDirectory = join(currentHome, "scans", "one"); + await writeFile( + join( + firstDirectory, + "artifacts", + "deep_discovery", + "execution-settings.json", + ), + JSON.stringify({ version: 1, settings: { codexHome: secondHome } }), + ); + await unlink( + join( + currentHome, + "scans", + "two", + "artifacts", + "deep_discovery", + "execution-settings.json", + ), + ); + expect( + (await Promise.all(trackers.map((tracker) => tracker.refresh()))).map( + (snapshot) => snapshot.cost?.inputTokens, + ), + ).toEqual([60, 63]); + const rebuilt = new ScanCostTracker({ + codexHome: currentHome, + scanDirectory: firstDirectory, + model: "gpt-5.6-sol", + maxCostUsd: 0.0002, + }); + rebuilt.setAttributionReader(async () => ({ + formatVersion: 1, + workerCodexHome: firstHome, + executionThreadIds: [ + "one-discovery", + "one-reducer", + "one-missing-attempt", + ], + owner: { threadId: "one-parent", turnId: "scan-turn", startedAt: at }, + startedAt: at, + completedAt: null, + })); + rebuilt.start("one-parent"); + trackers.push(rebuilt); + expect((await rebuilt.refresh()).usage).toMatchObject({ + input_tokens: 60, + coverage: "partial", + }); + expect((await rebuilt.refresh()).cost!.estimatedUsd).toBeGreaterThan( + 0.0002, + ); + await fixture(firstHome, "one-missing-attempt", 7); + expect((await rebuilt.refresh()).cost?.inputTokens).toBe(67); + expect((await trackers[1]!.refresh()).cost?.inputTokens).toBe(63); + } finally { + await Promise.all(trackers.map((tracker) => tracker.stop())); + } + }); + + test("reads a recorded directory alias only once", async () => { + const home = await codexHome(); + const alias = join(await codexHome(), "recorded-home"); + await symlink( + home, + alias, + process.platform === "win32" ? "junction" : "dir", + ); + const scanDirectory = join(home, "scan"); + const directory = join(scanDirectory, "artifacts", "deep_discovery"); + await mkdir(directory, { recursive: true }); + await writeFile( + join(directory, "execution-settings.json"), + JSON.stringify({ version: 1, settings: { codexHome: alias } }), + ); + await writeSession(home, "worker", { input_tokens: 100, output_tokens: 0 }); + const events: ScanSessionEvent[] = []; + const tracker = new ScanCostTracker({ + codexHome: home, + scanDirectory, + model: "gpt-5.6-sol", + onSessionEvent: (event) => events.push(event), + }); + tracker.start("worker"); + try { + expect((await tracker.stop()).cost?.inputTokens).toBe(100); + expect(events).toHaveLength(2); + } finally { + await tracker.stop(); + } + }); + + test.each([ + ["identical, attribution: false", "identical", false], + ["identical, attribution: true", "identical", true], + ["prefix-first, attribution: false", "prefix-first", false], + ["prefix-first, attribution: true", "prefix-first", true], + ["prefix-last, attribution: false", "prefix-last", false], + ["prefix-last, attribution: true", "prefix-last", true], + ["truncated-first, attribution: false", "truncated-first", false], + ["truncated-first, attribution: true", "truncated-first", true], + ["truncated-last, attribution: false", "truncated-last", false], + ["truncated-last, attribution: true", "truncated-last", true], + ["tail-first, attribution: false", "tail-first", false], + ["tail-first, attribution: true", "tail-first", true], + ["tail-last, attribution: false", "tail-last", false], + ["tail-last, attribution: true", "tail-last", true], + ] as const)( + "prices copied response records (%s)", + async (_label, copy, attributed) => { + const home = await codexHome(); + const recordedHome = await codexHome(); + const scanDirectory = join(home, "scan"); + const directory = join(scanDirectory, "artifacts", "deep_discovery"); + await mkdir(directory, { recursive: true }); + await writeFile( + join(directory, "execution-settings.json"), + JSON.stringify({ version: 1, settings: { codexHome: recordedHome } }), + ); + const path = await writeSession(home, "worker", {}); + for (const [id, model, input, output] of [ + ["response-one", "gpt-5.6-sol", 100, 10], + ["response-two", "gpt-6-astra", 50, 5], + ] as const) { + await appendFile( + path, + JSON.stringify({ + type: "token_usage_record", + timestamp: "2026-09-01T00:00:02Z", + payload: { + thread_id: "worker", + turn_id: "turn", + response_id: id, + model, + usage: { input_tokens: input, output_tokens: output }, + }, + }) + "\n", + ); + } + await mkdir(join(recordedHome, "sessions")); + const copiedPath = join(recordedHome, "sessions", "copied-worker.jsonl"); + await cp(path, copiedPath); + const prefix = + (await readFile(path, "utf8")) + .trimEnd() + .split("\n") + .slice(0, -1) + .join("\n") + "\n"; + if (copy === "prefix-first") await writeFile(path, prefix); + if (copy === "prefix-last") await writeFile(copiedPath, prefix); + const truncated = prefix + '{"type":"token_usage_record"'; + if (copy === "truncated-first") await writeFile(path, truncated); + if (copy === "truncated-last") await writeFile(copiedPath, truncated); + if (copy === "tail-first") await appendFile(path, '{"type":"event_msg"'); + if (copy === "tail-last") + await appendFile(copiedPath, '{"type":"event_msg"'); + const tracker = new ScanCostTracker({ + codexHome: home, + scanDirectory, + model: "gpt-5.6-sol", + }); + tracker.setAttributionReader(async () => ({ + formatVersion: 1, + ...(attributed ? {} : { legacy: true as const }), + workerCodexHome: recordedHome, + executionThreadIds: ["worker"], + owner: { + threadId: null, + turnId: null, + startedAt: "2026-09-01T00:00:00Z", + }, + startedAt: "2026-09-01T00:00:00Z", + completedAt: null, + })); + tracker.start("worker"); + try { + const snapshot = await tracker.stop(); + expect(snapshot.usage).toMatchObject({ + input_tokens: 150, + output_tokens: 15, + total_tokens: 165, + }); + expect(snapshot.usage).not.toMatchObject({ coverage: "partial" }); + expect( + Object.fromEntries( + snapshot.cost!.modelCosts!.map((part) => [ + part.model, + [part.inputTokens, part.outputTokens], + ]), + ), + ).toEqual({ + "gpt-5.6-sol": [100, 10], + "gpt-6-astra": [50, 5], + }); + } finally { + await tracker.stop(); + } + }, + ); +}); + +test.each( + [false, true].flatMap((fullInArchive) => + [false, true].flatMap((receipt) => + [false, true].map((attributed) => ({ + fullInArchive, + receipt, + attributed, + })), + ), + ), +)( + "keeps corrected usage from the complete rollout copy: %j", + async ({ fullInArchive, receipt, attributed }) => { + const home = await codexHome(); + const at = "2026-09-01T00:00:02Z"; + const usage = (cached: number) => ({ + input_tokens: 1_000_000, + cached_input_tokens: cached, + cache_write_input_tokens: 0, + output_tokens: 0, + }); + const counter = (cached: number) => ({ + type: "event_msg", + timestamp: at, + payload: { + type: "token_count", + info: { total_token_usage: usage(cached) }, + }, + }); + const prefix = + jsonLines([ + { type: "session_meta", payload: { id: "worker" } }, + { + type: "turn_context", + timestamp: at, + payload: { model: "gpt-5.6-sol", turn_id: "turn" }, + }, + counter(0), + ]) + "\n"; + const correction = receipt + ? { + type: "token_usage_record", + timestamp: at, + payload: { + thread_id: "worker", + turn_id: "turn", + response_id: "corrected-receipt", + model: "gpt-5.6-sol", + usage: usage(900_000), + thread_token_usage: usage(900_000), + }, + } + : counter(900_000); + const active = join(home, "sessions"); + const archive = join(home, "archived_sessions"); + await mkdir(active); + await mkdir(archive); + const fullPath = join(fullInArchive ? archive : active, "full.jsonl"); + const prefixPath = join(fullInArchive ? active : archive, "prefix.jsonl"); + await writeFile(fullPath, prefix + JSON.stringify(correction) + "\n"); + const observed: number[] = []; + const createTracker = () => { + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + maxCostUsd: 2, + onCost: (cost) => observed.push(cost.estimatedUsd), + onCostLowerBound: (cost) => observed.push(cost.estimatedUsd), + }); + if (attributed) + tracker.setAttributionReader(async () => ({ + formatVersion: 1, + executionThreadIds: [], + owner: { threadId: "worker", turnId: "turn", startedAt: at }, + startedAt: at, + completedAt: null, + })); + tracker.start("worker"); + return tracker; + }; + const tracker = createTracker(); + let reloaded: ScanCostTracker | undefined; + try { + const baseline = await tracker.refresh(); + expect(baseline.cost?.estimatedUsd).toBeCloseTo(0.76, 10); + await writeFile(prefixPath, prefix); + expect((await tracker.refresh()).cost).toEqual(baseline.cost); + reloaded = createTracker(); + const cold = await reloaded.stop(usage(0)); + expect(cold.cost).toEqual(baseline.cost); + expect(cold.usage).toMatchObject(usage(900_000)); + expect(observed.length).toBeGreaterThan(0); + expect(observed.every((cost) => cost < 2)).toBe(true); + } finally { + await tracker.stop(); + await reloaded?.stop(); + } + }, +); + +test.each( + [false, true].flatMap((fullInArchive) => + [false, true].map((attributed) => ({ fullInArchive, attributed })), + ), +)( + "retains the complete copy's counter lower bound with missing receipts: %j", + async ({ fullInArchive, attributed }) => { + const home = await codexHome(); + const at = "2026-09-01T00:00:02Z"; + const usage = (input: number) => ({ + input_tokens: input, + cached_input_tokens: 0, + cache_write_input_tokens: 0, + output_tokens: 0, + }); + const counter = (input: number) => ({ + type: "event_msg", + timestamp: at, + payload: { + type: "token_count", + info: { total_token_usage: usage(input) }, + }, + }); + const prefix = [ + { type: "session_meta", payload: { id: "worker" } }, + { + type: "turn_context", + timestamp: at, + payload: { model: "gpt-5.6-sol", turn_id: "turn" }, + }, + counter(1_000_000), + ]; + const complete = [ + ...prefix, + { + type: "token_usage_record", + timestamp: at, + payload: { + thread_id: "worker", + turn_id: "turn", + response_id: "partial-receipt", + model: "gpt-5.6-sol", + usage: usage(600_000), + thread_token_usage: usage(1_000_000), + }, + }, + counter(1_300_000), + ]; + const active = join(home, "sessions"); + const archive = join(home, "archived_sessions"); + await mkdir(active); + await mkdir(archive); + const fullPath = join(fullInArchive ? archive : active, "full.jsonl"); + const prefixPath = join(fullInArchive ? active : archive, "prefix.jsonl"); + await writeFile(fullPath, jsonLines(complete) + "\n"); + const observed: number[] = []; + const createTracker = () => { + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + maxCostUsd: 5, + onCost: (cost) => observed.push(cost.estimatedUsd), + onCostLowerBound: (cost) => observed.push(cost.estimatedUsd), + }); + if (attributed) + tracker.setAttributionReader(async () => ({ + formatVersion: 1, + executionThreadIds: [], + owner: { threadId: "worker", turnId: "turn", startedAt: at }, + startedAt: at, + completedAt: null, + })); + tracker.start("worker"); + return tracker; + }; + const tracker = createTracker(); + let reloaded: ScanCostTracker | undefined; + try { + const baseline = await tracker.refresh(); + expect(baseline.usage).toMatchObject({ + input_tokens: 1_300_000, + total_tokens: 1_300_000, + coverage: "partial", + }); + expect(observed.at(-1)).toBeCloseTo(5.2, 10); + await writeFile(prefixPath, jsonLines(prefix) + "\n"); + expect((await tracker.refresh()).usage).toEqual(baseline.usage); + expect(observed.at(-1)).toBeCloseTo(5.2, 10); + reloaded = createTracker(); + expect((await reloaded.stop()).usage).toEqual(baseline.usage); + expect(observed.at(-1)).toBeCloseTo(5.2, 10); + expect(observed.every((cost) => cost > 5)).toBe(true); + } finally { + await tracker.stop(); + await reloaded?.stop(); + } + }, +); + +test.each([ + ["150000/15000 vs 100000/200000", 150_000, 15_000, 100_000, 200_000], + ["150000/15000 vs 100000/20000", 150_000, 15_000, 100_000, 20_000], + ["100000/50000 vs 110000/10000", 100_000, 50_000, 110_000, 10_000], + ["100000/50000 vs 200000/10000", 100_000, 50_000, 200_000, 10_000], + ["100000/10000 vs 100000/10000", 100_000, 10_000, 100_000, 10_000], +] as const)( + "preserves receipt pricing across divergent counters %s", + async (_label, input, output, counterInput, counterOutput) => { + const home = await codexHome(); + const path = await writeSession(home, "worker", {}); + const counts = (input: number, output: number) => ({ + input_tokens: input, + output_tokens: output, + cached_input_tokens: 0, + cache_write_input_tokens: 0, + reasoning_output_tokens: 0, + total_tokens: input + output, + }); + await appendFile( + path, + JSON.stringify({ + type: "token_usage_record", + timestamp: "2026-09-01T00:00:02Z", + payload: { + thread_id: "worker", + turn_id: "turn", + response_id: "receipt", + model: "gpt-5.6-sol", + usage: counts(input, output), + thread_token_usage: counts(counterInput, counterOutput), + }, + }) + + "\n" + + JSON.stringify({ + type: "event_msg", + timestamp: "2026-09-01T00:00:02Z", + payload: { + type: "token_count", + info: { total_token_usage: counts(counterInput, counterOutput) }, + }, + }) + + "\n", + ); + const lowerBounds: Readonly[] = []; + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + maxCostUsd: 0.8, + onCost: (cost) => lowerBounds.push(cost), + onCostLowerBound: (cost) => lowerBounds.push(cost), + }); + tracker.start("worker"); + try { + const snapshot = await tracker.stop(); + expect(snapshot.usage).toMatchObject({ + input_tokens: Math.max(input, counterInput), + output_tokens: Math.max(output, counterOutput), + }); + if (counterInput > input || counterOutput > output) + expect(snapshot.usage).toMatchObject({ coverage: "partial" }); + else expect(snapshot.usage).not.toMatchObject({ coverage: "partial" }); + const measured = tokenUsage(snapshot.usage); + expect(measured).not.toBeNull(); + expect(lowerBounds.length).toBeGreaterThan(0); + expect(lowerBounds.at(-1)!.estimatedUsd).toBeGreaterThanOrEqual( + estimateScanCost("gpt-5.6-sol", counts(input, output))!.estimatedUsd, + ); + } finally { + await tracker.stop(); + } + }, +); + +test.each(["standard", "pending-deep", "legacy-deep", "bound-deep"] as const)( + "uses bound log homes rather than draft settings: %s", + async (kind) => { + const home = await codexHome(); + const foreign = await codexHome(); + const scanDirectory = join(home, "scan"); + const settings = join(scanDirectory, "artifacts", "deep_discovery"); + await mkdir(settings, { recursive: true }); + await writeFile( + join(settings, "execution-settings.json"), + JSON.stringify({ version: 1, settings: { codexHome: foreign } }), + ); + for (const [directory, input] of [ + [home, 100], + [foreign, 900_000], + ] as const) { + const path = await writeSession( + directory, + "scan-thread", + { + input_tokens: input, + output_tokens: 0, + cached_input_tokens: 0, + cache_write_input_tokens: 0, + }, + { timestamp: "2026-09-01T00:00:02Z" }, + ); + const lines = (await readFile(path, "utf8")) + .trimEnd() + .split("\n") + .map((line) => ({ + ...JSON.parse(line), + timestamp: "2026-09-01T00:00:02Z", + })); + await writeFile(path, jsonLines(lines) + "\n"); + } + const tracker = new ScanCostTracker({ + codexHome: home, + scanDirectory, + model: "gpt-5.6-sol", + }); + if (kind !== "standard") + tracker.setAttributionReader(async () => + kind === "pending-deep" + ? null + : { + formatVersion: 1, + ...(kind === "legacy-deep" + ? { legacy: true as const } + : { workerCodexHome: foreign }), + executionThreadIds: ["scan-thread"], + owner: { + threadId: null, + turnId: null, + startedAt: "2026-09-01T00:00:00Z", + }, + startedAt: "2026-09-01T00:00:00Z", + completedAt: null, + }, + ); + tracker.start("scan-thread"); + try { + expect((await tracker.stop()).usage).toMatchObject({ + input_tokens: kind === "bound-deep" ? 900_000 : 100, + }); + } finally { + await tracker.stop(); + } + }, +); + +test.each(["no-reader", "pending-reader"] as const)( + "enforces dedicated parent cost before orchestration: %s", + async (kind) => { + const home = await codexHome(); + await writeSession(home, "new-parent", { + input_tokens: 100_000, + output_tokens: 0, + cached_input_tokens: 0, + cache_write_input_tokens: 0, + }); + const costs: Readonly[] = []; + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + maxCostUsd: 0.003, + onCost: (cost) => costs.push(cost), + onCostLowerBound: (cost) => costs.push(cost), + }); + if (kind === "pending-reader") + tracker.setAttributionReader(async () => null); + tracker.start("new-parent"); + try { + const snapshot = await tracker.stop(); + expect(snapshot.cost?.estimatedUsd).toBeGreaterThan(0.003); + expect(costs.length).toBeGreaterThan(0); + } finally { + await tracker.stop(); + } + }, +); + +test.each(["single", "identical", "prefix-first", "prefix-last"] as const)( + "forwards copied worker activities without replaying completed calls: %s", + async (copy) => { + const { home, worker } = await workerSessionFixture(); + const recordedHome = await codexHome(); + await mkdir(join(recordedHome, "sessions")); + const second = join(recordedHome, "sessions", "worker-copy.jsonl"); + const call = (id: string) => ({ + type: "response_item", + payload: { + type: "function_call", + name: "exec_command", + call_id: id, + arguments: JSON.stringify({ cmd: "rg -n source routes/login.ts" }), + }, + }); + const done = (id: string) => ({ + type: "response_item", + payload: { type: "function_call_output", call_id: id }, + }); + const prefix = + (await readFile(worker, "utf8")) + jsonLines([call("first")]) + "\n"; + const complete = prefix + jsonLines([done("first")]) + "\n"; + await writeFile(worker, copy === "prefix-first" ? prefix : complete); + if (copy !== "single") + await writeFile(second, copy === "prefix-last" ? prefix : complete); + const activities: ScanActivity[] = []; + const tracker = new ScanCostTracker({ + codexHome: home, + scanDirectory: join(home, "scan"), + model: "gpt-5.6-sol", + repository: "/code/juice-shop", + onActivity: (activity) => activities.push(activity), + }); + tracker.setAttributionReader(async () => ({ + formatVersion: 1, + legacy: true, + workerCodexHome: recordedHome, + executionThreadIds: [], + owner: { + threadId: "scan-thread", + turnId: null, + startedAt: "2026-09-01T00:00:00Z", + }, + startedAt: "2026-09-01T00:00:00Z", + completedAt: null, + })); + tracker.start("scan-thread"); + try { + await tracker.refresh(); + expect(activities.map(({ id, status }) => [id, status])).toEqual([ + ["worker-thread:first", "running"], + ["worker-thread:first", "completed"], + ]); + await writeFile(worker, complete); + if (copy !== "single") await writeFile(second, complete); + await tracker.refresh(); + expect(activities).toHaveLength(2); + const later = jsonLines([call("second"), done("second")]) + "\n"; + await appendFile(worker, later); + await tracker.refresh(); + if (copy !== "single") await appendFile(second, later); + await tracker.stop(); + expect(activities.map(({ id, status }) => [id, status])).toEqual([ + ["worker-thread:first", "running"], + ["worker-thread:first", "completed"], + ["worker-thread:second", "running"], + ["worker-thread:second", "completed"], + ]); + } finally { + await tracker.stop(); + } + }, +); + +test.each([0, 100])( + "ignores only zero unpriced counter buckets: %i", + async (unpricedTokens) => { + const home = await codexHome(); + const path = await writeSession(home, "owner", { + input_tokens: unpricedTokens, + output_tokens: 0, + }); + await appendFile( + path, + jsonLines([ + { + type: "turn_context", + timestamp: "2026-09-01T00:00:02Z", + payload: { turn_id: "turn", model: "gpt-5.6-sol" }, + }, + { + type: "event_msg", + timestamp: "2026-09-01T00:00:02Z", + payload: { + type: "token_count", + info: { + total_token_usage: { + input_tokens: unpricedTokens + 1000, + output_tokens: 0, + }, + }, + }, + }, + ]) + "\n", + ); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + tracker.start("owner"); + try { + const snapshot = await tracker.refresh(); + expect(tokenUsage(snapshot.usage)?.input_tokens).toBe( + unpricedTokens + 1000, + ); + if (unpricedTokens === 0) + expect(snapshot.cost?.estimatedUsd).toBeCloseTo(0.004, 10); + else expect(snapshot.cost).toBeNull(); + } finally { + await tracker.stop(); + } + }, +); + +test.each(["other turn", "outside window"])( + "keeps owned legacy model before an unrelated first receipt: %s", + async (kind) => { + const home = await codexHome(); + const path = await writeSession(home, "owner", {}); + const at = "2026-09-01T00:00:02Z"; + await appendFile( + path, + jsonLines([ + { + type: "turn_context", + timestamp: at, + payload: { turn_id: "turn", model: "gpt-5.6-sol" }, + }, + { + type: "event_msg", + timestamp: at, + payload: { + type: "token_count", + info: { + total_token_usage: { input_tokens: 1000, output_tokens: 0 }, + }, + }, + }, + { + type: "token_usage_record", + timestamp: kind === "outside window" ? "2026-09-01T00:00:04Z" : at, + payload: { + thread_id: "owner", + turn_id: kind === "other turn" ? "other" : "turn", + response_id: "unrelated-response", + model: "gpt-6-astra", + usage: { input_tokens: 50, output_tokens: 0 }, + }, + }, + ]) + "\n", + ); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + tracker.setAttributionReader(async () => ({ + formatVersion: 1, + executionThreadIds: [], + owner: { threadId: "owner", turnId: "turn", startedAt: at }, + startedAt: at, + completedAt: "2026-09-01T00:00:03Z", + })); + tracker.start("owner"); + try { + const snapshot = await tracker.refresh(); + expect(tokenUsage(snapshot.usage)?.input_tokens).toBe(1000); + expect(snapshot.cost?.estimatedUsd).toBeCloseTo(0.004, 10); + expect(snapshot.usage).not.toMatchObject({ coverage: "partial" }); + } finally { + await tracker.stop(); + } + }, +); + +test.each(["sdk-owner", "unbound-owner", "unknown-worker"] as const)( + "prices known completion receipts without a rollout: %s", + async (kind) => { + const home = await codexHome(); + const observed: Readonly[] = []; + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + maxCostUsd: 0.003, + onCost: (cost) => observed.push(cost), + onCostLowerBound: (cost) => observed.push(cost), + }); + if (kind !== "unbound-owner") + tracker.setAttributionReader(async () => ({ + formatVersion: 1, + executionThreadIds: [kind === "unknown-worker" ? "worker" : "owner"], + owner: { + threadId: "owner", + turnId: "turn", + startedAt: "2026-09-01T00:00:00Z", + dedicated: true, + }, + startedAt: "2026-09-01T00:00:00Z", + completedAt: null, + })); + tracker.start("owner"); + try { + const receipt = { input_tokens: 1000, output_tokens: 0 }; + if (kind === "unknown-worker") tracker.recordUsage(receipt, "worker"); + const snapshot = await tracker.stop( + kind === "unknown-worker" ? undefined : receipt, + ); + expect(tokenUsage(snapshot.usage)?.input_tokens).toBe(1000); + if (kind === "unknown-worker") { + expect(snapshot.cost).toBeNull(); + expect(observed).toHaveLength(0); + } else { + expect(snapshot.cost?.estimatedUsd).toBeCloseTo(0.004, 10); + expect(observed.at(-1)?.estimatedUsd).toBeGreaterThan(0.003); + } + } finally { + await tracker.stop(); + } + }, +); + +test("cost polls reuse attribution until SQLite changes and refresh it at final accounting", async () => { + const home = await codexHome(); + const databasePath = join(home, "workbench.sqlite3"); + const database = new Database(databasePath); + database.run("PRAGMA journal_mode=WAL"); + database.run("CREATE TABLE attribution (value TEXT)"); + const attribution: ScanExecutionAttribution = { + formatVersion: 1, + executionThreadIds: [], + owner: { + threadId: "parent", + turnId: "parent-turn", + startedAt: "2026-01-01T00:00:00Z", + dedicated: true, + }, + startedAt: "2026-01-01T00:00:00Z", + completedAt: null, + }; + database + .query("INSERT INTO attribution VALUES (?)") + .run(JSON.stringify(attribution)); + for (const [thread, input] of [ + ["parent", 100], + ["worker", 200], + ] as const) { + const path = await writeSession(home, thread, {}); + await appendFile( + path, + jsonLines([ + { + type: "turn_context", + timestamp: "2026-09-01T00:00:02Z", + payload: { turn_id: `${thread}-turn`, model: "gpt-5.6-sol" }, + }, + { + type: "token_usage_record", + timestamp: "2026-09-01T00:00:02Z", + payload: { + thread_id: thread, + turn_id: `${thread}-turn`, + response_id: `${thread}-response`, + model: "gpt-5.6-sol", + usage: { input_tokens: input, output_tokens: 0 }, + }, + }, + ]) + "\n", + ); + } + let reads = 0; + const reader = cachedScanAttributionReader(home, async () => { + reads++; + return JSON.parse( + ( + database.query("SELECT value FROM attribution").get() as { + value: string; + } + ).value, + ) as ScanExecutionAttribution; + }); + const tracker = costTracker(home); + tracker.setAttributionReader(reader); + tracker.start("parent"); + try { + for (let poll = 0; poll < 8; poll++) + expect((await tracker.refresh()).usage).toMatchObject({ + input_tokens: 100, + }); + expect(reads).toBe(1); + attribution.executionThreadIds.push("worker"); + database + .query("UPDATE attribution SET value = ?") + .run(JSON.stringify(attribution)); + expect((await tracker.refresh()).usage).toMatchObject({ + input_tokens: 300, + }); + expect(reads).toBe(2); + database.run("PRAGMA wal_checkpoint(TRUNCATE)"); + expect((await tracker.refresh()).usage).toMatchObject({ + input_tokens: 300, + }); + expect(reads).toBe(3); + for (let poll = 0; poll < 8; poll++) await tracker.refresh(); + expect(reads).toBe(3); + expect((await tracker.stop()).usage).toMatchObject({ input_tokens: 300 }); + expect(reads).toBe(4); + } finally { + await tracker.stop(); + database.close(); + } +}); + +test("attribution caching notices database replacement and writes during a read", async () => { + const directory = await codexHome(); + const path = join(directory, "workbench.sqlite3"); + const next = join(directory, "replacement.sqlite3"); + await writeFile(path, "first"); + let changeDuringRead = false; + let reads = 0; + const reader = cachedScanAttributionReader(directory, async () => { + reads++; + const before = await readFile(path, "utf8"); + if (changeDuringRead) { + changeDuringRead = false; + await appendFile(path, "-committed-during-read"); + } + return { startedAt: before } as ScanExecutionAttribution; + }); + expect((await reader())?.startedAt).toBe("first"); + expect((await reader())?.startedAt).toBe("first"); + expect(reads).toBe(1); + await writeFile(next, "replacement"); + await rename(next, path); + changeDuringRead = true; + expect((await reader())?.startedAt).toBe("replacement"); + expect((await reader())?.startedAt).toBe("replacement-committed-during-read"); + expect(reads).toBe(3); +}); diff --git a/sdk/typescript/tests-ts/deep-finalization.test.ts b/sdk/typescript/tests-ts/deep-finalization.test.ts new file mode 100644 index 0000000000..d3be776106 --- /dev/null +++ b/sdk/typescript/tests-ts/deep-finalization.test.ts @@ -0,0 +1,1472 @@ +import { execFileSync, spawn } from "node:child_process"; +import { appendFile, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { afterEach, expect, test } from "bun:test"; +import type { ThreadEvent } from "@openai/codex-sdk"; +import { + ScanCostLimitExceededError, + ScanInterruptedError, +} from "../src/errors.js"; +import { prepareScanArtifactRestorer, runWorkbench } from "../src/runtime.js"; +import { resumeSelectedDeepScan } from "../src/deep-scan-finalization.js"; +import { TestClient } from "./support/api-client.js"; +import { completedEvents, preparedRuntime } from "./support/api-events.js"; +import { PLUGIN_ROOT } from "./plugin-root.js"; +import { createApiTestFixtures } from "./support/temporary-directories.js"; + +const { temporaryDirectory, cleanup } = createApiTestFixtures(); +afterEach(cleanup); +const threadId = "1af317a1-c9ed-4c73-b428-cb0d160cf8e8"; +const followUp = "Explain the selected finding."; + +for (const boundary of ["registration", "stream-start"] as const) { + test(`SDK cancels registered Deep Scan before first thread event: ${boundary}`, async () => { + const root = await temporaryDirectory(); + const repository = join(root, "repository"); + const scanDir = join(root, "scan"); + const codexHome = join(root, "codex-home"); + await Promise.all([ + mkdir(repository), + mkdir(scanDir, { mode: 0o700 }), + mkdir(codexHome), + ]); + await writeFile(join(repository, "source.py"), "# Synthetic source\n"); + const environment = { + ...process.env, + CODEX_HOME: codexHome, + CODEX_SECURITY_STATE_DIR: join(root, "state"), + }; + const cancellation = new AbortController(); + const reason = new Error( + "Synthetic cancellation before first thread event", + ); + const commands: string[][] = []; + let scanId = ""; + let savedOptions: Parameters[0]; + let startedTurns = 0; + const client = new TestClient( + {}, + { + environment, + prepareRuntime: async () => ({ + ...preparedRuntime(codexHome), + environment, + }), + resolvePluginPython: async () => "python3", + prepareOutputDir: async () => scanDir, + runWorkbench: async (options, args, input) => { + savedOptions = options; + commands.push([...args]); + const result = await runWorkbench(options, args, input); + if (args[0] === "register-cli-scan") { + scanId = result["scanId"] as string; + if (boundary === "registration") cancellation.abort(reason); + } + return result; + }, + createCodex: () => ({ + startThread: () => ({ + id: null, + runStreamed: async () => { + startedTurns++; + cancellation.abort(reason); + throw reason; + }, + }), + }), + }, + ); + try { + const error = await client + .run(repository, { + mode: "deep", + signal: cancellation.signal, + postScanPrompt: followUp, + }) + .catch((error: unknown) => error); + expect(error).toBeInstanceOf(ScanInterruptedError); + expect((error as ScanInterruptedError).cause).toBe(reason); + const stopped = await runWorkbench( + { ...savedOptions!, signal: undefined }, + ["get-scan", "--scan-id", scanId], + ); + expect(stopped["scan"]).toMatchObject({ + progress: { status: "canceled" }, + }); + expect(commands.filter((args) => args[0] === "cancel-scan")).toEqual([ + ["cancel-scan", "--scan-id", scanId], + ]); + expect(commands.some((args) => args[0] === "fail-scan")).toBe(false); + expect(commands.some((args) => args[0] === "set-scan-thread")).toBe( + false, + ); + expect(startedTurns).toBe(boundary === "registration" ? 0 : 1); + } finally { + await client.close(); + } + }, 30_000); +} + +const outcomes = [ + "failed", + "completed", + "completed-owner-suffix", + "completed-owner-explicit", + "restart", + "restart-missing-logs", + "restart-deleted-target", + "restart-changed-target", + "restart-corrupt-selection", + "restart-unavailable-runtime", + "restart-canceled-runtime", + "canceled-before-publication", + "canceled-during-publication", + "canceled-during-resumed-publication", + "published-before-cancellation", + "closed-during-publication", + "budget-during-publication", + "budget-after-deep-finish", + "budget-during-resumed-publication", + "closed-during-resumed-publication", + "lost-completion-response", + "lost-completion-response-followup-canceled", + "completion-before-commit-fails", + "followup-canceled", +] as const; +type BudgetCompletionFault = "lost" | "before-commit" | "lost-and-canceled"; +const cases: { + outcome: (typeof outcomes)[number]; + budgetCompletionFault?: BudgetCompletionFault; + initialResumeUsage?: boolean; + unpricedUsage?: boolean; + statusReadFault?: boolean; + resumeStatusFault?: "transport" | "missing-selection" | "stopped"; + cancellationFault?: "status-read" | "deep-state-read" | "cancel-response"; + historicalFindings?: boolean; + changedHistoricalFollowUp?: boolean; + completeBeforePreparation?: boolean; +}[] = [ + ...outcomes.map((outcome) => ({ outcome })), + { outcome: "restart", historicalFindings: true }, + { + outcome: "restart", + historicalFindings: true, + completeBeforePreparation: true, + }, + { outcome: "completed", historicalFindings: true }, + { + outcome: "completed", + historicalFindings: true, + changedHistoricalFollowUp: true, + }, + { outcome: "restart-corrupt-selection", historicalFindings: true }, + { outcome: "restart", statusReadFault: true }, + { outcome: "restart", resumeStatusFault: "transport" }, + { outcome: "restart", resumeStatusFault: "missing-selection" }, + { outcome: "restart", resumeStatusFault: "stopped" }, + { outcome: "restart-unavailable-runtime", initialResumeUsage: true }, + { + outcome: "restart-unavailable-runtime", + initialResumeUsage: true, + unpricedUsage: true, + }, + { + outcome: "restart-unavailable-runtime", + initialResumeUsage: true, + budgetCompletionFault: "lost", + }, + ...( + [ + "budget-during-publication", + "budget-after-deep-finish", + "budget-during-resumed-publication", + ] as const + ).flatMap((outcome) => [ + { outcome, unpricedUsage: true }, + { outcome, unpricedUsage: true, budgetCompletionFault: "lost" as const }, + ]), + ...( + [ + "budget-during-publication", + "budget-after-deep-finish", + "budget-during-resumed-publication", + ] as const + ).map((outcome) => ({ outcome, budgetCompletionFault: "lost" as const })), + { + outcome: "budget-after-deep-finish", + budgetCompletionFault: "before-commit", + }, + { + outcome: "budget-after-deep-finish", + budgetCompletionFault: "lost-and-canceled", + }, + { + outcome: "canceled-during-publication", + cancellationFault: "status-read", + }, + { + outcome: "canceled-during-publication", + cancellationFault: "cancel-response", + }, + { + outcome: "canceled-during-publication", + cancellationFault: "deep-state-read", + }, + { + outcome: "canceled-during-resumed-publication", + cancellationFault: "deep-state-read", + }, + { + outcome: "canceled-before-publication", + cancellationFault: "deep-state-read", + }, + { + outcome: "canceled-during-resumed-publication", + initialResumeUsage: true, + }, +]; +for (const { + outcome, + budgetCompletionFault, + cancellationFault, + initialResumeUsage, + unpricedUsage, + statusReadFault, + resumeStatusFault, + historicalFindings, + changedHistoricalFollowUp, + completeBeforePreparation, +} of cases) { + const resumedStop = outcome.includes("-resumed-"); + const restart = outcome.startsWith("restart") || resumedStop; + const unavailableStartup = + outcome.startsWith("restart-") && outcome !== "restart-changed-target"; + const recoveryBudget = + outcome === "restart-unavailable-runtime" && initialResumeUsage === true; + const closed = outcome.startsWith("closed-"); + const budgeted = outcome.startsWith("budget-"); + const canceledFollowUp = outcome.endsWith("followup-canceled"); + const loseCompletionResponse = outcome.startsWith("lost-completion-response"); + const name = + outcome === "followup-canceled" + ? "SDK preserves a selected aggregate when its follow-up is canceled" + : `SDK handles selected aggregate: ${outcome}${budgetCompletionFault ? ` (budget completion ${budgetCompletionFault})` : ""}${cancellationFault ? ` (cancellation ${cancellationFault})` : ""}${statusReadFault ? " (selection status unavailable)" : ""}${resumeStatusFault ? ` (resume status ${resumeStatusFault})` : ""}${initialResumeUsage ? " (initial resume cost)" : ""}${unpricedUsage ? " (unpriced remainder)" : ""}${historicalFindings ? " (historical findings)" : ""}${changedHistoricalFollowUp ? " (changed follow-up)" : ""}${completeBeforePreparation ? " (concurrent completion)" : ""}`; + const runCase = async () => { + const root = await temporaryDirectory(); + const repository = join(root, "repository"); + const scanDir = join(root, "scan"); + const stateDir = join(root, "state"); + const codexHome = unavailableStartup + ? join(stateDir, "codex-home") + : join(root, "codex-home"); + await Promise.all([ + mkdir(repository), + mkdir(scanDir, { mode: 0o700 }), + mkdir(codexHome, { recursive: true }), + ]); + await writeFile(join(repository, "extract.py"), "# Synthetic source\n"); + const environment = { + ...process.env, + CODEX_HOME: codexHome, + CODEX_SECURITY_STATE_DIR: stateDir, + }; + const cancellation = new AbortController(); + const parentError = new Error( + "Parent turn ended before its final completion tool call", + ); + let scanId = ""; + let workbenchOptions: Parameters[0]; + let publicationFails = restart && !statusReadFault; + let selectionReadLost = false; + let resumeStatusInjected = false; + let completedBeforePreparation = false; + let concurrentlyCompletedArtifacts: Buffer[] | undefined; + const resumeStatusError = new Error( + "Synthetic selected resume status unavailable", + ); + let completionReceiptLost = loseCompletionResponse; + let budgetReceiptLost = + budgetCompletionFault === "lost" || + budgetCompletionFault === "lost-and-canceled"; + let budgetTriggered = false; + let cancellationReadLost = false; + let lostCancellationDeepState: unknown; + const ownerSuffix = + outcome === "completed-owner-suffix" || + outcome === "completed-owner-explicit"; + let nativeOwnerStatus: unknown; + let originalFinalizationInput: unknown; + let selectedPath = ""; + let selectedBytes: Buffer; + let originalResumeSignal: AbortSignal | undefined; + let acceptedReport = ""; + let completedArtifacts: Buffer[] = []; + const modelInputs: string[] = []; + const commands: string[] = []; + const reportedCosts: number[] = []; + const usagePath = join( + codexHome, + "sessions", + "2026", + "01", + "01", + `rollout-${threadId}.jsonl`, + ); + const recordBudgetUsage = ( + inputTokens = 1_250, + outputTokens = 30, + timestamp = new Date().toISOString(), + ) => + appendFile( + usagePath, + JSON.stringify({ + timestamp, + type: "turn_context", + payload: { + turn_id: "synthetic-scan-turn", + model: "gpt-5.6-sol", + }, + }) + + "\n" + + JSON.stringify({ + timestamp, + type: "event_msg", + payload: { + type: "token_count", + info: { + total_token_usage: { + input_tokens: inputTokens, + cached_input_tokens: 200, + output_tokens: outputTokens, + }, + }, + }, + }) + + "\n" + + (unpricedUsage + ? [ + JSON.stringify({ + timestamp, + type: "turn_context", + payload: { + turn_id: "synthetic-scan-turn", + model: "synthetic-unpriced-model", + }, + }), + JSON.stringify({ + timestamp, + type: "event_msg", + payload: { + type: "token_count", + info: { + total_token_usage: { + input_tokens: 1_350, + cached_input_tokens: 200, + output_tokens: outputTokens, + }, + }, + }, + }), + "", + ].join("\n") + : ""), + ); + let closePromise: Promise | undefined; + let resumedStartup = false; + const warnings: string[] = []; + const makeClient = () => + new TestClient( + {}, + { + environment, + prepareScanArtifactRestorer, + prepareRuntime: async () => { + if (resumedStartup && outcome.endsWith("runtime")) + throw new Error("Synthetic original runtime unavailable"); + const runtime = preparedRuntime(codexHome); + const manifest = JSON.parse( + await readFile( + join(PLUGIN_ROOT, ".codex-plugin/plugin.json"), + "utf8", + ), + ); + return { + ...runtime, + environment, + persistentCredentialHome: true, + plugin: { ...runtime.plugin, version: manifest.version }, + }; + }, + resolvePluginPython: async () => "python3", + prepareOutputDir: async () => scanDir, + runWorkbench: async (options, args, input) => { + workbenchOptions = options; + if (args[0] === "get-cli-scan-resume") + originalResumeSignal = options.signal; + commands.push(args[0]!); + if ( + completeBeforePreparation && + resumedStartup && + !completedBeforePreparation && + args[0] === "prepare-scan-completion" + ) { + // Another resume can complete after registration, before this call acquires its lock. + completedBeforePreparation = true; + await runWorkbench(options, ["complete-scan", ...args.slice(1)]); + const saved = await runWorkbench(options, [ + "get-scan", + "--scan-id", + scanId, + ]); + expect(saved["scan"]).toMatchObject({ + progress: { status: "complete" }, + }); + concurrentlyCompletedArtifacts = await Promise.all( + [ + "scan-manifest.json", + "findings.json", + "coverage.json", + "report.md", + ].map((name) => readFile(join(scanDir, name))), + ); + } + if ( + args[0] === "get-deep-scan" && + resumedStartup && + resumeStatusFault && + !resumeStatusInjected + ) { + resumeStatusInjected = true; + if (resumeStatusFault === "transport") throw resumeStatusError; + if (resumeStatusFault === "missing-selection") { + const result = await runWorkbench(options, args, input); + delete (result["deepScan"] as Record)[ + "finalizationInput" + ]; + return result; + } + await runWorkbench(options, [ + "cancel-scan", + "--scan-id", + scanId, + "--thread-id", + threadId, + ]); + } + if ( + args[0] === "get-deep-scan" && + statusReadFault && + originalFinalizationInput !== undefined && + !selectionReadLost + ) { + selectionReadLost = true; + throw new Error("Synthetic SQLite status lookup failure"); + } + if ( + args[0] === "get-scan" && + cancellation.signal.aborted && + cancellationFault === "status-read" + ) { + throw new Error("Synthetic lost cancellation status response"); + } + if (args[0] === "write-scan-draft" && publicationFails) { + publicationFails = false; + throw new Error("Synthetic publication write failure"); + } + if ( + args[0] === "complete-scan" && + outcome === "completion-before-commit-fails" + ) + throw new Error("Synthetic completion failure before commit"); + if ( + args[0] === "complete-budget-exhausted-scan" && + budgetCompletionFault === "before-commit" + ) + throw new Error( + "Synthetic budget completion failure before commit", + ); + const result = await runWorkbench(options, args, input); + if ( + args[0] === "get-deep-scan" && + cancellation.signal.aborted && + cancellationFault === "deep-state-read" && + !cancellationReadLost + ) { + cancellationReadLost = true; + lostCancellationDeepState = result["deepScan"]; + throw new Error( + "Synthetic lost cancellation Deep state response", + ); + } + if ( + args[0] === "cancel-scan" && + cancellationFault === "cancel-response" + ) { + completedArtifacts = await Promise.all( + ["report.md", "scan-manifest.json"].map((name) => + readFile(join(scanDir, name)), + ), + ); + throw new Error("Synthetic lost cancellation response"); + } + if ( + args[0] === "complete-budget-exhausted-scan" && + budgetReceiptLost + ) { + budgetReceiptLost = false; + if (budgetCompletionFault === "lost-and-canceled") { + completedArtifacts = await Promise.all( + ["report.md", "scan-manifest.json"].map((name) => + readFile(join(scanDir, name)), + ), + ); + cancellation.abort( + "Synthetic cancellation after budget completion", + ); + } + throw Object.assign( + new Error("Synthetic lost budget completion response"), + { code: "ETIMEDOUT" }, + ); + } + if (args[0] === "complete-scan" && completionReceiptLost) { + completionReceiptLost = false; + throw new Error("Synthetic lost completion response"); + } + if ( + !budgetTriggered && + ((args[0] === "write-scan-draft" && + (outcome === "budget-during-publication" || + outcome === "budget-during-resumed-publication")) || + (args[0] === "finish-deep-scan" && + outcome === "budget-after-deep-finish")) + ) { + budgetTriggered = true; + await recordBudgetUsage(); + await new Promise((resolve) => { + if (options.signal?.aborted) resolve(); + else + options.signal!.addEventListener("abort", () => resolve(), { + once: true, + }); + }); + } + if (args[0] === "write-scan-draft" && closed) { + closePromise = client.close(); + } + if ( + args[0] === "write-scan-draft" && + (outcome === "canceled-during-publication" || + outcome === "canceled-during-resumed-publication" || + outcome === "restart-canceled-runtime") + ) { + const published = await runWorkbench(options, [ + "get-deep-scan", + "--scan-id", + scanId, + "--thread-id", + threadId, + ]); + const bound = (published["deepScan"] as Record)[ + "finalizationInput" + ]; + expect(bound).toMatchObject( + originalFinalizationInput as Record, + ); + expect(bound).toMatchObject({ + publicationSha256: expect.stringMatching(/^[a-f0-9]{64}$/u), + }); + originalFinalizationInput = bound; + if (initialResumeUsage) { + expect(originalResumeSignal?.reason).toBeInstanceOf( + ScanCostLimitExceededError, + ); + expect(options.signal?.aborted).toBe(false); + } + cancellation.abort("Synthetic user cancellation"); + } + if ( + args[0] === "complete-scan" && + outcome === "published-before-cancellation" + ) { + cancellation.abort( + "Synthetic user cancellation after completion", + ); + } + if (args[0] === "register-cli-scan") + scanId = result["scanId"] as string; + return result; + }, + createCodex: () => { + const thread = { + id: threadId, + async runStreamed(input: string) { + modelInputs.push(input); + if (input === followUp) { + if (changedHistoricalFollowUp) { + const file = join(scanDir, "findings.json"); + const document = JSON.parse(await readFile(file, "utf8")); + document.findings[0].root_cause.summary = + "New unaccepted model detail."; + await writeFile(file, JSON.stringify(document)); + } + if (canceledFollowUp) { + const reportPath = join(scanDir, "report.md"); + acceptedReport = await readFile(reportPath, "utf8"); + expect(acceptedReport).toContain( + "Validate the resolved destination", + ); + await writeFile( + reportPath, + "Incomplete follow-up report.\n", + ); + cancellation.abort( + "Synthetic cancellation during follow-up", + ); + } + return { events: completedEvents(threadId) }; + } + expect(modelInputs.length).toBe(1); + async function* events(): AsyncGenerator { + yield { type: "thread.started", thread_id: threadId }; + await runWorkbench(workbenchOptions, [ + "begin-deep-scan", + "--scan-id", + scanId, + "--thread-id", + threadId, + ]); + const draft = { + scanId, + complete: true, + findings: [ + { + ruleId: "path-traversal.archive", + title: "Unsafe archive extraction", + summary: + "An untrusted entry reaches a filesystem write.", + severity: { level: "high" }, + confidence: { + level: "high", + rationale: "Source evidence.", + }, + taxonomy: { + category: "path-traversal", + cwe: ["CWE-22"], + }, + locations: [{ path: "extract.py", startLine: 1 }], + remediation: + "Validate the resolved destination before writing.", + provenance: { + source: "local_plugin", + candidateId: "archive-entry", + }, + ...(historicalFindings + ? { + root_cause: { + summary: + "The original destination is unchecked.", + evidence_refs: ["legacy-proof"], + }, + code_evidence: [ + { + id: "legacy-proof", + code: "write(destination)", + }, + ], + } + : {}), + }, + ], + coverage: { + completeness: "partial", + surfaces: [], + explicitExclusions: [], + deferred: [ + { + id: "dependency", + reason: "A dependency remains unreviewed.", + }, + ], + }, + }; + const seeded = JSON.parse( + execFileSync( + "python3", + [ + fileURLToPath( + new URL( + "./fixtures/selected-deep-scan.py", + import.meta.url, + ), + ), + ], + { + input: JSON.stringify({ + scanId, + scanDir, + database: join(stateDir, "workbench.sqlite3"), + draft, + }), + encoding: "utf8", + env: environment, + }, + ), + ); + // Exercise the dedicated function bridge, without extending CLI arguments. + const selectionOutput = execFileSync( + "python3", + [ + "-c", + "import runpy, sys; script = sys.argv.pop(1); runpy.run_path(script)['main'](select_finalization=True)", + join(PLUGIN_ROOT, "scripts/workbench_db.py"), + "finish-deep-scan", + "--scan-id", + scanId, + "--coordinator-generation", + "2", + "--terminal-reason", + loseCompletionResponse ? "capped" : "saturated", + "--manifest-path", + join(scanDir, "scan-manifest.json"), + ], + { + input: JSON.stringify({ + resultPath: seeded.resultPath, + }), + encoding: "utf8", + env: environment, + }, + ); + originalFinalizationInput = + JSON.parse(selectionOutput).deepScan.finalizationInput; + selectedPath = join( + scanDir, + (originalFinalizationInput as { resultPath: string }) + .resultPath, + ); + selectedBytes = await readFile(selectedPath); + const sessions = join( + codexHome, + "sessions", + "2026", + "01", + "01", + ); + await mkdir(sessions, { recursive: true }); + await writeFile( + join(sessions, `rollout-${threadId}.jsonl`), + JSON.stringify({ + timestamp: new Date().toISOString(), + type: "session_meta", + payload: { id: threadId, cwd: scanDir }, + }) + "\n", + ); + if (ownerSuffix) { + await recordBudgetUsage(); + await resumeSelectedDeepScan({ + scanId, + threadId, + pluginRoot: PLUGIN_ROOT, + runWorkbench: (args) => + runWorkbench(workbenchOptions, args), + signal: cancellation.signal, + }); + await rejoinSelectedScanThroughMcp( + environment, + scanId, + outcome === "completed-owner-explicit", + ); + const saved = await runWorkbench(workbenchOptions, [ + "get-scan", + "--scan-id", + scanId, + ]); + const scan = saved["scan"] as { + progress: { status: string }; + executionAttribution: { completedAt: string | null }; + }; + nativeOwnerStatus = scan.progress.status; + const completedAt = scan.executionAttribution.completedAt; + await recordBudgetUsage( + 2_500, + 60, + new Date( + Math.max( + Date.now(), + completedAt ? Date.parse(completedAt) + 1 : 0, + ), + ).toISOString(), + ); + } + if (outcome === "canceled-before-publication") + cancellation.abort("Synthetic user cancellation"); + if (outcome === "completed" || ownerSuffix) { + yield { + type: "turn.completed", + usage: { + input_tokens: ownerSuffix ? 2_500 : 0, + cached_input_tokens: ownerSuffix ? 200 : 0, + cache_write_input_tokens: 0, + reasoning_output_tokens: 0, + output_tokens: ownerSuffix ? 60 : 0, + }, + }; + } else { + throw parentError; + } + } + return { events: events() }; + }, + }; + return { + startThread: () => thread, + resumeThread: (id: string) => { + expect(id).toBe(threadId); + return thread; + }, + }; + }, + }, + ); + let client = makeClient(); + // Native usage polling is unref'ed; the transport double has no child process. + const keepAlive = setTimeout(() => {}, 30_000); + try { + if (restart) { + await expect( + client.run(repository, { + mode: "deep", + postScanPrompt: resumedStop ? undefined : followUp, + ...(budgeted || recoveryBudget ? { maxCostUsd: 0.004 } : {}), + }), + ).rejects.toThrow( + statusReadFault + ? parentError.message + : "Synthetic publication write failure", + ); + const pending = await runWorkbench(workbenchOptions!, [ + "get-deep-scan", + "--scan-id", + scanId, + "--thread-id", + threadId, + ]); + expect(pending["deepScan"]).toMatchObject({ + status: "running", + terminalReason: "saturated", + }); + expect(commands).not.toContain("fail-scan"); + await client.close(); + if (outcome === "restart-deleted-target") + await rm(repository, { recursive: true }); + if ( + outcome === "restart-missing-logs" || + outcome === "restart-corrupt-selection" + ) + await rm(usagePath); + if (outcome === "restart-changed-target") + await writeFile( + join(repository, "extract.py"), + "# Changed after selection\n", + ); + if (outcome === "restart-corrupt-selection") + await appendFile(selectedPath, "\n"); + resumedStartup = true; + client = makeClient(); + if (initialResumeUsage) await recordBudgetUsage(); + } + if (resumeStatusFault) { + const previousModelCalls = modelInputs.length; + const failure = await client + .run(repository, { + mode: "deep", + resumeScanId: scanId, + outputDir: scanDir, + postScanPrompt: followUp, + }) + .catch((error: unknown) => error); + expect(resumeStatusInjected).toBe(true); + // Failure keeps the explicitly requested follow-up, but cannot restart scanning. + expect(modelInputs.slice(previousModelCalls)).toEqual([followUp]); + expect(modelInputs.filter((input) => input !== followUp)).toHaveLength( + 1, + ); + if (resumeStatusFault === "transport") + expect(failure).toBe(resumeStatusError); + else expect(failure).toBeInstanceOf(Error); + expect(commands).not.toContain("fail-scan"); + const persisted = await runWorkbench(workbenchOptions!, [ + "get-scan", + "--scan-id", + scanId, + ]); + expect(persisted["scan"]).toMatchObject({ + progress: { + status: resumeStatusFault === "stopped" ? "canceled" : "running", + }, + }); + const deep = await runWorkbench(workbenchOptions!, [ + "get-deep-scan", + "--scan-id", + scanId, + "--thread-id", + threadId, + ]); + expect( + (deep["deepScan"] as Record)["finalizationInput"], + ).toEqual(originalFinalizationInput); + expect(await readFile(selectedPath)).toEqual(selectedBytes!); + if (resumeStatusFault === "stopped") return; + } + if (budgeted || closed) { + const running = client.run(repository, { + mode: "deep", + signal: cancellation.signal, + ...(budgeted ? { maxCostUsd: 0.004 } : {}), + ...(resumedStop ? { resumeScanId: scanId, outputDir: scanDir } : {}), + ...(unpricedUsage + ? { onCost: (cost) => reportedCosts.push(cost.estimatedUsd) } + : {}), + postScanPrompt: followUp, + }); + if (budgeted) { + if ( + budgetCompletionFault === "before-commit" || + budgetCompletionFault === "lost-and-canceled" + ) { + await expect(running).rejects.toBeInstanceOf( + ScanCostLimitExceededError, + ); + expect( + commands.filter( + (command) => command === "complete-budget-exhausted-scan", + ), + ).toHaveLength(1); + expect(commands).not.toContain("complete-scan"); + expect(modelInputs).toHaveLength(1); + const saved = await runWorkbench( + { ...workbenchOptions!, signal: undefined }, + ["get-scan", "--scan-id", scanId], + ); + expect(saved["scan"]).toMatchObject({ + progress: { + status: + budgetCompletionFault === "before-commit" + ? "failed" + : "complete", + }, + findingCount: 1, + reportAvailable: true, + }); + if (budgetCompletionFault === "lost-and-canceled") { + expect( + await Promise.all( + ["report.md", "scan-manifest.json"].map((name) => + readFile(join(scanDir, name)), + ), + ), + ).toEqual(completedArtifacts); + } + return; + } + const result = await running; + expect(result.coverage["completeness"]).toBeUndefined(); + expect(JSON.stringify(result.coverage)).toContain("cost limit"); + if (unpricedUsage) { + expect(result.cost).toBeNull(); + expect(reportedCosts).toEqual([]); + const saved = await runWorkbench(workbenchOptions!, [ + "get-scan", + "--scan-id", + scanId, + ]); + expect( + (saved["scan"] as { cost?: unknown }).cost ?? null, + ).toBeNull(); + } else expect(result.cost?.estimatedUsd).toBeGreaterThan(0.004); + expect(result.threadId).toBe(threadId); + expect(modelInputs).toHaveLength(1); + expect(commands).toContain("complete-budget-exhausted-scan"); + expect(commands).not.toContain("fail-scan"); + expect( + commands.filter( + (command) => command === "complete-budget-exhausted-scan", + ), + ).toHaveLength(1); + if (budgetCompletionFault === "lost") { + expect( + commands.filter((command) => command === "complete-scan"), + ).toHaveLength(1); + expect(result.findings.findings[0]?.remediation).toBe( + "Validate the resolved destination before writing.", + ); + const saved = await runWorkbench(workbenchOptions!, [ + "get-scan", + "--scan-id", + scanId, + ]); + expect(saved["scan"]).toMatchObject({ + progress: { status: "complete" }, + findingCount: 1, + reportAvailable: true, + }); + } + const completed = await runWorkbench(workbenchOptions!, [ + "get-deep-scan", + "--scan-id", + scanId, + "--thread-id", + threadId, + ]); + expect(completed["deepScan"]).toMatchObject({ + status: "succeeded", + finalizationInput: { terminalReason: "saturated" }, + }); + return; + } + await expect(running).rejects.toThrow(/closed/); + await closePromise; + const stopped = await runWorkbench( + { ...workbenchOptions!, signal: undefined }, + ["get-scan", "--scan-id", scanId], + ); + const deep = await runWorkbench( + { ...workbenchOptions!, signal: undefined }, + ["get-deep-scan", "--scan-id", scanId, "--thread-id", threadId], + ); + expect(stopped["scan"]).toMatchObject({ + progress: { status: "failed" }, + findingCount: 1, + reportAvailable: true, + }); + expect(deep["deepScan"]).toMatchObject({ + status: "failed", + finalizationInput: { terminalReason: "saturated" }, + }); + expect(await readFile(join(scanDir, "report.md"), "utf8")).toContain( + "Validate the resolved destination", + ); + expect( + JSON.parse(await readFile(join(scanDir, "coverage.json"), "utf8")) + .completeness, + ).toBeUndefined(); + expect(commands).toContain("fail-scan"); + expect(modelInputs).toHaveLength(1); + await client.close(); + client = makeClient(); + await expect( + client.run(repository, { + mode: "deep", + resumeScanId: scanId, + outputDir: scanDir, + }), + ).rejects.toThrow(); + expect(modelInputs).toHaveLength(1); + return; + } + if (outcome.startsWith("canceled-")) { + const error = await client + .run(repository, { + mode: "deep", + signal: cancellation.signal, + ...(resumedStop + ? { resumeScanId: scanId, outputDir: scanDir } + : {}), + postScanPrompt: followUp, + ...(initialResumeUsage ? { maxCostUsd: 0.004 } : {}), + }) + .catch((error: unknown) => error); + if (initialResumeUsage) { + expect(error).toBeInstanceOf(ScanCostLimitExceededError); + expect(error).toBe(originalResumeSignal?.reason); + } else { + expect(error).toBeInstanceOf(ScanInterruptedError); + expect((error as ScanInterruptedError).cause).toBe( + outcome === "canceled-before-publication" + ? parentError + : cancellation.signal.reason, + ); + } + expect(await readFile(selectedPath)).toEqual(selectedBytes!); + if (cancellationReadLost) { + expect(lostCancellationDeepState).toMatchObject({ + status: "running", + finalizationInput: { terminalReason: "saturated" }, + }); + } + expect(originalFinalizationInput).toMatchObject({ + terminalReason: "saturated", + }); + const stopped = await runWorkbench( + { ...workbenchOptions!, signal: undefined }, + ["get-scan", "--scan-id", scanId], + ); + const deep = await runWorkbench( + { ...workbenchOptions!, signal: undefined }, + ["get-deep-scan", "--scan-id", scanId, "--thread-id", threadId], + ); + expect(stopped["scan"]).toMatchObject({ + progress: { status: "canceled" }, + findingCount: 1, + reportAvailable: true, + }); + expect( + JSON.parse(await readFile(join(scanDir, "coverage.json"), "utf8")) + .completeness, + ).toBeUndefined(); + expect(await readFile(join(scanDir, "report.md"), "utf8")).toContain( + "Validate the resolved destination", + ); + expect(deep["deepScan"]).toMatchObject({ + status: "canceled", + }); + expect( + (deep["deepScan"] as Record)["finalizationInput"], + ).toEqual(originalFinalizationInput); + expect( + commands.filter((command) => command === "cancel-scan"), + ).toHaveLength(1); + expect(commands).not.toContain("fail-scan"); + expect(modelInputs.length).toBe(1); + if (cancellationFault === "cancel-response") { + expect( + await Promise.all( + ["report.md", "scan-manifest.json"].map((name) => + readFile(join(scanDir, name)), + ), + ), + ).toEqual(completedArtifacts); + } + await expect( + client.run(repository, { + mode: "deep", + resumeScanId: scanId, + outputDir: scanDir, + }), + ).rejects.toThrow(); + expect(modelInputs.length).toBe(1); + return; + } + if (canceledFollowUp) { + await expect( + client.run(repository, { + mode: "deep", + signal: cancellation.signal, + postScanPrompt: followUp, + }), + ).rejects.toThrow(/interrupted/); + const completed = await runWorkbench( + { ...workbenchOptions!, signal: undefined }, + ["get-scan", "--scan-id", scanId], + ); + expect(completed["scan"]).toMatchObject({ + progress: { status: "complete" }, + findingCount: 1, + reportAvailable: true, + }); + expect(modelInputs.length).toBe(2); + expect(modelInputs[1]).toBe(followUp); + expect( + commands.filter((command) => command === "complete-scan"), + ).toHaveLength(loseCompletionResponse ? 2 : 1); + expect(commands).not.toContain("cancel-scan"); + expect(commands).not.toContain("fail-scan"); + expect(await readFile(join(scanDir, "report.md"), "utf8")).toBe( + acceptedReport, + ); + return; + } + if (outcome === "completion-before-commit-fails") { + await expect(client.run(repository, { mode: "deep" })).rejects.toThrow( + "Synthetic completion failure before commit", + ); + expect( + commands.filter((command) => command === "complete-scan"), + ).toHaveLength(1); + expect(commands).not.toContain("fail-scan"); + expect(modelInputs).toHaveLength(1); + const saved = await runWorkbench(workbenchOptions!, [ + "get-scan", + "--scan-id", + scanId, + ]); + expect(saved["scan"]).toMatchObject({ + progress: { status: "running" }, + }); + return; + } + if (outcome === "restart-canceled-runtime") { + await expect( + client.run(repository, { + mode: "deep", + resumeScanId: scanId, + outputDir: scanDir, + signal: cancellation.signal, + }), + ).rejects.toBeInstanceOf(ScanInterruptedError); + const stopped = await runWorkbench( + { ...workbenchOptions!, signal: undefined }, + ["get-scan", "--scan-id", scanId], + ); + expect(stopped["scan"]).toMatchObject({ + progress: { status: "canceled" }, + }); + expect( + commands.filter((command) => command === "cancel-scan"), + ).toHaveLength(1); + expect(commands).not.toContain("fail-scan"); + expect(await readFile(selectedPath)).toEqual(selectedBytes!); + return; + } + if (outcome === "restart-corrupt-selection") { + await expect( + client.run(repository, { + mode: "deep", + resumeScanId: scanId, + outputDir: scanDir, + }), + ).rejects.toThrow("changed after acceptance"); + const pending = await runWorkbench(workbenchOptions!, [ + "get-scan", + "--scan-id", + scanId, + ]); + expect(pending["scan"]).toMatchObject({ + progress: { status: "running" }, + }); + expect(commands).not.toContain("complete-scan"); + expect(commands).not.toContain("fail-scan"); + expect(modelInputs.filter((input) => input !== followUp)).toHaveLength( + 1, + ); + return; + } + const result = await client.run(repository, { + mode: "deep", + ...(recoveryBudget ? { maxCostUsd: 0.004 } : {}), + onWarning: (warning) => warnings.push(warning), + signal: cancellation.signal, + postScanPrompt: + outcome === "published-before-cancellation" ? undefined : followUp, + ...(restart ? { resumeScanId: scanId, outputDir: scanDir } : {}), + }); + expect(result.threadId).toBe(threadId); + if (completeBeforePreparation) { + expect(completedBeforePreparation).toBe(true); + expect( + await Promise.all( + [ + "scan-manifest.json", + "findings.json", + "coverage.json", + "report.md", + ].map((name) => readFile(join(scanDir, name))), + ), + ).toEqual(concurrentlyCompletedArtifacts!); + } + if (recoveryBudget) + expect(commands).toContain("complete-budget-exhausted-scan"); + if (outcome === "restart-changed-target") + expect( + warnings.some((warning) => /changed|revision/.test(warning)), + ).toBe(true); + if (outcome === "lost-completion-response") { + expect( + commands.filter((command) => command === "complete-scan"), + ).toHaveLength(2); + } + if (ownerSuffix) { + expect(nativeOwnerStatus).toBe("running"); + expect(result.cost?.inputTokens).toBe(2_500); + expect(result.cost?.outputTokens).toBe(60); + } else if (recoveryBudget && !unpricedUsage) { + expect(result.cost?.inputTokens).toBe(1_250); + expect(result.cost?.outputTokens).toBe(30); + expect(result.cost?.estimatedUsd).toBeGreaterThan(0.004); + } else { + // The synthetic accepted workers have no native usage receipts. + expect(result.cost).toBeNull(); + } + expect(result.coverage["completeness"]).toBeUndefined(); + expect(result.findings.findings[0]?.remediation).toBe( + "Validate the resolved destination before writing.", + ); + if (historicalFindings) { + const published = JSON.parse( + await readFile(join(scanDir, "findings.json"), "utf8"), + ); + expect(published.findings[0]).toMatchObject({ + root_cause: { + summary: "The original destination is unchecked.", + evidence_refs: ["legacy-proof"], + }, + code_evidence: [{ id: "legacy-proof", code: "write(destination)" }], + }); + expect(await readFile(selectedPath)).toEqual(selectedBytes!); + expect( + warnings.some((warning) => + warning.startsWith("Could not run post-scan instructions:"), + ), + ).toBe(changedHistoricalFollowUp === true); + } + // postScanPrompt retains its existing behavior on each caller invocation. + expect(modelInputs.filter((input) => input !== followUp).length).toBe(1); + expect(modelInputs.filter((input) => input === followUp).length).toBe( + outcome === "published-before-cancellation" + ? 0 + : unavailableStartup + ? 1 + : restart + ? resumeStatusFault + ? 3 + : 2 + : 1, + ); + if (unavailableStartup) { + expect( + warnings.some((warning) => + warning.includes("Could not run post-scan instructions"), + ), + ).toBe(true); + expect(await readFile(selectedPath)).toEqual(selectedBytes!); + expect(result.turnResult.finalResponse).toContain( + "Validate the resolved destination", + ); + } + const completed = await runWorkbench( + { ...workbenchOptions!, signal: undefined }, + ["get-scan", "--scan-id", scanId], + ); + expect(completed["scan"]).toMatchObject({ + progress: { status: "complete" }, + }); + expect(await readFile(join(scanDir, "report.md"), "utf8")).toContain( + "Validate the resolved destination", + ); + expect(commands).not.toContain("fail-scan"); + } finally { + clearTimeout(keepAlive); + await client.close(); + } + }; + test(name, runCase, 30_000); +} + +async function rejoinSelectedScanThroughMcp( + environment: NodeJS.ProcessEnv, + scanId: string, + explicitCompletion = false, +): Promise { + const child = spawn( + process.execPath, + [join(PLUGIN_ROOT, "mcp/server.mjs"), "--stdio"], + { + env: environment, + stdio: ["pipe", "pipe", "pipe"], + }, + ); + let stderr = ""; + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk: string) => { + stderr += chunk; + }); + const exited = new Promise((resolve) => + child.once("exit", () => resolve()), + ); + const result = new Promise((resolve, reject) => { + let buffer = ""; + child.once("error", reject); + child.once("exit", () => + reject(new Error(`MCP closed before selected rejoin: ${stderr}`)), + ); + child.stdout.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { + buffer += chunk; + for (;;) { + const newline = buffer.indexOf("\n"); + if (newline < 0) break; + const line = buffer.slice(0, newline); + buffer = buffer.slice(newline + 1); + try { + const response = JSON.parse(line) as { + id?: number; + error?: unknown; + result?: { + isError?: boolean; + structuredContent?: { manifestPath?: string }; + }; + }; + if (response.id === 1) { + child.stdin.write( + JSON.stringify({ + jsonrpc: "2.0", + id: 2, + method: "tools/call", + params: { + name: explicitCompletion + ? "complete_codex_security_scan" + : "start_codex_security_deep_scan", + arguments: { scanId }, + _meta: { + "openai/threadId": threadId, + "codex/sandbox-state-meta": { + permissionProfile: { + type: "managed", + file_system: { + type: "restricted", + entries: [ + { + path: { + type: "special", + value: { kind: "root" }, + }, + access: "read", + }, + ], + }, + network: "restricted", + }, + sandboxCwd: pathToFileURL(PLUGIN_ROOT).href, + }, + }, + }, + }) + "\n", + ); + } + if (response.id === 2) { + expect(response.error).toBeUndefined(); + expect(response.result?.isError).toBeUndefined(); + if (!explicitCompletion) + expect( + response.result?.structuredContent?.manifestPath, + ).toBeDefined(); + resolve(); + } + } catch (error) { + reject(error); + } + } + }); + }); + child.stdin.write( + JSON.stringify({ + jsonrpc: "2.0", + id: 1, + method: "initialize", + params: { + protocolVersion: "2025-11-25", + capabilities: {}, + clientInfo: { name: "sdk-owner-completion", version: "0.1.0" }, + }, + }) + "\n", + ); + try { + await result; + } finally { + child.stdin.end(); + child.kill(); + await exited; + } +} diff --git a/sdk/typescript/tests-ts/deep-scan-coverage.test.ts b/sdk/typescript/tests-ts/deep-scan-coverage.test.ts new file mode 100644 index 0000000000..dfd5b17bf4 --- /dev/null +++ b/sdk/typescript/tests-ts/deep-scan-coverage.test.ts @@ -0,0 +1,113 @@ +import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { expect, test } from "bun:test"; +import { main } from "../src/cli.js"; +import { loadContract } from "../src/contract.js"; +import { ScanResult } from "../src/result.js"; +import { capture, dependencies } from "./cli-fixtures.js"; + +const fixtureUrl = new URL( + "../../../plugins/codex-security/mcp-app/tests/deep_scan_coverage_fixture.mjs", + import.meta.url, +); + +test.each([ + ["partial", false, false], + ["unknown", false, false], + ["complete", false, false], + ["partial", true, false], + ["partial", true, true], +] as const)( + "publishes %s source coverage through CLI results (resume: %p, continued: %p)", + async (completeness, resume, continueAfterResume) => { + const root = await mkdtemp(join(tmpdir(), "deep-coverage-publication-")); + try { + await mkdir(join(root, "fixture"), { mode: 0o700 }); + // Keep the real workbench outside other suites' persistent module mocks. + const child = Bun.spawn( + [ + Bun.which("node")!, + fileURLToPath(fixtureUrl), + join(root, "fixture"), + completeness, + String(resume), + String(continueAfterResume), + ], + { stdout: "pipe", stderr: "pipe" }, + ); + const [output, errors, exitCode] = await Promise.all([ + new Response(child.stdout).text(), + new Response(child.stderr).text(), + child.exited, + ]); + expect(exitCode, errors).toBe(0); + const { scanDir, threadId, terminal } = JSON.parse(output); + const contract = await loadContract(scanDir, { + pluginRoot: fileURLToPath( + new URL("../../../plugins/codex-security/", import.meta.url), + ), + }); + const result = new ScanResult({ + ...contract, + scanDir, + threadId, + turnResult: { status: "completed" }, + }); + expect(result.coverage["completeness"]).toBeUndefined(); + const coverage = JSON.parse( + await readFile(join(scanDir, "coverage.json"), "utf8"), + ); + expect(coverage.reviews[0].attempt).toBe(2); + const report = await readFile(join(scanDir, "report.md"), "utf8"); + expect(report).not.toContain("| Coverage |"); + expect(coverage.explicitExclusions).toHaveLength(coverage.reviews.length); + for (const review of coverage.reviews) + expect(report).toContain(review.workerId); + if (completeness === "partial") { + expect( + coverage.deferred.map((item: { reason: string }) => item.reason), + ).toEqual(["Verify entry boundaries.", "Verify symbolic links."]); + expect( + new Set(coverage.deferred.map((item: { id: string }) => item.id)) + .size, + ).toBe(2); + expect( + coverage.reviews.every( + (review: Record) => + review["completeness"] === undefined, + ), + ).toBe(true); + for (const item of coverage.deferred) { + expect(item.provenance.candidateId).toBe("candidate-1"); + expect(report).toContain(item.reason); + expect( + coverage.surfaces.some((surface: { id: string }) => + item.surfaceIds.includes(surface.id), + ), + ).toBe(true); + } + } + for (const surface of coverage.surfaces) { + expect( + await readFile(join(scanDir, surface.receiptRefs[0]), "utf8"), + ).toContain("Synthetic review evidence."); + } + const stdout = capture(); + const stderr = capture(); + expect( + await main( + ["scan", "--mode", "deep", "--json"], + stdout.stream, + stderr.stream, + dependencies({ result, onWorkbench: () => ({ deepScan: terminal }) }), + ), + ).toBe(0); + expect(JSON.parse(stdout.text()).coverage).toEqual(coverage); + } finally { + await rm(root, { recursive: true, force: true }); + } + }, + 60_000, +); diff --git a/sdk/typescript/tests-ts/deep-scan-workbench.test.ts b/sdk/typescript/tests-ts/deep-scan-workbench.test.ts index e392d8064f..06ccab8733 100644 --- a/sdk/typescript/tests-ts/deep-scan-workbench.test.ts +++ b/sdk/typescript/tests-ts/deep-scan-workbench.test.ts @@ -23,12 +23,12 @@ const deepScanOwnershipProbe = [ "connection.executescript('''", "CREATE TABLE workspaces (id TEXT PRIMARY KEY, thread_id TEXT, updated_at TEXT);", "CREATE TABLE scans (id TEXT PRIMARY KEY, workspace_id TEXT, mode TEXT, status TEXT, recipe_json TEXT, handoff_status TEXT, handoff_claim_token TEXT, deep_scan_owner_thread_id TEXT, updated_at TEXT);", - "CREATE TABLE deep_scan_runs (scan_id TEXT PRIMARY KEY);", + "CREATE TABLE deep_scan_runs (scan_id TEXT PRIMARY KEY, schema_version INTEGER NOT NULL DEFAULT 1, workflow_version TEXT NOT NULL DEFAULT 'deep-scan-mcp/v1');", "''')", "scan_id = '11111111-1111-4111-8111-111111111111'", "connection.execute(\"INSERT INTO workspaces VALUES ('workspace', NULL, 'before')\")", "connection.execute(\"INSERT INTO scans VALUES (?, 'workspace', 'deep', 'running', '{}', 'delivered', ?, NULL, 'before')\", (scan_id, case['storedToken']))", - "connection.execute('INSERT INTO deep_scan_runs VALUES (?)', (scan_id,))", + "connection.execute('INSERT INTO deep_scan_runs (scan_id) VALUES (?)', (scan_id,))", "connection.commit()", "if case.get('mutation') == 'rotate':", " connection.executescript(\"CREATE TRIGGER rotate_claim BEFORE UPDATE OF thread_id ON workspaces BEGIN UPDATE scans SET handoff_claim_token = '33333333-3333-4333-8333-333333333333' WHERE workspace_id = NEW.id; END\")", diff --git a/sdk/typescript/tests-ts/direct-deep-scan.test.ts b/sdk/typescript/tests-ts/direct-deep-scan.test.ts index 25437155ad..4f63bb7b10 100644 --- a/sdk/typescript/tests-ts/direct-deep-scan.test.ts +++ b/sdk/typescript/tests-ts/direct-deep-scan.test.ts @@ -1,5 +1,6 @@ import assert from "node:assert/strict"; import { + appendFile, copyFile, mkdir, readFile, @@ -165,7 +166,7 @@ test("preserves complete worker usage and cost after direct engine completion", const codexHome = join(root, "codex-home"); const scanDir = join(root, "scan"); await mkdir(scanDir); - await writeSession( + const discoveryLog = await writeSession( codexHome, "discovery-session", { @@ -186,7 +187,7 @@ test("preserves complete worker usage and cost after direct engine completion", timestamp: "2026-07-26T12:00:01.000Z", }, ); - await writeSession( + const reducerLog = await writeSession( codexHome, "reducer-session", { @@ -200,6 +201,39 @@ test("preserves complete worker usage and cost after direct engine completion", timestamp: "2026-07-26T12:00:02.000Z", }, ); + for (const [path, thread] of [ + [discoveryLog, "discovery-session"], + [reducerLog, "reducer-session"], + ] as const) { + const records = (await readFile(path, "utf8")) + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + const usage = records[1].payload.info.total_token_usage; + await appendFile( + path, + [ + { + type: "turn_context", + timestamp: "2026-07-26T12:00:03.000Z", + payload: { turn_id: `${thread}-turn`, model: "gpt-6.1-sol" }, + }, + { + type: "token_usage_record", + timestamp: "2026-07-26T12:00:03.000Z", + payload: { + thread_id: thread, + turn_id: `${thread}-turn`, + response_id: `${thread}-response`, + model: "gpt-6.1-sol", + usage, + }, + }, + ] + .map((value) => JSON.stringify(value)) + .join("\n") + "\n", + ); + } await writeFile( join(codexHome, "sessions", "owner.jsonl"), JSON.stringify({ @@ -238,9 +272,28 @@ test("preserves complete worker usage and cost after direct engine completion", scanDirectory: scanDir, model: "gpt-6.1-sol", }); + const executionThreadIds = [ + "engine-session", + "discovery-session", + "reducer-session", + ]; + tracker.setAttributionReader(async () => ({ + formatVersion: 1, + workerCodexHome: codexHome, + executionThreadIds, + owner: { + threadId: "engine-session", + turnId: null, + dedicated: true, + startedAt: "2026-07-26T12:00:00.000Z", + }, + startedAt: "2026-07-26T12:00:00.000Z", + completedAt: null, + })); tracker.recordUsage(DIRECT_DEEP_SCAN_CONTROL_USAGE, "engine-session"); tracker.start("engine-session"); const running = await tracker.refresh(); + assert.ok(running.usage && typeof running.usage === "object"); await writeSession( codexHome, "waiting-session", @@ -249,7 +302,11 @@ test("preserves complete worker usage and cost after direct engine completion", parent: "engine-session", }, ); - assert.deepEqual(await tracker.refresh(), { usage: null, cost: null }); + executionThreadIds.push("waiting-session"); + assert.deepEqual(await tracker.refresh(), { + usage: { ...running.usage, coverage: "partial" }, + cost: { ...running.cost, coverage: "partial" }, + }); tracker.recordUsage( { input_tokens: 0, @@ -262,7 +319,7 @@ test("preserves complete worker usage and cost after direct engine completion", const completed = (await events.next()).value; assert.equal(completed?.type, "turn.completed"); const final = await tracker.stop(completed?.["usage"]); - assert.deepEqual(running.usage, { + assert.partialDeepStrictEqual(running.usage, { input_tokens: 1_500, cached_input_tokens: 250, cache_write_input_tokens: 140, diff --git a/sdk/typescript/tests-ts/fixtures/bundle-worker-settings.mjs b/sdk/typescript/tests-ts/fixtures/bundle-worker-settings.mjs new file mode 100644 index 0000000000..b13942be8b --- /dev/null +++ b/sdk/typescript/tests-ts/fixtures/bundle-worker-settings.mjs @@ -0,0 +1,22 @@ +import { readFile, writeFile } from "node:fs/promises"; +import { dirname } from "node:path"; +import { pathToFileURL } from "node:url"; +import { build } from "esbuild"; + +const [executor, output] = process.argv.slice(2); +const bundled = await build({ + bundle: true, + format: "esm", + platform: "node", + write: false, + define: { "import.meta.url": JSON.stringify(pathToFileURL(executor).href) }, + stdin: { + contents: + (await readFile(executor, "utf8")) + + "\nexport { workerRuntimeSettings };", + loader: "ts", + resolveDir: dirname(executor), + sourcefile: executor, + }, +}); +await writeFile(output, bundled.outputFiles[0].contents); diff --git a/sdk/typescript/tests-ts/fixtures/selected-deep-scan.py b/sdk/typescript/tests-ts/fixtures/selected-deep-scan.py new file mode 100644 index 0000000000..5b741021ca --- /dev/null +++ b/sdk/typescript/tests-ts/fixtures/selected-deep-scan.py @@ -0,0 +1,67 @@ +"""Synthetic accepted workers for the installed finalization/SDK contract test.""" +import hashlib +import json +import sqlite3 +import sys +import uuid +from pathlib import Path + +payload = json.load(sys.stdin) +scan_id = payload["scanId"] +scan_dir = Path(payload["scanDir"]) +with sqlite3.connect(payload["database"]) as connection: + connection.execute("PRAGMA foreign_keys = ON") + timestamp = connection.execute( + "SELECT created_at FROM deep_scan_runs WHERE scan_id = ?", (scan_id,) + ).fetchone()[0] + connection.execute( + "UPDATE deep_scan_runs SET workflow_version = 'deep-security-scan/v2', " + "coordinator_generation = 2, phase = 'reducing', " + "discovery_runs_dispatched = 2, completion_sequence = 2, " + "consecutive_no_new = 2, stop_after_no_new = 2, max_discovery_runs = 2 " + "WHERE scan_id = ?", (scan_id,), + ) + discoveries = [] + for kind, label in [("discovery", "review-1"), ("discovery", "review-2"), ("dedup", "merge-1")]: + draft = dict(payload["draft"]) + if kind == "dedup": + draft["sourceCoverage"] = draft.pop("coverage") + encoded = json.dumps(draft).encode() + digest = hashlib.sha256(encoded).hexdigest() + worker_id = str(uuid.uuid4()) + output = scan_dir / "artifacts" / "deep_discovery" / label / "output" + output.mkdir(parents=True) + prompt = output.parent / "prompt.md" + prompt.write_text("Synthetic accepted audit\n") + accepted = output / "checkpoints" / f"{digest}.json" + accepted.parent.mkdir() + accepted.write_bytes(encoded) + result = output / "result.json" + result.write_bytes(encoded) + sequence = len(discoveries) + 1 if kind == "discovery" else None + connection.execute( + "INSERT INTO deep_scan_workers " + "(id, scan_id, kind, status, merge_state, prompt_path, artifact_dir, " + "result_manifest_path, attempt, completion_sequence, created_at, updated_at, completed_at) " + "VALUES (?, ?, ?, 'succeeded', ?, ?, ?, ?, 1, ?, ?, ?, ?)", + (worker_id, scan_id, kind, "merged" if kind == "discovery" else "none", + str(prompt), str(output), str(result), sequence, timestamp, timestamp, timestamp), + ) + connection.execute( + "INSERT INTO deep_scan_attempts " + "(scan_id, worker_id, attempt, status, started_at, completed_at, accepted_result_path, accepted_result_sha256) " + "VALUES (?, ?, 1, 'succeeded', ?, ?, ?, ?)", + (scan_id, worker_id, timestamp, timestamp, str(accepted), digest), + ) + if kind == "discovery": + discoveries.append(worker_id) + else: + for order, discovery in enumerate(discoveries): + connection.execute( + "INSERT INTO deep_scan_dedup_inputs " + "(scan_id, dedup_worker_id, discovery_worker_id, input_order) VALUES (?, ?, ?, ?)", + (scan_id, worker_id, discovery, order), + ) + # The committed immutable reference survives loss of the replaceable output. + result.unlink() +print(json.dumps({"resultPath": str(result), "acceptedPath": str(accepted)})) diff --git a/sdk/typescript/tests-ts/hosted-scan.test.ts b/sdk/typescript/tests-ts/hosted-scan.test.ts index 4b45a063ab..c8c4d3f198 100644 --- a/sdk/typescript/tests-ts/hosted-scan.test.ts +++ b/sdk/typescript/tests-ts/hosted-scan.test.ts @@ -1099,6 +1099,31 @@ test("SDK cancellation waits for the executor receipt without losing known sessi }); }); +test("SDK cancellation retains an unknown executor acceptance receipt", async () => { + const controller = new AbortController(); + let requestId = ""; + const executor = { + run: mock(async (request: ScanExecutionRequest) => { + requestId = request.requestId; + controller.abort(); + await Promise.resolve(); + throw new Error("Synthetic response lost after dispatch"); + }), + }; + const result = await runHostedScan(await fixture(), { + signal: controller.signal, + executor, + }); + expect(executor.run).toHaveBeenCalledTimes(1); + expect(requestId).not.toBe(""); + expect(result.status).toBe("acceptance_unknown"); + expect(result.execution).toEqual({ + requestId, + status: "acceptance_unknown", + message: "Synthetic response lost after dispatch", + }); +}); + test("mismatched execution receipts are never accepted or retried", async () => { const input = await fixture(); const executor = { diff --git a/sdk/typescript/tests-ts/plugin-root.ts b/sdk/typescript/tests-ts/plugin-root.ts index cce6d818d8..16aa148678 100644 --- a/sdk/typescript/tests-ts/plugin-root.ts +++ b/sdk/typescript/tests-ts/plugin-root.ts @@ -19,7 +19,7 @@ let bundledRuntime: Promise | undefined; export function loadBundledRuntime(): Promise { return (bundledRuntime ??= Promise.all( - ["000", "001"].map((part) => + ["000", "001", "002"].map((part) => readFile(new URL(`mcp/server.mjs.br.part-${part}`, bundledPlugin)), ), ).then((parts) => diff --git a/sdk/typescript/tests-ts/reasoning-summary.test.ts b/sdk/typescript/tests-ts/reasoning-summary.test.ts new file mode 100644 index 0000000000..4bff796150 --- /dev/null +++ b/sdk/typescript/tests-ts/reasoning-summary.test.ts @@ -0,0 +1,164 @@ +import { execFileSync } from "node:child_process"; +import { + mkdir, + mkdtemp, + readFile, + realpath, + rm, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { expect, test } from "bun:test"; +import { captureOriginalReasoningSummary } from "../src/reasoning-summary.js"; +import type { JsonObject } from "../src/config.js"; +import { createProviderProfile } from "../src/provider-profile.js"; + +test.each([ + { model_reasoning_summary: "none" }, + { model_reasoning_summary: "concise" }, + { model_reasoning_summary: null }, + { model_reasoning_summary: "" }, + { + profile: "selected", + profiles: { selected: { model_reasoning_summary: "auto" } }, + }, +] as JsonObject[])( + "preserves explicit summary without a native lookup: %j", + async (config) => { + expect( + await captureOriginalReasoningSummary({ + config, + command: { command: "unused-native-executable" }, + cwd: tmpdir(), + environment: {}, + signal: new AbortController().signal, + }), + ).toBeUndefined(); + }, +); + +test.each(["unknown-model", "unsupported-command", "missing-metadata"])( + "keeps an unavailable original model default unknown: %s", + async (scenario) => { + const root = await mkdtemp(join(tmpdir(), "summary-selection-")); + try { + const cwd = join(root, "original output"); + const script = join(root, "native.mjs"); + const receipt = join(root, "receipt.json"); + await mkdir(cwd); + await writeFile( + script, + [ + 'import { writeFileSync } from "node:fs";', + `writeFileSync(${JSON.stringify(receipt)}, JSON.stringify({cwd:process.cwd(),args:process.argv,home:process.env.CODEX_HOME}));`, + `console.log(JSON.stringify({models:[{slug:${JSON.stringify(scenario === "unknown-model" ? "another-model" : "selected-model")}${scenario === "missing-metadata" ? "" : ',default_reasoning_summary:"none"'}}]}));`, + `process.exit(${scenario === "unsupported-command" ? 1 : 0});`, + ].join("\n"), + ); + const command = execFileSync("node", ["-p", "process.execPath"], { + encoding: "utf8", + }).trim(); + const value = await captureOriginalReasoningSummary({ + config: { model: "selected-model", model_reasoning_effort: "high" }, + command: { command }, + cwd, + environment: { + CODEX_HOME: root, + NODE_OPTIONS: `--import=${pathToFileURL(script).href}`, + }, + signal: new AbortController().signal, + }); + expect(value).toBeUndefined(); + const recorded = JSON.parse(await readFile(receipt, "utf8")); + expect(await realpath(recorded.cwd)).toBe(await realpath(cwd)); + expect(recorded.home).toBe(root); + expect(recorded.args).toContain('model="selected-model"'); + expect(recorded.args).toContain('model_reasoning_effort="high"'); + } finally { + await rm(root, { recursive: true, force: true }); + } + }, +); + +test.each(["literal", "selected", "default"])( + "original model lookup keeps %s provider credentials in its native profile", + async (scenario) => { + const root = await mkdtemp(join(tmpdir(), "summary-provider-")); + let profile: Awaited> | undefined; + try { + const script = join(root, "native.mjs"); + const receipt = join(root, "arguments.json"); + const provider = "synthetic.gateway"; + const definition = { + name: "Synthetic gateway", + http_headers: { Authorization: "synthetic-summary-header" }, + }; + const config: JsonObject = { + model: "selected-model", + model_reasoning_effort: "high", + ...(scenario === "default" + ? {} + : scenario === "selected" + ? { + profile: "selected", + profiles: { + selected: { + model_provider: provider, + model_providers: { [provider]: definition }, + }, + }, + } + : { + model_provider: provider, + model_providers: { [provider]: definition }, + }), + }; + if (scenario !== "default") + profile = await createProviderProfile(root, config); + await writeFile( + script, + [ + 'import { writeFileSync, readFileSync } from "node:fs";', + 'import { join } from "node:path";', + `writeFileSync(${JSON.stringify(receipt)}, JSON.stringify(process.argv));`, + 'const i = process.argv.indexOf("--profile");', + "const selected = i === -1 ? undefined : process.argv[i + 1];", + ...(scenario === "default" + ? [] + : [ + 'if (!selected || !readFileSync(join(process.env.CODEX_HOME, selected + ".config.toml"), "utf8").includes("synthetic-summary-header")) process.exit(13);', + ]), + 'console.log(JSON.stringify({models:[{slug:"selected-model",default_reasoning_summary:"none"}]}));', + "process.exit(0);", + ].join("\n"), + ); + const command = execFileSync("node", ["-p", "process.execPath"], { + encoding: "utf8", + }).trim(); + const value = await captureOriginalReasoningSummary({ + config, + command: { command }, + cwd: root, + environment: { + CODEX_HOME: root, + NODE_OPTIONS: `--import=${pathToFileURL(script).href}`, + }, + signal: new AbortController().signal, + ...(profile === undefined ? {} : { nativeProfile: profile.name }), + }); + const args: string[] = JSON.parse(await readFile(receipt, "utf8")); + expect(args.some((arg) => arg.includes("synthetic-summary-header"))).toBe( + false, + ); + expect(value).toBe("none"); + expect(args).toContain('model="selected-model"'); + if (profile !== undefined) + expect(args[args.indexOf("--profile") + 1]).toBe(profile.name); + } finally { + await profile?.cleanup(); + await rm(root, { recursive: true, force: true }); + } + }, +); diff --git a/sdk/typescript/tests-ts/scan-logs-node.test.ts b/sdk/typescript/tests-ts/scan-logs-node.test.ts index 7b2deca11a..d41b7d81ed 100644 --- a/sdk/typescript/tests-ts/scan-logs-node.test.ts +++ b/sdk/typescript/tests-ts/scan-logs-node.test.ts @@ -1,6 +1,7 @@ +import { runTestInSubprocess } from "./support/test-subprocess.js"; import { execFile } from "node:child_process"; -import { mkdir, writeFile } from "node:fs/promises"; -import { join } from "node:path"; +import { mkdir, symlink, writeFile } from "node:fs/promises"; +import { join, sep } from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { promisify } from "node:util"; import { zstdCompressSync } from "node:zlib"; @@ -100,3 +101,75 @@ test.each(["native", "without zstd"])( ]); }, ); + +for (const directory of ["sessions", "archived_sessions"]) { + const name = `Node reads recorded homes through directory links in ${directory}`; + test(name, async () => { + if (runTestInSubprocess(import.meta.path, name)) return; + const root = await temporaryDirectory(); + const current = join(root, "current"); + const original = join(root, "original"); + await mkdir(current); + await mkdir(join(original, "child"), { recursive: true }); + await mkdir(join(original, directory)); + const link = join(current, "original-home-link"); + const windows = process.platform === "win32"; + await symlink( + windows ? original : join(original, "child"), + link, + windows ? "junction" : "dir", + ); + const recorded = windows ? link : `${link}${sep}..`; + const timestamp = "2026-08-11T12:01:00.000Z"; + const event = { + type: "response_item", + timestamp, + payload: { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: "saved original activity" }], + }, + }; + await writeFile( + join(original, directory, "owner.jsonl"), + [ + { type: "session_meta", payload: { id: "owner" } }, + { type: "turn_context", timestamp, payload: { turn_id: "scan-turn" } }, + event, + ] + .map((value) => JSON.stringify(value) + "\n") + .join(""), + ); + const built = await Bun.build({ + entrypoints: [ + fileURLToPath(new URL("../src/scan-logs.ts", import.meta.url)), + ], + target: "node", + format: "esm", + }); + expect(built.success).toBe(true); + const modulePath = join(root, "scan-logs.mjs"); + await writeFile(modulePath, await built.outputs[0]!.text()); + const { stdout } = await promisify(execFile)("node", [ + "--input-type=module", + "--eval", + ` + import assert from "node:assert/strict"; + import { realpath } from "node:fs/promises"; + assert.equal(await realpath(${JSON.stringify(recorded)}), await realpath(${JSON.stringify(original)})); + const { readSavedScanLogs } = await import(${JSON.stringify(pathToFileURL(modulePath).href)}); + const attribution = { formatVersion: 1, workerCodexHome: ${JSON.stringify(recorded)}, executionThreadIds: [], + owner: { threadId: "owner", turnId: "scan-turn", startedAt: ${JSON.stringify(timestamp)} }, + startedAt: ${JSON.stringify(timestamp)}, completedAt: ${JSON.stringify(timestamp)}, + }; + const logs = await readSavedScanLogs({ scanId: "scan-1", mode: "deep", scanDir: ${JSON.stringify(root)}, continuationThreadId: "owner", executionAttribution: attribution }, ${JSON.stringify(current)}); + assert.equal(logs.sessions.length, 1); + assert.equal(logs.sessions[0].path, ${JSON.stringify(join(original, directory, "owner.jsonl"))}); + assert.deepEqual(logs.events.at(-1).event, ${JSON.stringify(event)}); + assert.equal(attribution.workerCodexHome, ${JSON.stringify(recorded)}); + console.log("original logs recovered"); + `, + ]); + expect(stdout.trim()).toBe("original logs recovered"); + }); +} diff --git a/sdk/typescript/tests-ts/scan-logs.test.ts b/sdk/typescript/tests-ts/scan-logs.test.ts index ac076bbc76..11c0aa9874 100644 --- a/sdk/typescript/tests-ts/scan-logs.test.ts +++ b/sdk/typescript/tests-ts/scan-logs.test.ts @@ -63,6 +63,414 @@ function commandEvent(command: string, id: string, timestamp?: string) { } describe("saved scan logs", () => { + test.each(["standard", "deep"])( + "ignores model-owned log homes without host ownership for %s scans", + async (mode) => { + const configured = await temporaryHome(); + const foreign = await temporaryHome(); + const scanDir = await temporaryHome(); + const settings = join(scanDir, "artifacts", "deep_discovery"); + await mkdir(settings, { recursive: true }); + await writeFile( + join(settings, "execution-settings.json"), + JSON.stringify({ version: 1, settings: { codexHome: foreign } }), + ); + const configuredEvent = commandEvent( + "configured owner", + "configured-call", + ); + await writeSession(configured, "owner", [configuredEvent]); + await writeSession(foreign, "owner", [ + configuredEvent, + commandEvent("foreign suffix", "foreign-call"), + ]); + const result = await readSavedScanLogs( + { scanId: "scan-1", mode, scanDir, continuationThreadId: "owner" }, + configured, + ); + expect(result.events.map(({ event }) => event)).toEqual([ + { type: "session_meta", payload: { id: "owner" } }, + configuredEvent, + ]); + expect(result.sessions.map(({ path }) => path)).toEqual([ + join(configured, "sessions", "2026", "08", "11", "rollout-owner.jsonl"), + ]); + }, + ); + + test.each( + ["sessions", "archived_sessions"].flatMap((directory) => + ["prefix only", "unrelated owner turn"].map((tail) => [directory, tail]), + ), + )( + "reads the complete same-thread recorded copy from %s after %s", + async (directory, tail) => { + const current = await temporaryHome(); + const original = await temporaryHome(); + const scanDir = await temporaryHome(); + const settings = join(scanDir, "artifacts", "deep_discovery"); + await mkdir(settings, { recursive: true }); + await writeFile( + join(settings, "execution-settings.json"), + JSON.stringify({ version: 1, settings: { codexHome: original } }), + ); + const timestamp = "2026-08-11T12:01:00.000Z"; + const prefix = [ + { type: "turn_context", timestamp, payload: { turn_id: "scan-turn" } }, + commandEvent("prefix", "prefix-call", timestamp), + ]; + const suffix = commandEvent("saved suffix", "suffix-call", timestamp); + await writeSession(current, "owner", [ + ...prefix, + ...(tail === "unrelated owner turn" + ? [ + { + type: "turn_context", + timestamp, + payload: { turn_id: "other-turn" }, + }, + commandEvent( + "unrelated first copy", + "other-first-call", + timestamp, + ), + ] + : []), + ]); + const currentPath = join( + current, + "sessions", + "2026", + "08", + "11", + "rollout-owner.jsonl", + ); + await writeFile( + currentPath, + (await readFile(currentPath, "utf8")) + '\n\n42\n{"type":', + ); + await writeSession(original, "owner", [ + ...prefix, + suffix, + suffix, + { type: "turn_context", timestamp, payload: { turn_id: "other-turn" } }, + commandEvent("unrelated turn", "other-call", timestamp), + ]); + if (directory === "archived_sessions") { + await rename(join(original, "sessions"), join(original, directory)); + } + const result = await readSavedScanLogs( + { + scanId: "scan-1", + mode: "deep", + scanDir, + continuationThreadId: "owner", + executionAttribution: { + formatVersion: 1, + workerCodexHome: original, + executionThreadIds: [], + owner: { + threadId: "owner", + turnId: "scan-turn", + startedAt: timestamp, + }, + startedAt: timestamp, + completedAt: timestamp, + }, + }, + current, + ); + expect(result.sessions.map(({ threadId }) => threadId)).toEqual([ + "owner", + ]); + expect(result.events.map(({ event }) => event)).toEqual([ + { type: "session_meta", payload: { id: "owner" } }, + ...prefix, + suffix, + suffix, + ]); + expect(result.sessions[0]?.path).toStartWith(join(original, directory!)); + }, + ); + + test.each(["equal", "shorter", "divergent"])( + "keeps the first rollout when attributed occurrences are %s", + async (copy) => { + const first = await temporaryHome(); + const second = await temporaryHome(); + const timestamp = "2026-08-11T12:01:00.000Z"; + const turn = { + type: "turn_context", + timestamp, + payload: { turn_id: "scan-turn" }, + }; + const prefix = commandEvent("first", "first-call", timestamp); + const suffix = commandEvent("complete", "complete-call", timestamp); + const unrelated = { + type: "turn_context", + timestamp, + payload: { turn_id: "other-turn" }, + }; + await writeSession(first, "owner", [turn, prefix, suffix, unrelated]); + await writeSession(second, "owner", [ + turn, + ...(copy === "equal" + ? [prefix, suffix] + : copy === "shorter" + ? [prefix] + : [ + prefix, + commandEvent("different", "different-call", timestamp), + suffix, + ]), + unrelated, + commandEvent("more unrelated work", "other-call", timestamp), + ]); + const result = await readScanLogs({ + scanId: "scan-1", + threadId: "owner", + codexHome: [first, second], + executionAttribution: { + formatVersion: 1, + executionThreadIds: [], + owner: { + threadId: "owner", + turnId: "scan-turn", + startedAt: timestamp, + }, + startedAt: timestamp, + completedAt: timestamp, + }, + }); + expect(result.sessions[0]?.path).toStartWith(first); + expect(result.events.map(({ event }) => event)).toEqual([ + { type: "session_meta", payload: { id: "owner" } }, + turn, + prefix, + suffix, + ]); + }, + ); + + test.each(["equal", "shorter", "divergent"])( + "keeps the first rollout when a later copy is %s", + async (copy) => { + const first = await temporaryHome(); + const second = await temporaryHome(); + const prefix = commandEvent("first", "first-call"); + const suffix = commandEvent("complete", "complete-call"); + await writeSession(first, "owner", [prefix, suffix]); + await writeSession( + second, + "owner", + copy === "equal" + ? [prefix, suffix] + : copy === "shorter" + ? [prefix] + : [prefix, commandEvent("different", "different-call"), suffix], + ); + const result = await readScanLogs({ + scanId: "scan-1", + threadId: "owner", + codexHome: [first, second], + }); + expect(result.sessions[0]?.path).toStartWith(first); + expect(result.events.map(({ event }) => event)).toEqual([ + { type: "session_meta", payload: { id: "owner" } }, + prefix, + suffix, + ]); + }, + ); + + test.each([ + ["current", "original"], + ["current", "substituted"], + ["current", "deleted"], + ["original", "original"], + ["original", "substituted"], + ["original", "deleted"], + ] as const)( + "reads recovered Deep workers with owner in %s home and %s artifact", + async (ownerHome, artifact) => { + const current = await temporaryHome(); + const original = await temporaryHome(); + const scanDir = await temporaryHome(); + const settingsDirectory = join(scanDir, "artifacts", "deep_discovery"); + await mkdir(settingsDirectory, { recursive: true }); + await writeFile( + join(settingsDirectory, "execution-settings.json"), + JSON.stringify({ + version: 1, + settings: { codexHome: original, codexPath: join(original, "codex") }, + }), + ); + const startedAt = "2026-08-11T12:00:00.000Z"; + const timestamp = "2026-08-11T12:01:00.000Z"; + await writeSession( + ownerHome === "current" ? current : original, + "owner", + [ + { + type: "turn_context", + timestamp, + payload: { turn_id: "scan-turn" }, + }, + commandEvent("scan owner", "owner-call", timestamp), + { + type: "turn_context", + timestamp, + payload: { turn_id: "later-turn" }, + }, + commandEvent("unrelated owner turn", "other-turn", timestamp), + ], + ); + for (const id of ["discovery", "resumed-discovery", "reducer"]) { + await writeSession(original, id, [commandEvent(id, id, timestamp)]); + } + await writeSession( + original, + "worker-child", + [commandEvent("worker child", "child-call", timestamp)], + "discovery", + ); + // Keep the existing first-home preference for duplicate active rollouts. + await writeSession(current, "reducer", [ + commandEvent("current reducer", "current-reducer-call", timestamp), + ]); + for (const home of [current, original]) { + await writeSession(home, "unrelated", [ + commandEvent("unrelated scan", "unrelated-call", timestamp), + ]); + } + const settingsPath = join(settingsDirectory, "execution-settings.json"); + if (artifact === "deleted") unlinkSync(settingsPath); + if (artifact === "substituted") + await writeFile( + settingsPath, + JSON.stringify({ version: 1, settings: { codexHome: current } }), + ); + const result = await readSavedScanLogs( + { + scanId: "scan-1", + mode: "deep", + scanDir, + continuationThreadId: "owner", + executionAttribution: { + formatVersion: 1, + workerCodexHome: original, + executionThreadIds: ["discovery", "resumed-discovery", "reducer"], + owner: { threadId: "owner", turnId: "scan-turn", startedAt }, + startedAt, + completedAt: "2026-08-11T12:02:00.000Z", + }, + }, + current, + ); + expect(result.sessions.map(({ threadId }) => threadId).sort()).toEqual([ + "discovery", + "owner", + "reducer", + "resumed-discovery", + "worker-child", + ]); + expect( + result.events.filter(({ threadId }) => threadId === "reducer"), + ).toHaveLength(2); + expect(JSON.stringify(result)).toContain("current reducer"); + expect(JSON.stringify(result)).not.toContain("unrelated"); + }, + ); + + test("reads the complete attributed worker copy despite differing inherited history", async () => { + const home = await temporaryHome(); + const current = await temporaryHome(); + await writeSession(home, "parent", []); + const startedAt = "2026-08-11T12:02:00.900Z"; + await writeSession( + home, + "worker", + [ + { + type: "session_meta", + payload: { id: "parent", timestamp: "2026-08-11T12:00:00.000Z" }, + }, + { + type: "event_msg", + payload: { + type: "task_started", + started_at: Date.parse("2026-08-11T12:00:00.000Z") / 1_000, + }, + }, + { + type: "event_msg", + payload: { + type: "agent_message", + message: "PRIVATE PRE-SCAN CONVERSATION", + }, + }, + { + type: "event_msg", + timestamp: startedAt, + payload: { + type: "task_started", + started_at: Math.floor(Date.parse(startedAt) / 1_000), + }, + }, + { + type: "event_msg", + timestamp: startedAt, + payload: { + type: "agent_message", + message: "Reviewing authorization", + }, + }, + ], + "parent", + startedAt, + ); + + const path = join( + home, + "sessions", + "2026", + "08", + "11", + "rollout-worker.jsonl", + ); + const currentPath = join(current, "sessions", "rollout-worker.jsonl"); + await mkdir(join(current, "sessions"), { recursive: true }); + await writeFile( + currentPath, + (await readFile(path, "utf8")) + .split("\n") + .slice(0, 4) + .join("\n") + .replace( + "PRIVATE PRE-SCAN CONVERSATION", + "OTHER PRE-SCAN CONVERSATION", + ), + ); + const result = await readScanLogs({ + scanId: "scan-1", + threadId: "parent", + codexHome: [current, home], + executionAttribution: { + formatVersion: 1, + executionThreadIds: ["worker"], + owner: { threadId: "parent", turnId: "scan-turn", startedAt }, + startedAt, + completedAt: null, + }, + }); + expect(JSON.stringify(result)).toContain("Reviewing authorization"); + expect(JSON.stringify(result)).not.toContain("PRIVATE PRE-SCAN"); + expect(JSON.stringify(result)).not.toContain("OTHER PRE-SCAN"); + expect( + result.sessions.find(({ threadId }) => threadId === "worker")?.path, + ).toBe(path); + }); + test.each([ ["prefix first", [0], [0, 1, 1, 2], 1, false, false], ["complete first", [0, 1, 1, 2], [0], 0, false, false], diff --git a/sdk/typescript/tests-ts/scan-resume.test.ts b/sdk/typescript/tests-ts/scan-resume.test.ts index 868b4005e6..c67baa0801 100644 --- a/sdk/typescript/tests-ts/scan-resume.test.ts +++ b/sdk/typescript/tests-ts/scan-resume.test.ts @@ -211,10 +211,16 @@ async function interruptedScan( const sessionPath = join(codexHome, "sessions", `rollout-${threadId}.jsonl`); await writeFile( sessionPath, - JSON.stringify({ - type: "session_meta", - payload: { id: threadId, cwd: scanDir }, - }) + "\n", + [ + { type: "session_meta", payload: { id: threadId, cwd: scanDir } }, + { + type: "turn_context", + timestamp: new Date().toISOString(), + payload: { turn_id: "synthetic-scan-turn", model: "gpt-5.6-sol" }, + }, + ] + .map((event) => JSON.stringify(event)) + .join("\n") + "\n", ); if (mode === "deep") { await command([ @@ -674,6 +680,7 @@ test.each([ f.sessionPath, JSON.stringify({ type: "event_msg", + timestamp: new Date().toISOString(), payload: { type: "token_count", info: { @@ -1297,6 +1304,104 @@ test("bulk Deep resume stages campaign knowledge after its source is removed", a ).toMatchObject({ knowledgeBasePaths: [document] }); }); +test.each([ + [false, "none", "deleted"], + [true, "none", "deleted"], + [false, "saved", "deleted"], + [false, "saved", "moved"], + [false, "saved", "available"], + [false, "modified", "available"], + [false, "missing", "available"], + [false, "missing", "deleted"], +] as const)( + "CLI restores selected finalization without current inputs or session logs (bulk: %p, knowledge: %s, checkout: %s)", + async (bulk, knowledge, checkout) => { + const document = join(await temporaryDirectory(), "architecture.md"); + await writeFile(document, "Original architecture."); + const f = await interruptedScan("deep", bulk, { + postScanPrompt: "Keep the saved follow-up behavior.", + ...(knowledge === "none" ? {} : { knowledgeBasePaths: [document] }), + }); + await writeFile(document, "Edited source context must not be loaded."); + if (knowledge === "modified") { + const snapshotPath = join(f.scanDir, ".scan-knowledge.json"); + const snapshot = JSON.parse(await readFile(snapshotPath, "utf8")); + snapshot.documents["0-architecture.md.txt"] = "Modified saved context."; + await writeFile(snapshotPath, JSON.stringify(snapshot)); + } else if (knowledge === "missing") { + await rm(join(f.scanDir, ".scan-knowledge.json")); + } + await rm(f.sessionPath); + if (checkout === "moved") + await rename(f.repository, `${f.repository}-moved`); + else if (checkout === "deleted") + await rm(f.repository, { recursive: true }); + const { stdout, stderr, runCli } = createCliTest(main); + const calls: { repository: string; options: ScanOptions }[] = []; + const deps = dependencies({ + environment: f.environment, + currentDirectory: f.root, + onTurn(repository, options) { + calls.push({ repository, options }); + }, + }); + const code = await runCli( + bulk + ? ["bulk-scan", f.input, "--output-dir", f.root, "--recover", "--json"] + : ["scans", "resume", f.scanId, "--json"], + { + ...deps, + runWorkbench: async (args, input) => + args[0] === "get-cli-scan-resume" + ? { + ...f.registration, + scanId: f.scanId, + scanDir: f.scanDir, + threadId: f.threadId, + selectedFinalization: true, + userContext: null, + recipe: { ...f.recipe, validationMode: "custom" }, + } + : f.command(args, input), + }, + ); + if (knowledge === "modified" || knowledge === "missing") { + expect(code, stderr.text()).toBe(2); + expect(calls).toHaveLength(0); + expect(stderr.text()).toContain( + knowledge === "modified" ? "snapshot changed" : ".scan-knowledge.json", + ); + return; + } + expect(code, stderr.text()).toBe(0); + expect(calls).toHaveLength(1); + if (knowledge === "saved" && checkout === "available") { + expect(calls[0]!.options.knowledgeBaseSnapshot?.documents).toEqual({ + "0-architecture.md.txt": "Original architecture.", + }); + } else { + expect(calls[0]!.options.knowledgeBaseSnapshot).toBeUndefined(); + } + expect(calls[0]).toMatchObject({ + repository: f.repository, + options: { + resumeScanId: f.scanId, + outputDir: f.scanDir, + postScanPrompt: "Keep the saved follow-up behavior.", + }, + }); + expect(calls[0]!.options.scanPrompt).toBeUndefined(); + expect(calls[0]!.options.validationPrompt).toBeUndefined(); + if (bulk) { + const result = JSON.parse(stdout.text()); + const receipts = await readJsonLines(result.resultsPath); + expect(receipts).toHaveLength(2); + expect(receipts[1]).toMatchObject({ attempt: 1, outputDir: f.scanDir }); + expect(stderr.text()).not.toContain("starting a new one"); + } + }, +); + test("missing session logs do not create another session or fail the original scan", async () => { const f = await interruptedScan(); await rm(f.sessionPath); diff --git a/sdk/typescript/tests-ts/scan-usage-reconciliation.test.ts b/sdk/typescript/tests-ts/scan-usage-reconciliation.test.ts new file mode 100644 index 0000000000..628e9f08fd --- /dev/null +++ b/sdk/typescript/tests-ts/scan-usage-reconciliation.test.ts @@ -0,0 +1,856 @@ +import { describe, expect, test } from "bun:test"; +import { appendFile, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { estimateScanCost } from "../src/cost-model.js"; +import { ScanCostTracker } from "../src/cost.js"; +import { readScanLogs } from "../src/scan-logs.js"; +import type { ScanExecutionAttribution } from "../src/scan-sessions.js"; + +describe("scan usage reconciliation", () => { + test("SDK usage and logs share attempt membership and the original owner turn", async () => { + const home = await mkdtemp(join(tmpdir(), "usage-attribution-")); + const observed: unknown[] = []; + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + onSessionEvent: (event) => observed.push(event), + }); + const at = (second: number) => + `2026-09-01T00:00:${String(second).padStart(2, "0")}Z`; + const attribution: ScanExecutionAttribution = { + formatVersion: 1, + executionThreadIds: ["old-worker", "replacement-worker"], + owner: { threadId: "parent", turnId: "scan-turn", startedAt: at(1) }, + startedAt: at(1), + completedAt: at(10), + }; + const token = (second: number, count: number) => ({ + timestamp: at(second), + type: "event_msg", + payload: { + type: "token_count", + info: { total_token_usage: { input_tokens: count, output_tokens: 0 } }, + }, + }); + const context = (second: number, turn: string) => ({ + timestamp: at(second), + type: "turn_context", + payload: { turn_id: turn, model: "gpt-5.6-sol" }, + }); + try { + await mkdir(join(home, "sessions")); + for (const [id, parent, events] of [ + [ + "parent", + null, + [ + context(0, "prior-turn"), + token(0, 100), + context(1, "scan-turn"), + token(2, 110), + context(3, "unrelated-turn"), + token(4, 1010), + ], + ], + [ + "old-worker", + null, + [context(1, "worker-turn"), token(2, 20), token(11, 120)], + ], + ["replacement-worker", null, [context(3, "worker-turn"), token(4, 30)]], + ["worker-child", "old-worker", [context(3, "child-turn"), token(4, 5)]], + ["unrelated-child", "parent", [context(3, "side-turn"), token(4, 900)]], + ] as const) { + const records = [ + { + type: "session_meta", + payload: { id, ...(parent ? { parent_thread_id: parent } : {}) }, + }, + ...events, + ]; + await writeFile( + join(home, "sessions", `${id}.jsonl`), + records.map((value) => JSON.stringify(value)).join("\n") + "\n", + ); + } + tracker.setAttributionReader(async () => attribution); + tracker.start("parent"); + const snapshot = await tracker.stop(); + expect(snapshot.cost?.inputTokens).toBe(65); + expect(JSON.stringify(observed)).not.toContain("unrelated-turn"); + expect(JSON.stringify(observed)).not.toContain(at(11)); + const logs = await readScanLogs({ + scanId: "scan", + threadId: "parent", + codexHome: home, + executionAttribution: attribution, + }); + expect(logs.sessions.map((session) => session.threadId).sort()).toEqual([ + "old-worker", + "parent", + "replacement-worker", + "worker-child", + ]); + expect( + logs.events.some(({ event }) => + JSON.stringify(event).includes("unrelated-turn"), + ), + ).toBe(false); + expect( + logs.events.some(({ event }) => JSON.stringify(event).includes(at(11))), + ).toBe(false); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } + }); + + test("retains known usage before attribution and uncertainty until missing attempt usage arrives", async () => { + const home = await mkdtemp(join(tmpdir(), "usage-delayed-")); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + let attribution: ScanExecutionAttribution | null = null; + tracker.setAttributionReader(async () => attribution); + const at = "2026-09-01T00:00:02Z"; + const records = (id: string, count: number) => + [ + { type: "session_meta", payload: { id } }, + { + timestamp: at, + type: "turn_context", + payload: { model: "gpt-5.6-sol", turn_id: "own" }, + }, + { + timestamp: at, + type: "event_msg", + payload: { + type: "token_count", + info: { + total_token_usage: { input_tokens: count, output_tokens: 0 }, + }, + }, + }, + ] + .map((event) => JSON.stringify(event)) + .join("\n") + "\n"; + try { + await mkdir(join(home, "sessions")); + await writeFile( + join(home, "sessions", "worker.jsonl"), + records("worker", 20), + ); + tracker.start("worker"); + tracker.recordUsage({ input_tokens: 20, output_tokens: 0 }); + expect((await tracker.refresh()).cost).toMatchObject({ inputTokens: 20 }); + attribution = { + formatVersion: 1, + executionThreadIds: ["worker", "failed-attempt"], + owner: { threadId: "worker", turnId: "own", startedAt: at }, + startedAt: "2026-09-01T00:00:01Z", + completedAt: "2026-09-01T00:00:10Z", + }; + expect((await tracker.refresh()).cost).toMatchObject({ + inputTokens: 20, + coverage: "partial", + }); + await writeFile( + join(home, "sessions", "failed.jsonl"), + records("failed-attempt", 5), + ); + const final = await tracker.stop(); + expect(final.cost?.inputTokens).toBe(25); + expect(final.cost?.coverage).toBeUndefined(); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } + }); + + test("preserves receipt accounting for a resumed legacy Deep scan", async () => { + const home = await mkdtemp(join(tmpdir(), "usage-legacy-")); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + tracker.setAttributionReader(async () => ({ + formatVersion: 1, + legacy: true, + executionThreadIds: ["legacy-parent"], + owner: { + threadId: "legacy-parent", + turnId: null, + startedAt: "2026-09-01T00:00:00Z", + }, + startedAt: "2026-09-01T00:00:00Z", + completedAt: null, + })); + try { + tracker.start("legacy-parent"); + const snapshot = await tracker.stop({ + input_tokens: 10000, + output_tokens: 100, + }); + expect(snapshot.cost?.inputTokens).toBe(10000); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } + }); + + test("prices each observed model instead of repricing the sum with the parent", () => { + const usage = { + input_tokens: 200, + output_tokens: 20, + modelUsage: [ + { model: "gpt-5.6-sol", input_tokens: 100, output_tokens: 10 }, + { model: "gpt-6-astra", input_tokens: 100, output_tokens: 10 }, + ], + }; + const expected = + estimateScanCost("gpt-5.6-sol", usage.modelUsage[0])!.estimatedUsd + + estimateScanCost("gpt-6-astra", usage.modelUsage[1])!.estimatedUsd; + expect(estimateScanCost("gpt-5.6-sol", usage)?.estimatedUsd).toBe(expected); + }); + + test("keeps incomplete model attribution unpriced", () => { + expect( + estimateScanCost("gpt-5.6-sol", { + input_tokens: 200, + output_tokens: 20, + modelUsage: [ + { model: "gpt-5.6-sol", input_tokens: 100, output_tokens: 10 }, + { model: null, input_tokens: 100, output_tokens: 10 }, + ], + }), + ).toBeNull(); + }); + + test("reconciles stale and missing cumulative receipts without reducing usage", async () => { + const home = await mkdtemp(join(tmpdir(), "usage-receipts-")); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + try { + tracker.start("worker"); + tracker.recordUsage({ input_tokens: 160, output_tokens: 0 }); + tracker.recordUsage({ input_tokens: 100, output_tokens: 0 }); + tracker.recordUsage(null); + expect((await tracker.stop()).cost?.inputTokens).toBe(160); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } + }); + + test("tracks per-model deltas within one resumed session", async () => { + const home = await mkdtemp(join(tmpdir(), "usage-models-")); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + try { + await mkdir(join(home, "sessions")); + const records = [ + { type: "session_meta", payload: { id: "worker" } }, + { + type: "turn_context", + payload: { model: "gpt-5.6-sol", turn_id: "turn-1" }, + }, + { + type: "event_msg", + payload: { + type: "token_count", + info: { + total_token_usage: { input_tokens: 100, output_tokens: 10 }, + }, + }, + }, + { + type: "event_msg", + payload: { + type: "token_count", + info: { total_token_usage: { input_tokens: 60, output_tokens: 6 } }, + }, + }, + { + type: "turn_context", + payload: { model: "gpt-6-astra", turn_id: "turn-2" }, + }, + { + type: "event_msg", + payload: { + type: "token_count", + info: { + total_token_usage: { input_tokens: 200, output_tokens: 20 }, + }, + }, + }, + ]; + await writeFile( + join(home, "sessions", "worker.jsonl"), + records.map((record) => JSON.stringify(record)).join("\n") + "\n", + ); + tracker.start("worker"); + tracker.recordUsage({ input_tokens: 200, output_tokens: 20 }); + const snapshot = await tracker.stop(); + expect(snapshot.cost?.inputTokens).toBe(200); + expect(snapshot.cost?.estimatedUsd).toBeCloseTo(0.0021, 12); + expect(snapshot.cost?.modelCosts?.map((cost) => cost.model)).toEqual([ + "gpt-5.6-sol", + "gpt-6-astra", + ]); + tracker.recordUsage({ input_tokens: 250, output_tokens: 25 }); + expect((await tracker.refresh()).cost).toBeNull(); + await appendFile( + join(home, "sessions", "worker.jsonl"), + JSON.stringify({ + type: "event_msg", + payload: { + type: "token_count", + info: { + total_token_usage: { input_tokens: 250, output_tokens: 25 }, + }, + }, + }) + "\n", + ); + expect((await tracker.refresh()).cost?.estimatedUsd).toBeCloseTo( + 0.00285, + 12, + ); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } + }); +}); + +describe("charged response receipts", () => { + for (const exactOnly of [false, true]) { + test(`counts compaction and deduplicates responses across counter resets (exact only: ${exactOnly})`, async () => { + const home = await mkdtemp(join(tmpdir(), "usage-response-receipts-")); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + const usage = (input: number, cached: number, output: number) => ({ + input_tokens: input, + cached_input_tokens: cached, + cache_write_input_tokens: 0, + output_tokens: output, + reasoning_output_tokens: 0, + total_tokens: input + output, + }); + const record = ( + id: string, + count: unknown, + cumulative: unknown, + model = "gpt-5.6-sol", + ) => ({ + type: "token_usage_record", + payload: { + thread_id: "worker", + turn_id: "turn", + response_id: id, + model, + usage: count, + thread_token_usage: cumulative, + }, + }); + const first = record("normal-1", usage(100, 80, 10), usage(100, 80, 10)); + const compact = record( + "compaction", + usage(50, 40, 5), + usage(150, 120, 15), + "gpt-6-astra", + ); + const second = record("normal-2", usage(120, 90, 12), usage(120, 90, 12)); + const counter = (count: unknown) => ({ + type: "event_msg", + payload: { + type: "token_count", + info: { total_token_usage: count }, + }, + }); + const events = [ + { type: "session_meta", payload: { id: "worker" } }, + first, + ...(!exactOnly ? [counter(usage(100, 80, 10))] : []), + compact, + { + type: "compacted", + payload: { message: "Synthetic context summary" }, + }, + compact, + second, + ...(!exactOnly ? [counter(usage(220, 170, 22))] : []), + first, + ]; + try { + await mkdir(join(home, "sessions")); + await writeFile( + join(home, "sessions", "worker.jsonl"), + events.map((e) => JSON.stringify(e)).join("\n") + "\n", + ); + tracker.start("worker"); + tracker.recordUsage(usage(220, 170, 22)); + const result = await tracker.stop(); + expect(result.usage).toMatchObject(usage(270, 210, 27)); + expect( + result.cost?.modelCosts?.map((part) => [ + part.model, + part.inputTokens + part.outputTokens, + ]), + ).toEqual([ + ["gpt-5.6-sol", 242], + ["gpt-6-astra", 55], + ]); + expect((await tracker.refresh()).usage).toEqual(result.usage); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } + }); + } + + test("uses receipt turn identity for shared-parent usage and logs", async () => { + const home = await mkdtemp(join(tmpdir(), "usage-response-owner-")); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + const attribution: ScanExecutionAttribution = { + formatVersion: 1, + executionThreadIds: [], + owner: { + threadId: "parent", + turnId: "scan-turn", + startedAt: "2026-09-01T00:00:01Z", + }, + startedAt: "2026-09-01T00:00:01Z", + completedAt: "2026-09-01T00:00:10Z", + }; + const receipt = ( + id: string, + turn: string, + second: string, + input: number, + ) => ({ + type: "token_usage_record", + timestamp: `2026-09-01T00:00:${second}Z`, + payload: { + response_id: id, + thread_id: "parent", + turn_id: turn, + model: "gpt-5.6-sol", + usage: { input_tokens: input, output_tokens: 0 }, + }, + }); + try { + await mkdir(join(home, "sessions")); + await writeFile( + join(home, "sessions", "parent.jsonl"), + [ + { type: "session_meta", payload: { id: "parent" } }, + receipt("prior", "prior-turn", "00", 900), + receipt("owned", "scan-turn", "02", 20), + receipt("side", "other-turn", "03", 800), + receipt("post", "scan-turn", "11", 700), + ] + .map((e) => JSON.stringify(e)) + .join("\n") + "\n", + ); + tracker.setAttributionReader(async () => attribution); + tracker.start("parent"); + expect((await tracker.stop()).cost?.inputTokens).toBe(20); + const logs = await readScanLogs({ + scanId: "scan", + threadId: "parent", + codexHome: home, + executionAttribution: attribution, + }); + const ids = logs.events + .map( + ({ event }) => + (event as { payload?: Record })["payload"]?.[ + "response_id" + ], + ) + .filter(Boolean); + expect(ids).toEqual(["owned"]); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } + }); +}); + +test("delayed response receipts resolve cumulative gaps without treating smaller counters as stale", async () => { + const home = await mkdtemp(join(tmpdir(), "usage-delayed-response-")); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + const record = (id: string, tokens: number, cumulative: number) => + JSON.stringify({ + type: "token_usage_record", + payload: { + response_id: id, + thread_id: "worker", + model: "gpt-5.6-sol", + usage: { input_tokens: tokens, output_tokens: 0 }, + thread_token_usage: { input_tokens: cumulative, output_tokens: 0 }, + }, + }) + "\n"; + try { + await mkdir(join(home, "sessions")); + const file = join(home, "sessions", "worker.jsonl"); + await writeFile( + file, + JSON.stringify({ type: "session_meta", payload: { id: "worker" } }) + + "\n" + + record("first", 100, 100) + + record("third", 50, 180), + ); + tracker.start("worker"); + expect((await tracker.refresh()).cost).toMatchObject({ + inputTokens: 150, + coverage: "partial", + }); + await appendFile(file, record("second", 30, 130)); + const result = await tracker.stop(); + expect(result.cost?.inputTokens).toBe(180); + expect(result.cost?.coverage).toBeUndefined(); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } +}); + +test("a reader installed before the native attribution writer preserves legacy receipts", async () => { + const home = await mkdtemp(join(tmpdir(), "usage-reader-first-")); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + try { + tracker.setAttributionReader(async () => undefined); + tracker.start("parent"); + tracker.recordUsage({ input_tokens: 100, output_tokens: 0 }); + expect((await tracker.stop()).cost?.inputTokens).toBe(100); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } +}); + +test("late exact receipts replace an overlapping legacy counter without adding it twice", async () => { + const home = await mkdtemp(join(tmpdir(), "usage-overlap-")); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + const usage = (count: number) => ({ input_tokens: count, output_tokens: 0 }); + const receipt = (id: string, count: number, cumulative: number) => ({ + type: "token_usage_record", + payload: { + thread_id: "worker", + response_id: id, + model: "gpt-5.6-sol", + usage: usage(count), + thread_token_usage: usage(cumulative), + }, + }); + try { + await mkdir(join(home, "sessions")); + await writeFile( + join(home, "sessions", "worker.jsonl"), + [ + { type: "session_meta", payload: { id: "worker" } }, + { + type: "event_msg", + payload: { + type: "token_count", + info: { total_token_usage: usage(100) }, + }, + }, + receipt("new", 10, 110), + receipt("old", 100, 100), + { + type: "event_msg", + payload: { + type: "token_count", + info: { total_token_usage: usage(10) }, + }, + }, + ] + .map((e) => JSON.stringify(e)) + .join("\n") + "\n", + ); + tracker.start("worker"); + const result = await tracker.stop(); + expect(result.cost?.inputTokens).toBe(110); + expect(result.cost?.coverage).toBeUndefined(); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } +}); + +test.each([ + [false, false], + [false, true], + [true, false], + [true, true], +] as const)( + "preserves receipt gaps across compaction (attributed: %p, repaired before reset: %p)", + async (attributed, repaired) => { + const home = await mkdtemp(join(tmpdir(), "usage-compaction-gap-")); + const trackers: ScanCostTracker[] = []; + const usage = (tokens: number) => ({ + input_tokens: tokens, + output_tokens: 0, + }); + const receipt = ( + thread: string, + id: string, + tokens: number, + cumulative: number, + ) => ({ + type: "token_usage_record", + timestamp: "2026-09-01T00:00:02Z", + payload: { + thread_id: thread, + turn_id: `${thread}-turn`, + response_id: id, + model: "gpt-5.6-sol", + usage: usage(tokens), + thread_token_usage: usage(cumulative), + }, + }); + const compacted = { + type: "compacted", + payload: { message: "Synthetic summary" }, + }; + const lines = (events: unknown[]) => + events.map((event) => JSON.stringify(event)).join("\n") + "\n"; + const startTracker = () => { + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + maxCostUsd: 0.0007, + }); + if (attributed) + tracker.setAttributionReader(async () => ({ + formatVersion: 1, + executionThreadIds: ["child"], + owner: { + threadId: "parent", + turnId: "parent-turn", + startedAt: "2026-09-01T00:00:00Z", + }, + startedAt: "2026-09-01T00:00:00Z", + completedAt: null, + })); + trackers.push(tracker); + tracker.start("parent"); + tracker.recordUsage(usage(10)); + return tracker; + }; + try { + await mkdir(join(home, "sessions")); + await writeFile( + join(home, "sessions", "parent.jsonl"), + lines([ + { type: "session_meta", payload: { id: "parent" } }, + receipt("parent", "parent-receipt", 10, 10), + ]), + ); + const childPath = join(home, "sessions", "child.jsonl"); + const first = receipt("child", "before-compaction", 50, 100); + await writeFile( + childPath, + lines([ + { + type: "session_meta", + payload: { id: "child", parent_thread_id: "parent" }, + }, + first, + ]), + ); + const tracker = startTracker(); + expect((await tracker.refresh()).cost).toMatchObject({ + inputTokens: 60, + coverage: "partial", + }); + if (repaired) { + await appendFile( + childPath, + lines([receipt("child", "delayed-before-compaction", 50, 50)]), + ); + expect((await tracker.refresh()).cost?.coverage).toBeUndefined(); + } + const second = receipt("child", "after-compaction", 100, 100); + await appendFile(childPath, lines([compacted, first, second])); + const snapshot = await tracker.refresh(); + expect(snapshot.cost?.inputTokens).toBe(repaired ? 210 : 160); + expect(snapshot.cost?.coverage).toBe(repaired ? undefined : "partial"); + if (repaired) expect(snapshot.usage).not.toHaveProperty("coverage"); + else expect(snapshot.usage).toMatchObject({ coverage: "partial" }); + if (!repaired) { + expect(snapshot.cost!.estimatedUsd).toBeLessThan(0.0007); + expect( + estimateScanCost("gpt-5.6-sol", usage(210))!.estimatedUsd, + ).toBeGreaterThan(0.0007); + } + await appendFile( + childPath, + lines([compacted, second, receipt("child", "later", 20, 20)]), + ); + const final = await tracker.stop(); + expect(final.cost?.inputTokens).toBe(repaired ? 230 : 180); + expect(final.cost?.coverage).toBe(repaired ? undefined : "partial"); + const reloaded = await startTracker().stop(); + expect(reloaded.usage).toEqual(final.usage); + expect(reloaded.cost).toEqual(final.cost); + } finally { + await Promise.all(trackers.map((tracker) => tracker.stop())); + await rm(home, { recursive: true, force: true }); + } + }, +); + +test.each([ + ["before", "time"], + ["before", "turn"], + ["after", "time"], + ["after", "turn"], +] as const)( + "unowned receipt epochs do not taint scan coverage (%s, %s)", + async (position, exclusion) => { + const home = await mkdtemp(join(tmpdir(), "usage-compaction-scope-")); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + const at = (second: string) => `2026-09-01T00:00:${second}Z`; + const receipt = ( + id: string, + tokens: number, + turn: string, + timestamp: string, + ) => ({ + type: "token_usage_record", + timestamp, + payload: { + thread_id: "worker", + turn_id: turn, + response_id: id, + model: "gpt-5.6-sol", + usage: { input_tokens: tokens, output_tokens: 0 }, + thread_token_usage: { input_tokens: 100, output_tokens: 0 }, + }, + }); + const owned = receipt("owned", 100, "scan-turn", at("02")); + const ignored = receipt( + "ignored", + 50, + exclusion === "turn" ? "other-turn" : "scan-turn", + exclusion === "time" ? at(position === "before" ? "00" : "11") : at("02"), + ); + tracker.setAttributionReader(async () => ({ + formatVersion: 1, + executionThreadIds: [], + owner: { threadId: "worker", turnId: "scan-turn", startedAt: at("01") }, + startedAt: at("01"), + completedAt: at("10"), + })); + try { + await mkdir(join(home, "sessions")); + await writeFile( + join(home, "sessions", "worker.jsonl"), + [ + { type: "session_meta", payload: { id: "worker" } }, + position === "before" ? ignored : owned, + { type: "compacted", payload: { message: "Synthetic summary" } }, + position === "before" ? owned : ignored, + ] + .map((event) => JSON.stringify(event)) + .join("\n") + "\n", + ); + tracker.start("worker"); + const snapshot = await tracker.stop(); + expect(snapshot.cost?.inputTokens).toBe(100); + expect(snapshot.cost?.coverage).toBeUndefined(); + expect(snapshot.usage).not.toHaveProperty("coverage"); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } + }, +); + +test.each([false, true])( + "keeps priced legacy history under a native cumulative floor (later counter: %p)", + async (laterCounter) => { + const home = await mkdtemp(join(tmpdir(), "usage-resumed-floor-")); + const lowerBounds: number[] = []; + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + onCostLowerBound: (cost) => lowerBounds.push(cost.estimatedUsd), + }); + const usage = (input: number, output: number) => ({ + input_tokens: input, + output_tokens: output, + }); + const counter = (input: number, output: number) => ({ + type: "event_msg", + payload: { + type: "token_count", + info: { total_token_usage: usage(input, output) }, + }, + }); + try { + await mkdir(join(home, "sessions")); + await writeFile( + join(home, "sessions", "worker.jsonl"), + [ + { + type: "session_meta", + payload: { id: "worker", model: "gpt-5.6-sol" }, + }, + counter(1000, 100), + { + type: "token_usage_record", + payload: { + thread_id: "worker", + response_id: "new-response", + model: "gpt-5.6-sol", + usage: usage(20, 2), + thread_token_usage: usage(1020, 102), + }, + }, + ...(laterCounter ? [counter(1010, 101)] : []), + ] + .map((event) => JSON.stringify(event)) + .join("\n") + "\n", + ); + tracker.start("worker"); + const result = await tracker.stop(); + expect(result.usage).toMatchObject({ total_tokens: 1122 }); + expect(lowerBounds.length).toBeGreaterThan(0); + const knownHistory = estimateScanCost( + "gpt-5.6-sol", + usage(laterCounter ? 1010 : 1000, laterCounter ? 101 : 100), + ); + expect(lowerBounds.at(-1)).toBeCloseTo(knownHistory!.estimatedUsd, 12); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } + }, +);